From ec3118e7bc1c63f004516e12dae488056dee8f7f Mon Sep 17 00:00:00 2001
From: Ryan Salazar <ryaninwilton@gmail.com>
Date: Wed, 27 May 2026 19:57:26 +0000
Subject: [PATCH] rewrite: dual-endpoint Ollama failover (AI_001 + AI_002)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Was: single OLLAMA_ENDPOINT — when AI_002 saturated, every rewrite job
backed off in place waiting for it to drain.

Now: tries OLLAMA_ENDPOINT (AI_002) first, instantly falls over to
OLLAMA_FALLBACK_ENDPOINT (AI_001) on 5xx/timeout before burning a
backoff sleep. Both nodes have identical model sets (llama3.1:70b,
llama3, qwen2.5-coder, etc.) so behavior is consistent across either.

Net effect: 2× GPU capacity, lower P95 latency, rewrites stop bunching
on a single saturated queue. The bb-forum-burst script already had this
pattern — pulling it into the BB rewrite path closes the gap.

OLLAMA_FALLBACK_ENDPOINT=http://10.10.0.66:11434 already set on the BB
Coolify app env.
---
 src/lib/ai/rewrite-client.ts | 76 +++++++++++++++++++-----------------
 1 file changed, 40 insertions(+), 36 deletions(-)

diff --git a/src/lib/ai/rewrite-client.ts b/src/lib/ai/rewrite-client.ts
index fec7cbd..0b383e7 100644
--- a/src/lib/ai/rewrite-client.ts
+++ b/src/lib/ai/rewrite-client.ts
@@ -153,8 +153,10 @@ export const ollamaProvider: RewriteProvider = {
   name: "ollama",
 
   async rewrite(req: RewriteRequest): Promise<RewriteResult> {
-    const endpoint = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, "");
-    if (!endpoint) throw new Error("OLLAMA_ENDPOINT not set in env");
+    const primary = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, "");
+    if (!primary) throw new Error("OLLAMA_ENDPOINT not set in env");
+    const fallback = (process.env.OLLAMA_FALLBACK_ENDPOINT || "").replace(/\/$/, "");
+    const endpoints = [primary, fallback].filter(Boolean);
     const model =
       process.env.BB_REWRITE_MODEL ||
       process.env.OLLAMA_MODEL_DEFAULT ||
@@ -196,45 +198,47 @@ export const ollamaProvider: RewriteProvider = {
     const apiKey = process.env.OLLAMA_API_KEY;
     if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
 
-    // Retry-with-backoff against Ollama 503 / "server busy" / fetch failures.
-    // The single AI_002 node is shared with the ask-bb-ai + forum-burst
-    // workloads, so pending-request saturation is common — burning 3 attempts
-    // in 0.5s (as the outer orchestrator does) doesn't give it room to drain.
-    const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s
+    // Retry-with-backoff + multi-endpoint failover. We try primary first;
+    // if it returns 503/busy/timeout, we immediately try the fallback before
+    // burning a backoff sleep. Each round trip through all endpoints counts
+    // as one "attempt" against RETRY_DELAYS_MS. Net effect: when AI_002 is
+    // saturated, AI_001 picks up the slack with no delay.
+    const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // between rounds
     let res: Response | null = null;
     let lastErr = "";
-    for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
+    outer: for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
       if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i]));
-      try {
-        res = await fetch(`${endpoint}/v1/chat/completions`, {
-          method: "POST",
-          headers,
-          body: JSON.stringify({
-            model,
-            messages: [
-              { role: "system", content: systemBlock },
-              { role: "user", content: userPrompt },
-            ],
-            stream: false,
-            response_format: { type: "json_object" },
-            temperature: 0.6,
-            max_tokens: MAX_OUTPUT_TOKENS,
-          }),
-          signal: AbortSignal.timeout(180_000),
-        });
-        if (res.ok) break;
-        const txt = await res.text().catch(() => "");
-        lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`;
-        // Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent.
-        if (res.status >= 400 && res.status < 500 && res.status !== 429) {
-          throw new Error(lastErr);
+      for (const endpoint of endpoints) {
+        try {
+          res = await fetch(`${endpoint}/v1/chat/completions`, {
+            method: "POST",
+            headers,
+            body: JSON.stringify({
+              model,
+              messages: [
+                { role: "system", content: systemBlock },
+                { role: "user", content: userPrompt },
+              ],
+              stream: false,
+              response_format: { type: "json_object" },
+              temperature: 0.6,
+              max_tokens: MAX_OUTPUT_TOKENS,
+            }),
+            signal: AbortSignal.timeout(180_000),
+          });
+          if (res.ok) break outer;
+          const txt = await res.text().catch(() => "");
+          lastErr = `Ollama API ${res.status} @ ${endpoint}: ${txt.slice(0, 400)}`;
+          // 4xx other than 429 is a permanent error — no point hitting fallback
+          if (res.status >= 400 && res.status < 500 && res.status !== 429) {
+            throw new Error(lastErr);
+          }
+          // Retryable — fall through to next endpoint within the same round
+        } catch (e: any) {
+          lastErr = `${e?.message || String(e)} @ ${endpoint}`;
         }
-        if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break;
-      } catch (e: any) {
-        lastErr = e?.message || String(e);
-        // network / abort — retry
       }
-      if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries");
+      if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries across all endpoints");
     }
     if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed");