localadministrator/avbeat-com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite: dual-endpoint Ollama failover (AI_001 + AI_002)

Browse Source 
Was: single OLLAMA_ENDPOINT — when AI_002 saturated, every rewrite job
backed off in place waiting for it to drain.

Now: tries OLLAMA_ENDPOINT (AI_002) first, instantly falls over to
OLLAMA_FALLBACK_ENDPOINT (AI_001) on 5xx/timeout before burning a
backoff sleep. Both nodes have identical model sets (llama3.1:70b,
llama3, qwen2.5-coder, etc.) so behavior is consistent across either.

Net effect: 2× GPU capacity, lower P95 latency, rewrites stop bunching
on a single saturated queue. The bb-forum-burst script already had this
pattern — pulling it into the BB rewrite path closes the gap.

OLLAMA_FALLBACK_ENDPOINT=http://10.10.0.66:11434 already set on the BB
Coolify app env.
This commit is contained in:
Ryan Salazar
2026-05-27 19:57:26 +00:00
parent 61910d6c8f
commit ec3118e7bc
1 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/lib/ai/rewrite-client.ts
View File

@@ -153,8 +153,10 @@ export const ollamaProvider: RewriteProvider = {
name: "ollama", name: "ollama",
async rewrite(req: RewriteRequest): Promise<RewriteResult> { async rewrite(req: RewriteRequest): Promise<RewriteResult> {
const endpoint = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, ""); const primary = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, "");
if (!endpoint) throw new Error("OLLAMA_ENDPOINT not set in env"); if (!primary) throw new Error("OLLAMA_ENDPOINT not set in env");
const fallback = (process.env.OLLAMA_FALLBACK_ENDPOINT || "").replace(/\/$/, "");
const endpoints = [primary, fallback].filter(Boolean);
const model = const model =
process.env.BB_REWRITE_MODEL || process.env.BB_REWRITE_MODEL ||
process.env.OLLAMA_MODEL_DEFAULT || process.env.OLLAMA_MODEL_DEFAULT ||
@@ -196,15 +198,17 @@ export const ollamaProvider: RewriteProvider = {
const apiKey = process.env.OLLAMA_API_KEY; const apiKey = process.env.OLLAMA_API_KEY;
if (apiKey) headers["authorization"] = `Bearer ${apiKey}`; if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
// Retry-with-backoff against Ollama 503 / "server busy" / fetch failures. // Retry-with-backoff + multi-endpoint failover. We try primary first;
// The single AI_002 node is shared with the ask-bb-ai + forum-burst // if it returns 503/busy/timeout, we immediately try the fallback before
// workloads, so pending-request saturation is common — burning 3 attempts // burning a backoff sleep. Each round trip through all endpoints counts
// in 0.5s (as the outer orchestrator does) doesn't give it room to drain. // as one "attempt" against RETRY_DELAYS_MS. Net effect: when AI_002 is
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s // saturated, AI_001 picks up the slack with no delay.
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // between rounds
let res: Response | null = null; let res: Response | null = null;
let lastErr = ""; let lastErr = "";
for (let i = 0; i < RETRY_DELAYS_MS.length; i++) { outer: for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i])); if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i]));
for (const endpoint of endpoints) {
try { try {
res = await fetch(`${endpoint}/v1/chat/completions`, { res = await fetch(`${endpoint}/v1/chat/completions`, {
method: "POST", method: "POST",
@@ -222,19 +226,19 @@ export const ollamaProvider: RewriteProvider = {
}), }),
signal: AbortSignal.timeout(180_000), signal: AbortSignal.timeout(180_000),
}); });
if (res.ok) break; if (res.ok) break outer;
const txt = await res.text().catch(() => ""); const txt = await res.text().catch(() => "");
lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`; lastErr = `Ollama API ${res.status} @ ${endpoint}: ${txt.slice(0, 400)}`;
// Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent. // 4xx other than 429 is a permanent error — no point hitting fallback
if (res.status >= 400 && res.status < 500 && res.status !== 429) { if (res.status >= 400 && res.status < 500 && res.status !== 429) {
throw new Error(lastErr); throw new Error(lastErr);
} }
if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break; // Retryable — fall through to next endpoint within the same round
} catch (e: any) { } catch (e: any) {
lastErr = e?.message || String(e); lastErr = `${e?.message || String(e)} @ ${endpoint}`;
// network / abort — retry
} }
if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries"); }
if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries across all endpoints");
} }
if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed"); if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed");