localadministrator/avbeat-com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite: dual-endpoint Ollama failover (AI_001 + AI_002)

Browse Source 
Was: single OLLAMA_ENDPOINT — when AI_002 saturated, every rewrite job
backed off in place waiting for it to drain.

Now: tries OLLAMA_ENDPOINT (AI_002) first, instantly falls over to
OLLAMA_FALLBACK_ENDPOINT (AI_001) on 5xx/timeout before burning a
backoff sleep. Both nodes have identical model sets (llama3.1:70b,
llama3, qwen2.5-coder, etc.) so behavior is consistent across either.

Net effect: 2× GPU capacity, lower P95 latency, rewrites stop bunching
on a single saturated queue. The bb-forum-burst script already had this
pattern — pulling it into the BB rewrite path closes the gap.

OLLAMA_FALLBACK_ENDPOINT=http://10.10.0.66:11434 already set on the BB
Coolify app env.
This commit is contained in:
Ryan Salazar
2026-05-27 19:57:26 +00:00
parent 61910d6c8f
commit ec3118e7bc
1 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/lib/ai/rewrite-client.ts
View File

@@ -153,8 +153,10 @@ export const ollamaProvider: RewriteProvider = {
name: "ollama", name: "ollama",
async rewrite(req: RewriteRequest): Promise<RewriteResult> { async rewrite(req: RewriteRequest): Promise<RewriteResult> {
const endpoint = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, ""); const primary = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, "");
if (!endpoint) throw new Error("OLLAMA_ENDPOINT not set in env"); if (!primary) throw new Error("OLLAMA_ENDPOINT not set in env");
const fallback = (process.env.OLLAMA_FALLBACK_ENDPOINT || "").replace(/\/$/, "");
const endpoints = [primary, fallback].filter(Boolean);
const model = const model =
process.env.BB_REWRITE_MODEL || process.env.BB_REWRITE_MODEL ||
process.env.OLLAMA_MODEL_DEFAULT || process.env.OLLAMA_MODEL_DEFAULT ||
@@ -196,45 +198,47 @@ export const ollamaProvider: RewriteProvider = {
const apiKey = process.env.OLLAMA_API_KEY; const apiKey = process.env.OLLAMA_API_KEY;
if (apiKey) headers["authorization"] = `Bearer ${apiKey}`; if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
// Retry-with-backoff against Ollama 503 / "server busy" / fetch failures. // Retry-with-backoff + multi-endpoint failover. We try primary first;
// The single AI_002 node is shared with the ask-bb-ai + forum-burst // if it returns 503/busy/timeout, we immediately try the fallback before
// workloads, so pending-request saturation is common — burning 3 attempts // burning a backoff sleep. Each round trip through all endpoints counts
// in 0.5s (as the outer orchestrator does) doesn't give it room to drain. // as one "attempt" against RETRY_DELAYS_MS. Net effect: when AI_002 is
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s // saturated, AI_001 picks up the slack with no delay.
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // between rounds
let res: Response | null = null; let res: Response | null = null;
let lastErr = ""; let lastErr = "";
for (let i = 0; i < RETRY_DELAYS_MS.length; i++) { outer: for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i])); if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i]));
try { for (const endpoint of endpoints) {
res = await fetch(`${endpoint}/v1/chat/completions`, { try {
method: "POST", res = await fetch(`${endpoint}/v1/chat/completions`, {
headers, method: "POST",
body: JSON.stringify({ headers,
model, body: JSON.stringify({
messages: [ model,
{ role: "system", content: systemBlock }, messages: [
{ role: "user", content: userPrompt }, { role: "system", content: systemBlock },
], { role: "user", content: userPrompt },
stream: false, ],
response_format: { type: "json_object" }, stream: false,
temperature: 0.6, response_format: { type: "json_object" },
max_tokens: MAX_OUTPUT_TOKENS, temperature: 0.6,
}), max_tokens: MAX_OUTPUT_TOKENS,
signal: AbortSignal.timeout(180_000), }),
}); signal: AbortSignal.timeout(180_000),
if (res.ok) break; });
const txt = await res.text().catch(() => ""); if (res.ok) break outer;
lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`; const txt = await res.text().catch(() => "");
// Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent. lastErr = `Ollama API ${res.status} @ ${endpoint}: ${txt.slice(0, 400)}`;
if (res.status >= 400 && res.status < 500 && res.status !== 429) { // 4xx other than 429 is a permanent error — no point hitting fallback
throw new Error(lastErr); if (res.status >= 400 && res.status < 500 && res.status !== 429) {
throw new Error(lastErr);
}
// Retryable — fall through to next endpoint within the same round
} catch (e: any) {
lastErr = `${e?.message || String(e)} @ ${endpoint}`;
} }
if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break;
} catch (e: any) {
lastErr = e?.message || String(e);
// network / abort — retry
} }
if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries"); if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries across all endpoints");
} }
if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed"); if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed");