localadministrator/avbeat-com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite: dual-endpoint Ollama failover (AI_001 + AI_002)

Browse Source 
Was: single OLLAMA_ENDPOINT — when AI_002 saturated, every rewrite job
backed off in place waiting for it to drain.

Now: tries OLLAMA_ENDPOINT (AI_002) first, instantly falls over to
OLLAMA_FALLBACK_ENDPOINT (AI_001) on 5xx/timeout before burning a
backoff sleep. Both nodes have identical model sets (llama3.1:70b,
llama3, qwen2.5-coder, etc.) so behavior is consistent across either.

Net effect: 2× GPU capacity, lower P95 latency, rewrites stop bunching
on a single saturated queue. The bb-forum-burst script already had this
pattern — pulling it into the BB rewrite path closes the gap.

OLLAMA_FALLBACK_ENDPOINT=http://10.10.0.66:11434 already set on the BB
Coolify app env.
This commit is contained in:
Ryan Salazar
2026-05-27 19:57:26 +00:00
parent 61910d6c8f
commit ec3118e7bc
1 changed files with 40 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/lib/ai/rewrite-client.ts
View File

@@ -153,8 +153,10 @@ export const ollamaProvider: RewriteProvider = {
name: "ollama",
async rewrite(req: RewriteRequest): Promise<RewriteResult> {
const endpoint = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, "");
if (!endpoint) throw new Error("OLLAMA_ENDPOINT not set in env");
const primary = (process.env.OLLAMA_ENDPOINT || "").replace(/\/$/, "");
if (!primary) throw new Error("OLLAMA_ENDPOINT not set in env");
const fallback = (process.env.OLLAMA_FALLBACK_ENDPOINT || "").replace(/\/$/, "");
const endpoints = [primary, fallback].filter(Boolean);
const model =
process.env.BB_REWRITE_MODEL ||
process.env.OLLAMA_MODEL_DEFAULT ||
@@ -196,45 +198,47 @@ export const ollamaProvider: RewriteProvider = {
const apiKey = process.env.OLLAMA_API_KEY;
if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
// Retry-with-backoff against Ollama 503 / "server busy" / fetch failures.
// The single AI_002 node is shared with the ask-bb-ai + forum-burst
// workloads, so pending-request saturation is common — burning 3 attempts
// in 0.5s (as the outer orchestrator does) doesn't give it room to drain.
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s
// Retry-with-backoff + multi-endpoint failover. We try primary first;
// if it returns 503/busy/timeout, we immediately try the fallback before
// burning a backoff sleep. Each round trip through all endpoints counts
// as one "attempt" against RETRY_DELAYS_MS. Net effect: when AI_002 is
// saturated, AI_001 picks up the slack with no delay.
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // between rounds
let res: Response | null = null;
let lastErr = "";
for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
outer: for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i]));
try {
res = await fetch(`${endpoint}/v1/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify({
model,
messages: [
{ role: "system", content: systemBlock },
{ role: "user", content: userPrompt },
],
stream: false,
response_format: { type: "json_object" },
temperature: 0.6,
max_tokens: MAX_OUTPUT_TOKENS,
}),
signal: AbortSignal.timeout(180_000),
});
if (res.ok) break;
const txt = await res.text().catch(() => "");
lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`;
// Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent.
if (res.status >= 400 && res.status < 500 && res.status !== 429) {
throw new Error(lastErr);
for (const endpoint of endpoints) {
try {
res = await fetch(`${endpoint}/v1/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify({
model,
messages: [
{ role: "system", content: systemBlock },
{ role: "user", content: userPrompt },
],
stream: false,
response_format: { type: "json_object" },
temperature: 0.6,
max_tokens: MAX_OUTPUT_TOKENS,
}),
signal: AbortSignal.timeout(180_000),
});
if (res.ok) break outer;
const txt = await res.text().catch(() => "");
lastErr = `Ollama API ${res.status} @ ${endpoint}: ${txt.slice(0, 400)}`;
// 4xx other than 429 is a permanent error — no point hitting fallback
if (res.status >= 400 && res.status < 500 && res.status !== 429) {
throw new Error(lastErr);
}
// Retryable — fall through to next endpoint within the same round
} catch (e: any) {
lastErr = `${e?.message || String(e)} @ ${endpoint}`;
}
if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break;
} catch (e: any) {
lastErr = e?.message || String(e);
// network / abort — retry
}
if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries");
if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries across all endpoints");
}
if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed");