diff --git a/src/lib/ai/rewrite-client.ts b/src/lib/ai/rewrite-client.ts index c471987..fec7cbd 100644 --- a/src/lib/ai/rewrite-client.ts +++ b/src/lib/ai/rewrite-client.ts @@ -196,29 +196,47 @@ export const ollamaProvider: RewriteProvider = { const apiKey = process.env.OLLAMA_API_KEY; if (apiKey) headers["authorization"] = `Bearer ${apiKey}`; - const res = await fetch(`${endpoint}/v1/chat/completions`, { - method: "POST", - headers, - body: JSON.stringify({ - model, - messages: [ - { role: "system", content: systemBlock }, - { role: "user", content: userPrompt }, - ], - stream: false, - // Ollama recognises response_format for JSON-mode on recent builds; the - // legacy `format:"json"` option also works on /api/generate but the - // OpenAI-compatible surface uses response_format. - response_format: { type: "json_object" }, - temperature: 0.6, - max_tokens: MAX_OUTPUT_TOKENS, - }), - }); - - if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`Ollama API ${res.status}: ${errText.slice(0, 500)}`); + // Retry-with-backoff against Ollama 503 / "server busy" / fetch failures. + // The single AI_002 node is shared with the ask-bb-ai + forum-burst + // workloads, so pending-request saturation is common — burning 3 attempts + // in 0.5s (as the outer orchestrator does) doesn't give it room to drain. + const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s + let res: Response | null = null; + let lastErr = ""; + for (let i = 0; i < RETRY_DELAYS_MS.length; i++) { + if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i])); + try { + res = await fetch(`${endpoint}/v1/chat/completions`, { + method: "POST", + headers, + body: JSON.stringify({ + model, + messages: [ + { role: "system", content: systemBlock }, + { role: "user", content: userPrompt }, + ], + stream: false, + response_format: { type: "json_object" }, + temperature: 0.6, + max_tokens: MAX_OUTPUT_TOKENS, + }), + signal: AbortSignal.timeout(180_000), + }); + if (res.ok) break; + const txt = await res.text().catch(() => ""); + lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`; + // Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent. + if (res.status >= 400 && res.status < 500 && res.status !== 429) { + throw new Error(lastErr); + } + if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break; + } catch (e: any) { + lastErr = e?.message || String(e); + // network / abort — retry + } + if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries"); } + if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed"); const data: any = await res.json(); const text = (data?.choices?.[0]?.message?.content || "").toString(); diff --git a/src/lib/ai/rewrite-pipeline.ts b/src/lib/ai/rewrite-pipeline.ts index 355615c..56f4043 100644 --- a/src/lib/ai/rewrite-pipeline.ts +++ b/src/lib/ai/rewrite-pipeline.ts @@ -164,6 +164,37 @@ export async function rewriteSubmission(opts: RewriteOptions): Promise