From 346868fea3bf6a65cd6a078ba7bbfb332b82b6c4 Mon Sep 17 00:00:00 2001 From: Ryan Salazar Date: Wed, 27 May 2026 17:38:59 +0000 Subject: [PATCH] rewrite-pipeline: Ollama backoff + auto-publish to wp_imported_posts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real bugs: 1. Ollama 503s killed every rewrite job. The outer orchestrator retried 3× but they all hit the same saturated Ollama in a 500ms window. New inner retry inside ollamaProvider.rewrite() with 0/8s/30s/90s back- off gives AI_002 room to drain its pending queue. 180s per-attempt timeout. Only retries on 5xx / 429 / "busy"-style errors — 4xx other than 429 surfaces immediately as a permanent error. 2. Successful rewrites landed in ai_rewritten_articles but the /news pages render from wp_imported_posts.content. Result: rewrites were invisible. Fix: on success, also INSERT INTO wp_imported_posts with the rewritten title/body/excerpt/category/tags + featured_image from the original submission. Uses a negative wp_id (-unix_ts) so it doesn't collide with WP-imported sequence values and is easy to spot in admin queries. Failure to publish doesn't fail the whole pipeline — the rewrite still exists in ai_rewritten_articles for backfill. NEXT (separate session): a cron to retry the 12 still-failed jobs from the last 7 days now that backoff is in place + a backfill script to walk the 28k existing press-release articles and rewrite them in batches when Ollama is idle (probably nights). --- src/lib/ai/rewrite-client.ts | 62 ++++++++++++++++++++++------------ src/lib/ai/rewrite-pipeline.ts | 31 +++++++++++++++++ 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/src/lib/ai/rewrite-client.ts b/src/lib/ai/rewrite-client.ts index c471987..fec7cbd 100644 --- a/src/lib/ai/rewrite-client.ts +++ b/src/lib/ai/rewrite-client.ts @@ -196,29 +196,47 @@ export const ollamaProvider: RewriteProvider = { const apiKey = process.env.OLLAMA_API_KEY; if (apiKey) headers["authorization"] = `Bearer ${apiKey}`; - const res = await fetch(`${endpoint}/v1/chat/completions`, { - method: "POST", - headers, - body: JSON.stringify({ - model, - messages: [ - { role: "system", content: systemBlock }, - { role: "user", content: userPrompt }, - ], - stream: false, - // Ollama recognises response_format for JSON-mode on recent builds; the - // legacy `format:"json"` option also works on /api/generate but the - // OpenAI-compatible surface uses response_format. - response_format: { type: "json_object" }, - temperature: 0.6, - max_tokens: MAX_OUTPUT_TOKENS, - }), - }); - - if (!res.ok) { - const errText = await res.text().catch(() => ""); - throw new Error(`Ollama API ${res.status}: ${errText.slice(0, 500)}`); + // Retry-with-backoff against Ollama 503 / "server busy" / fetch failures. + // The single AI_002 node is shared with the ask-bb-ai + forum-burst + // workloads, so pending-request saturation is common — burning 3 attempts + // in 0.5s (as the outer orchestrator does) doesn't give it room to drain. + const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s + let res: Response | null = null; + let lastErr = ""; + for (let i = 0; i < RETRY_DELAYS_MS.length; i++) { + if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i])); + try { + res = await fetch(`${endpoint}/v1/chat/completions`, { + method: "POST", + headers, + body: JSON.stringify({ + model, + messages: [ + { role: "system", content: systemBlock }, + { role: "user", content: userPrompt }, + ], + stream: false, + response_format: { type: "json_object" }, + temperature: 0.6, + max_tokens: MAX_OUTPUT_TOKENS, + }), + signal: AbortSignal.timeout(180_000), + }); + if (res.ok) break; + const txt = await res.text().catch(() => ""); + lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`; + // Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent. + if (res.status >= 400 && res.status < 500 && res.status !== 429) { + throw new Error(lastErr); + } + if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break; + } catch (e: any) { + lastErr = e?.message || String(e); + // network / abort — retry + } + if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries"); } + if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed"); const data: any = await res.json(); const text = (data?.choices?.[0]?.message?.content || "").toString(); diff --git a/src/lib/ai/rewrite-pipeline.ts b/src/lib/ai/rewrite-pipeline.ts index 355615c..56f4043 100644 --- a/src/lib/ai/rewrite-pipeline.ts +++ b/src/lib/ai/rewrite-pipeline.ts @@ -164,6 +164,37 @@ export async function rewriteSubmission(opts: RewriteOptions): Promise