localadministrator/avbeat-com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite-pipeline: Ollama backoff + auto-publish to wp_imported_posts

Browse Source 
Two real bugs:

1. Ollama 503s killed every rewrite job. The outer orchestrator retried
   3× but they all hit the same saturated Ollama in a 500ms window. New
   inner retry inside ollamaProvider.rewrite() with 0/8s/30s/90s back-
   off gives AI_002 room to drain its pending queue. 180s per-attempt
   timeout. Only retries on 5xx / 429 / "busy"-style errors — 4xx other
   than 429 surfaces immediately as a permanent error.

2. Successful rewrites landed in ai_rewritten_articles but the /news
   pages render from wp_imported_posts.content. Result: rewrites were
   invisible. Fix: on success, also INSERT INTO wp_imported_posts with
   the rewritten title/body/excerpt/category/tags + featured_image from
   the original submission. Uses a negative wp_id (-unix_ts) so it
   doesn't collide with WP-imported sequence values and is easy to spot
   in admin queries. Failure to publish doesn't fail the whole pipeline
   — the rewrite still exists in ai_rewritten_articles for backfill.

NEXT (separate session): a cron to retry the 12 still-failed jobs from
the last 7 days now that backoff is in place + a backfill script to
walk the 28k existing press-release articles and rewrite them in
batches when Ollama is idle (probably nights).
This commit is contained in:
Ryan Salazar
2026-05-27 17:38:59 +00:00
parent 0d542fabf8
commit 346868fea3
2 changed files with 71 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
src/lib/ai/rewrite-client.ts
View File

@@ -196,29 +196,47 @@ export const ollamaProvider: RewriteProvider = {
const apiKey = process.env.OLLAMA_API_KEY;
if (apiKey) headers["authorization"] = `Bearer ${apiKey}`;
const res = await fetch(`${endpoint}/v1/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify({
model,
messages: [
{ role: "system", content: systemBlock },
{ role: "user", content: userPrompt },
],
stream: false,
// Ollama recognises response_format for JSON-mode on recent builds; the
// legacy `format:"json"` option also works on /api/generate but the
// OpenAI-compatible surface uses response_format.
response_format: { type: "json_object" },
temperature: 0.6,
max_tokens: MAX_OUTPUT_TOKENS,
}),
});
if (!res.ok) {
const errText = await res.text().catch(() => "");
throw new Error(`Ollama API ${res.status}: ${errText.slice(0, 500)}`);
// Retry-with-backoff against Ollama 503 / "server busy" / fetch failures.
// The single AI_002 node is shared with the ask-bb-ai + forum-burst
// workloads, so pending-request saturation is common — burning 3 attempts
// in 0.5s (as the outer orchestrator does) doesn't give it room to drain.
const RETRY_DELAYS_MS = [0, 8000, 30000, 90000]; // 0, 8s, 30s, 90s
let res: Response | null = null;
let lastErr = "";
for (let i = 0; i < RETRY_DELAYS_MS.length; i++) {
if (RETRY_DELAYS_MS[i] > 0) await new Promise((r) => setTimeout(r, RETRY_DELAYS_MS[i]));
try {
res = await fetch(`${endpoint}/v1/chat/completions`, {
method: "POST",
headers,
body: JSON.stringify({
model,
messages: [
{ role: "system", content: systemBlock },
{ role: "user", content: userPrompt },
],
stream: false,
response_format: { type: "json_object" },
temperature: 0.6,
max_tokens: MAX_OUTPUT_TOKENS,
}),
signal: AbortSignal.timeout(180_000),
});
if (res.ok) break;
const txt = await res.text().catch(() => "");
lastErr = `Ollama API ${res.status}: ${txt.slice(0, 500)}`;
// Only retry on 5xx / busy / timeout. 4xx other than 429 is permanent.
if (res.status >= 400 && res.status < 500 && res.status !== 429) {
throw new Error(lastErr);
}
if (!/busy|pending|queue|503|timeout/i.test(txt) && res.status !== 503 && res.status !== 429) break;
} catch (e: any) {
lastErr = e?.message || String(e);
// network / abort — retry
}
if (i === RETRY_DELAYS_MS.length - 1) throw new Error(lastErr || "Ollama request failed after retries");
}
if (!res || !res.ok) throw new Error(lastErr || "Ollama request failed");
const data: any = await res.json();
const text = (data?.choices?.[0]?.message?.content || "").toString();

31
src/lib/ai/rewrite-pipeline.ts
View File

@@ -164,6 +164,37 @@ export async function rewriteSubmission(opts: RewriteOptions): Promise<RewriteOu
.single();
if (artErr || !art) throw new Error(`Could not save rewritten article: ${artErr?.message}`);
// ── PUBLISH to wp_imported_posts so the /news pages actually serve the
// rewritten content. Without this step, ai_rewritten_articles fills
// up with rewrites that never surface publicly. Uses a negative wp_id
// so the rewrite doesn't collide with WP-imported sequence values.
try {
const featuredImage = sub.metadata?.featured_image_url || null;
const authorName = persona.name || sub.contributor_name || "Broadcast Beat";
const finalSlug = `${slug}-rw-${(art as any).id.toString().slice(0, 6)}`;
const wpRow = {
wp_id: -Math.floor(Date.now() / 1000) - attempt,
wp_slug: finalSlug,
title: rewrite.title,
excerpt: rewrite.excerpt,
content: rewrite.body,
author_name: authorName,
category: rewrite.category || "News",
tags: Array.isArray(rewrite.tags) ? rewrite.tags : [],
featured_image: featuredImage,
status: "published" as const,
wp_published_at: new Date().toISOString(),
imported_at: new Date().toISOString(),
ai_rewritten_at: new Date().toISOString(),
site_id: 1,
};
await sb.from("wp_imported_posts").insert(wpRow);
} catch (publishErr: any) {
// Don't fail the whole pipeline — the rewrite still landed in
// ai_rewritten_articles. Just log so we can backfill later.
console.warn(`[rewrite-pipeline] published rewrite ${(art as any).id} to ai_rewritten_articles but wp_imported_posts insert failed:`, publishErr?.message || publishErr);
}
return {
submissionId: sub.id,
rewriteJobId: lastJobId,