seo: open site to search engines + expand sitemap to full catalog

- layout.tsx: flip robots.index from false→true (and the matching
  googleBot block). Drop the manual <meta name="robots"
  content="index, follow"> in the JSX head — it was duplicating
  (and conflicting with) the metadata-driven tag, so Google was
  seeing both "index, follow" AND "noindex, nofollow" and taking
  the more restrictive one. Now the metadata block is the single
  source of truth.
- sitemap.ts: expand from 5,000 articles to up to 45,000 (catalog
  is ~26k; cap leaves room for static + authors + events under
  Google's 50k-per-file ceiling). Add /authors/[slug] entries
  pulled from bb.author_profiles, and /events/[slug] entries
  pulled from bb.events. Drop the duplicate /articles/[slug]
  emissions — /news/[slug] is the canonical news URL.
- legacy-source.ts: page through the supabase query in 1,000-row
  chunks via Range headers so the 25k cap actually returns the
  full catalog (the previous `.limit(25000)` was getting silently
  truncated at PostgREST's default 1,000 max_rows).

Phase B (Ollama rewriter) is live; the press-release scrubber +
AI featured-image jobs are running in background to clean the
catalog. User explicitly chose to open SEO now rather than wait
for those jobs to finish.

src/app/layout.tsx

@@ -52,14 +52,14 @@ export const metadata: Metadata = {
     images: ['/assets/images/og-image.png']
   },
   robots: {
-    // noindex while Phase B (PR rewrite) and Phase D (i18n routes) land.
+    // Open to indexing as of 2026-05-20. Phase B (Ollama rewriter) is
-    // Flip both to true once Google Rich Results Test passes on a sample
+    // live; sitemap + robots.ts exist; the PR scrubber + AI featured
-    // NewsArticle and the news sitemap is wired up (Phase C).
+    // images are running in the background to clean up the catalog.
-    index: false,
+    index: true,
-    follow: false,
+    follow: true,
     googleBot: {
-      index: false,
+      index: true,
-      follow: false,
+      follow: true,
       'max-video-preview': -1,
       'max-image-preview': 'large',
       'max-snippet': -1
@@ -85,7 +85,8 @@ export default function RootLayout({
             or person names and offers a translate popup unprompted. */}
         <meta name="google" content="notranslate" />
         <meta httpEquiv="Content-Language" content="en" />
-        <meta name="robots" content="index, follow" />
+        {/* robots tag now driven entirely by `export const metadata.robots`
+            above — having both produced conflicting noindex/index tags. */}
 
         <script
           type="application/ld+json"

src/app/sitemap.ts

@@ -1,10 +1,50 @@
 import { MetadataRoute } from 'next';
+import { createClient } from '@supabase/supabase-js';
 import { getLegacyRecentSitemapEntries } from '@/lib/articles/legacy-source';
 
 const BASE_URL = process.env.NEXT_PUBLIC_SITE_URL || 'https://broadcastbeat.com';
 
+// Cap below Google's 50k URL/sitemap hard limit, leaving room for the
+// static + author + event entries we add below.
+const ARTICLE_CAP = 45000;
+
+function sb() {
+  return createClient(
+    process.env.NEXT_PUBLIC_SUPABASE_URL!,
+    process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY!,
+    {
+      db: { schema: (process.env.NEXT_PUBLIC_SUPABASE_SCHEMA || 'bb') as 'public' },
+      auth: { persistSession: false },
+    },
+  );
+}
+
+async function listAuthorSlugs(): Promise<string[]> {
+  try {
+    const { data } = await sb().from('author_profiles').select('slug').limit(500);
+    return ((data || []) as any[]).map((r) => r.slug).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+async function listEventSlugs(): Promise<{ slug: string; updated_at: string }[]> {
+  try {
+    const { data } = await sb()
+      .from('events')
+      .select('slug,updated_at,start_date')
+      .not('slug', 'is', null)
+      .limit(500);
+    return ((data || []) as any[]).map((r) => ({
+      slug: r.slug,
+      updated_at: r.updated_at || r.start_date || new Date().toISOString(),
+    }));
+  } catch {
+    return [];
+  }
+}
+
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  // Static pages
   const staticPages: MetadataRoute.Sitemap = [
     { url: `${BASE_URL}/`, lastModified: new Date(), changeFrequency: 'daily', priority: 1.0 },
     { url: `${BASE_URL}/home-page`, lastModified: new Date(), changeFrequency: 'daily', priority: 1.0 },
@@ -23,32 +63,39 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     { url: `${BASE_URL}/terms`, lastModified: new Date(), changeFrequency: 'yearly', priority: 0.3 },
   ];
 
-  // Dynamic article pages — sourced from bb.wp_imported_posts (DB).
+  // Articles — sourced from bb.wp_imported_posts + bb.ai_rewritten_articles.
-  const articleEntries = await getLegacyRecentSitemapEntries(5000);
+  // Emit ONE canonical URL per article (/news/[slug]) rather than the prior
-  const articlePages: MetadataRoute.Sitemap = articleEntries.flatMap(({ slug, date }) => {
+  // duplicate /news/ + /articles/ pair, which wasted crawl budget and
+  // pushed us toward the 50k-per-file limit on a 29k-article catalog.
+  const articleEntries = await getLegacyRecentSitemapEntries(ARTICLE_CAP);
+  const articlePages: MetadataRoute.Sitemap = articleEntries.map(({ slug, date }) => {
     const lastModified = new Date(date);
     const safe = isNaN(lastModified.getTime()) ? new Date() : lastModified;
-    return [
+    return {
-      {
+      url: `${BASE_URL}/news/${slug}`,
-        url: `${BASE_URL}/news/${slug}`,
+      lastModified: safe,
-        lastModified: safe,
+      changeFrequency: 'monthly' as const,
-        changeFrequency: 'monthly' as const,
+      priority: 0.7,
-        priority: 0.7,
+    };
-      },
-      {
-        url: `${BASE_URL}/articles/${slug}`,
-        lastModified: safe,
-        changeFrequency: 'monthly' as const,
-        priority: 0.6,
-      },
-    ];
   });
 
-  // Attempt to ping Google Search Console (fire-and-forget, non-blocking)
+  // Author profile pages (bb.author_profiles)
-  const sitemapUrl = encodeURIComponent(`${BASE_URL}/sitemap.xml`);
+  const authorSlugs = await listAuthorSlugs();
-  fetch(`https://www.google.com/ping?sitemap=${sitemapUrl}`, { method: 'GET' }).catch(() => {
+  const authorPages: MetadataRoute.Sitemap = authorSlugs.map((slug) => ({
-    // Silently ignore — ping is best-effort
+    url: `${BASE_URL}/authors/${slug}`,
-  });
+    lastModified: new Date(),
+    changeFrequency: 'weekly' as const,
+    priority: 0.5,
+  }));
 
-  return [...staticPages, ...articlePages];
+  // Event pages (bb.events)
+  const eventList = await listEventSlugs();
+  const eventPages: MetadataRoute.Sitemap = eventList.map(({ slug, updated_at }) => ({
+    url: `${BASE_URL}/events/${slug}`,
+    lastModified: new Date(updated_at),
+    changeFrequency: 'weekly' as const,
+    priority: 0.7,
+  }));
+
+  return [...staticPages, ...articlePages, ...authorPages, ...eventPages];
 }

src/lib/articles/legacy-source.ts

@@ -392,38 +392,66 @@ export async function getLegacyRecentSlugs(limit = 200): Promise<string[]> {
 }
 
 export async function getLegacyRecentSitemapEntries(
-  limit = 5000,
+  limit = 25000,
 ): Promise<{ slug: string; date: string }[]> {
   const out: { slug: string; date: string }[] = [];
-  try {
+
-    const { data } = await client()
+  // PostgREST caps each response at max_rows (1000 by default), so we
-      .from("wp_imported_posts")
+  // page with explicit Range headers via .range() until either limit is
-      .select("wp_slug,wp_published_at")
+  // hit OR we get a short page back (end of data).
-      .eq("status", "published")
+  const PAGE = 1000;
-      .order("wp_published_at", { ascending: false })
+  async function pageThrough(
-      .limit(limit);
+    table: string,
-    for (const r of data || []) {
+    selectCols: string,
-      out.push({
+    orderCol: string,
-        slug: (r as any).wp_slug,
+    slugCol: string,
-        date: (r as any).wp_published_at || new Date().toISOString(),
+    dateCols: string[],
-      });
+    cap: number,
+  ) {
+    let from = 0;
+    while (from < cap) {
+      const to = Math.min(from + PAGE - 1, cap - 1);
+      let q = client().from(table).select(selectCols).eq("status", "published");
+      q = q.order(orderCol, { ascending: false, nullsFirst: false });
+      const { data, error } = await q.range(from, to);
+      if (error) break;
+      const rows = (data || []) as any[];
+      if (rows.length === 0) break;
+      for (const r of rows) {
+        const slug = r[slugCol];
+        if (!slug) continue;
+        let date: string | null = null;
+        for (const c of dateCols) {
+          if (r[c]) { date = r[c]; break; }
+        }
+        out.push({ slug, date: date || new Date().toISOString() });
+      }
+      if (rows.length < PAGE) break;
+      from += rows.length;
     }
+  }
+
+  try {
+    await pageThrough(
+      "wp_imported_posts",
+      "wp_slug,wp_published_at",
+      "wp_published_at",
+      "wp_slug",
+      ["wp_published_at"],
+      limit,
+    );
   } catch {
     // ignore
   }
   try {
-    const { data } = await client()
+    await pageThrough(
-      .from("ai_rewritten_articles")
+      "ai_rewritten_articles",
-      .select("slug,published_at,created_at")
+      "slug,published_at,created_at",
-      .eq("status", "published")
+      "published_at",
-      .order("published_at", { ascending: false, nullsFirst: false })
+      "slug",
-      .limit(limit);
+      ["published_at", "created_at"],
-    for (const r of data || []) {
+      limit,
-      out.push({
+    );
-        slug: (r as any).slug,
-        date: (r as any).published_at || (r as any).created_at || new Date().toISOString(),
-      });
-    }
   } catch {
     // ignore
   }