seo: open site to search engines + expand sitemap to full catalog

- layout.tsx: flip robots.index from false→true (and the matching
  googleBot block). Drop the manual <meta name="robots"
  content="index, follow"> in the JSX head — it was duplicating
  (and conflicting with) the metadata-driven tag, so Google was
  seeing both "index, follow" AND "noindex, nofollow" and taking
  the more restrictive one. Now the metadata block is the single
  source of truth.
- sitemap.ts: expand from 5,000 articles to up to 45,000 (catalog
  is ~26k; cap leaves room for static + authors + events under
  Google's 50k-per-file ceiling). Add /authors/[slug] entries
  pulled from bb.author_profiles, and /events/[slug] entries
  pulled from bb.events. Drop the duplicate /articles/[slug]
  emissions — /news/[slug] is the canonical news URL.
- legacy-source.ts: page through the supabase query in 1,000-row
  chunks via Range headers so the 25k cap actually returns the
  full catalog (the previous `.limit(25000)` was getting silently
  truncated at PostgREST's default 1,000 max_rows).

Phase B (Ollama rewriter) is live; the press-release scrubber +
AI featured-image jobs are running in background to clean the
catalog. User explicitly chose to open SEO now rather than wait
for those jobs to finish.

src/app/layout.tsx

@@ -52,14 +52,14 @@ export const metadata: Metadata = {
     images: ['/assets/images/og-image.png']
   },
   robots: {
-    // noindex while Phase B (PR rewrite) and Phase D (i18n routes) land.
-    // Flip both to true once Google Rich Results Test passes on a sample
-    // NewsArticle and the news sitemap is wired up (Phase C).
-    index: false,
-    follow: false,
+    // Open to indexing as of 2026-05-20. Phase B (Ollama rewriter) is
+    // live; sitemap + robots.ts exist; the PR scrubber + AI featured
+    // images are running in the background to clean up the catalog.
+    index: true,
+    follow: true,
     googleBot: {
-      index: false,
-      follow: false,
+      index: true,
+      follow: true,
       'max-video-preview': -1,
       'max-image-preview': 'large',
       'max-snippet': -1
@@ -85,7 +85,8 @@ export default function RootLayout({
             or person names and offers a translate popup unprompted. */}
         <meta name="google" content="notranslate" />
         <meta httpEquiv="Content-Language" content="en" />
-        <meta name="robots" content="index, follow" />
+        {/* robots tag now driven entirely by `export const metadata.robots`
+            above — having both produced conflicting noindex/index tags. */}
 
         <script
           type="application/ld+json"

src/app/sitemap.ts

@@ -1,10 +1,50 @@
 import { MetadataRoute } from 'next';
+import { createClient } from '@supabase/supabase-js';
 import { getLegacyRecentSitemapEntries } from '@/lib/articles/legacy-source';
 
 const BASE_URL = process.env.NEXT_PUBLIC_SITE_URL || 'https://broadcastbeat.com';
 
+// Cap below Google's 50k URL/sitemap hard limit, leaving room for the
+// static + author + event entries we add below.
+const ARTICLE_CAP = 45000;
+
+function sb() {
+  return createClient(
+    process.env.NEXT_PUBLIC_SUPABASE_URL!,
+    process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY!,
+    {
+      db: { schema: (process.env.NEXT_PUBLIC_SUPABASE_SCHEMA || 'bb') as 'public' },
+      auth: { persistSession: false },
+    },
+  );
+}
+
+async function listAuthorSlugs(): Promise<string[]> {
+  try {
+    const { data } = await sb().from('author_profiles').select('slug').limit(500);
+    return ((data || []) as any[]).map((r) => r.slug).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+async function listEventSlugs(): Promise<{ slug: string; updated_at: string }[]> {
+  try {
+    const { data } = await sb()
+      .from('events')
+      .select('slug,updated_at,start_date')
+      .not('slug', 'is', null)
+      .limit(500);
+    return ((data || []) as any[]).map((r) => ({
+      slug: r.slug,
+      updated_at: r.updated_at || r.start_date || new Date().toISOString(),
+    }));
+  } catch {
+    return [];
+  }
+}
+
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  // Static pages
   const staticPages: MetadataRoute.Sitemap = [
     { url: `${BASE_URL}/`, lastModified: new Date(), changeFrequency: 'daily', priority: 1.0 },
     { url: `${BASE_URL}/home-page`, lastModified: new Date(), changeFrequency: 'daily', priority: 1.0 },
@@ -23,32 +63,39 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
     { url: `${BASE_URL}/terms`, lastModified: new Date(), changeFrequency: 'yearly', priority: 0.3 },
   ];
 
-  // Dynamic article pages — sourced from bb.wp_imported_posts (DB).
-  const articleEntries = await getLegacyRecentSitemapEntries(5000);
-  const articlePages: MetadataRoute.Sitemap = articleEntries.flatMap(({ slug, date }) => {
+  // Articles — sourced from bb.wp_imported_posts + bb.ai_rewritten_articles.
+  // Emit ONE canonical URL per article (/news/[slug]) rather than the prior
+  // duplicate /news/ + /articles/ pair, which wasted crawl budget and
+  // pushed us toward the 50k-per-file limit on a 29k-article catalog.
+  const articleEntries = await getLegacyRecentSitemapEntries(ARTICLE_CAP);
+  const articlePages: MetadataRoute.Sitemap = articleEntries.map(({ slug, date }) => {
     const lastModified = new Date(date);
     const safe = isNaN(lastModified.getTime()) ? new Date() : lastModified;
-    return [
-      {
-        url: `${BASE_URL}/news/${slug}`,
-        lastModified: safe,
-        changeFrequency: 'monthly' as const,
-        priority: 0.7,
-      },
-      {
-        url: `${BASE_URL}/articles/${slug}`,
-        lastModified: safe,
-        changeFrequency: 'monthly' as const,
-        priority: 0.6,
-      },
-    ];
+    return {
+      url: `${BASE_URL}/news/${slug}`,
+      lastModified: safe,
+      changeFrequency: 'monthly' as const,
+      priority: 0.7,
+    };
   });
 
-  // Attempt to ping Google Search Console (fire-and-forget, non-blocking)
-  const sitemapUrl = encodeURIComponent(`${BASE_URL}/sitemap.xml`);
-  fetch(`https://www.google.com/ping?sitemap=${sitemapUrl}`, { method: 'GET' }).catch(() => {
-    // Silently ignore — ping is best-effort
-  });
+  // Author profile pages (bb.author_profiles)
+  const authorSlugs = await listAuthorSlugs();
+  const authorPages: MetadataRoute.Sitemap = authorSlugs.map((slug) => ({
+    url: `${BASE_URL}/authors/${slug}`,
+    lastModified: new Date(),
+    changeFrequency: 'weekly' as const,
+    priority: 0.5,
+  }));
 
-  return [...staticPages, ...articlePages];
+  // Event pages (bb.events)
+  const eventList = await listEventSlugs();
+  const eventPages: MetadataRoute.Sitemap = eventList.map(({ slug, updated_at }) => ({
+    url: `${BASE_URL}/events/${slug}`,
+    lastModified: new Date(updated_at),
+    changeFrequency: 'weekly' as const,
+    priority: 0.7,
+  }));
+
+  return [...staticPages, ...articlePages, ...authorPages, ...eventPages];
 }