feat: add web URL content fetcher for chat context
Add ability to fetch and inject web page content into chat as context. Includes SSRF protection (blocks private IPs, localhost), content extraction (strips scripts/styles/nav), and a clean popover UI. Reimplements the concept from PR #1703 without the issues (duplicated ChatBox, dual API routes, SSRF vulnerability, window.prompt UX). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
104
app/routes/api.web-search.ts
Normal file
104
app/routes/api.web-search.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { json } from '@remix-run/cloudflare';
|
||||
import type { ActionFunctionArgs } from '@remix-run/cloudflare';
|
||||
import { isAllowedUrl } from '~/utils/url';
|
||||
|
||||
const MAX_CONTENT_LENGTH = 8000;
|
||||
|
||||
const FETCH_HEADERS = {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
};
|
||||
|
||||
function extractTitle(html: string): string {
|
||||
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
return match ? match[1].trim() : '';
|
||||
}
|
||||
|
||||
function extractMetaDescription(html: string): string {
|
||||
const match = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["'][^>]*>/i);
|
||||
|
||||
if (match) {
|
||||
return match[1].trim();
|
||||
}
|
||||
|
||||
// Try reverse attribute order
|
||||
const altMatch = html.match(/<meta[^>]*content=["']([^"']*)["'][^>]*name=["']description["'][^>]*>/i);
|
||||
|
||||
return altMatch ? altMatch[1].trim() : '';
|
||||
}
|
||||
|
||||
function extractTextContent(html: string): string {
|
||||
return html
|
||||
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, ' ')
|
||||
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, ' ')
|
||||
.replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, ' ')
|
||||
.replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, ' ')
|
||||
.replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function action({ request }: ActionFunctionArgs) {
|
||||
if (request.method !== 'POST') {
|
||||
return json({ error: 'Method not allowed' }, { status: 405 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { url } = (await request.json()) as { url?: string };
|
||||
|
||||
if (!url || typeof url !== 'string') {
|
||||
return json({ error: 'URL is required' }, { status: 400 });
|
||||
}
|
||||
|
||||
if (!isAllowedUrl(url)) {
|
||||
return json({ error: 'URL is not allowed. Only public HTTP/HTTPS URLs are accepted.' }, { status: 400 });
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: FETCH_HEADERS,
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return json({ error: `Failed to fetch URL: ${response.status} ${response.statusText}` }, { status: 502 });
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
|
||||
if (!contentType.includes('text/html') && !contentType.includes('text/plain')) {
|
||||
return json({ error: 'URL must point to an HTML or text page' }, { status: 400 });
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const title = extractTitle(html);
|
||||
const description = extractMetaDescription(html);
|
||||
const content = extractTextContent(html);
|
||||
|
||||
return json({
|
||||
success: true,
|
||||
data: {
|
||||
title,
|
||||
description,
|
||||
content: content.length > MAX_CONTENT_LENGTH ? content.slice(0, MAX_CONTENT_LENGTH) + '...' : content,
|
||||
sourceUrl: url,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
return json({ error: 'Request timed out after 10 seconds' }, { status: 504 });
|
||||
}
|
||||
|
||||
console.error('Web search error:', error);
|
||||
|
||||
return json({ error: error instanceof Error ? error.message : 'Failed to fetch URL' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user