feat: add web URL content fetcher for chat context
Add ability to fetch and inject web page content into chat as context. Includes SSRF protection (blocks private IPs, localhost), content extraction (strips scripts/styles/nav), and a clean popover UI. Reimplements the concept from PR #1703 without the issues (duplicated ChatBox, dual API routes, SSRF vulnerability, window.prompt UX). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -81,6 +81,7 @@ interface BaseChatProps {
|
||||
selectedElement?: ElementInfo | null;
|
||||
setSelectedElement?: (element: ElementInfo | null) => void;
|
||||
addToolResult?: ({ toolCallId, result }: { toolCallId: string; result: any }) => void;
|
||||
onWebSearchResult?: (result: string) => void;
|
||||
}
|
||||
|
||||
export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
|
||||
@@ -130,6 +131,7 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
|
||||
addToolResult = () => {
|
||||
throw new Error('addToolResult not implemented');
|
||||
},
|
||||
onWebSearchResult,
|
||||
},
|
||||
ref,
|
||||
) => {
|
||||
@@ -465,6 +467,7 @@ export const BaseChat = React.forwardRef<HTMLDivElement, BaseChatProps>(
|
||||
setDesignScheme={setDesignScheme}
|
||||
selectedElement={selectedElement}
|
||||
setSelectedElement={setSelectedElement}
|
||||
onWebSearchResult={onWebSearchResult}
|
||||
/>
|
||||
</div>
|
||||
</StickToBottom>
|
||||
|
||||
@@ -594,6 +594,20 @@ export const ChatImpl = memo(
|
||||
Cookies.set('selectedProvider', newProvider.name, { expires: 30 });
|
||||
};
|
||||
|
||||
const handleWebSearchResult = useCallback(
|
||||
(result: string) => {
|
||||
const currentInput = input || '';
|
||||
const newInput = currentInput.length > 0 ? `${result}\n\n${currentInput}` : result;
|
||||
|
||||
// Update the input via the same mechanism as handleInputChange
|
||||
const syntheticEvent = {
|
||||
target: { value: newInput },
|
||||
} as React.ChangeEvent<HTMLTextAreaElement>;
|
||||
handleInputChange(syntheticEvent);
|
||||
},
|
||||
[input, handleInputChange],
|
||||
);
|
||||
|
||||
return (
|
||||
<BaseChat
|
||||
ref={animationScope}
|
||||
@@ -664,6 +678,7 @@ export const ChatImpl = memo(
|
||||
selectedElement={selectedElement}
|
||||
setSelectedElement={setSelectedElement}
|
||||
addToolResult={addToolResult}
|
||||
onWebSearchResult={handleWebSearchResult}
|
||||
/>
|
||||
);
|
||||
},
|
||||
|
||||
@@ -19,6 +19,7 @@ import { ColorSchemeDialog } from '~/components/ui/ColorSchemeDialog';
|
||||
import type { DesignScheme } from '~/types/design-scheme';
|
||||
import type { ElementInfo } from '~/components/workbench/Inspector';
|
||||
import { McpTools } from './MCPTools';
|
||||
import { WebSearch } from './WebSearch.client';
|
||||
|
||||
interface ChatBoxProps {
|
||||
isModelSettingsCollapsed: boolean;
|
||||
@@ -55,6 +56,7 @@ interface ChatBoxProps {
|
||||
handleStop?: (() => void) | undefined;
|
||||
enhancingPrompt?: boolean | undefined;
|
||||
enhancePrompt?: (() => void) | undefined;
|
||||
onWebSearchResult?: (result: string) => void;
|
||||
chatMode?: 'discuss' | 'build';
|
||||
setChatMode?: (mode: 'discuss' | 'build') => void;
|
||||
designScheme?: DesignScheme;
|
||||
@@ -265,6 +267,7 @@ export const ChatBox: React.FC<ChatBoxProps> = (props) => {
|
||||
<IconButton title="Upload file" className="transition-all" onClick={() => props.handleFileUpload()}>
|
||||
<div className="i-ph:paperclip text-xl"></div>
|
||||
</IconButton>
|
||||
<WebSearch onSearchResult={(result) => props.onWebSearchResult?.(result)} disabled={props.isStreaming} />
|
||||
<IconButton
|
||||
title="Enhance prompt"
|
||||
disabled={props.input.length === 0 || props.enhancingPrompt}
|
||||
|
||||
163
app/components/chat/WebSearch.client.tsx
Normal file
163
app/components/chat/WebSearch.client.tsx
Normal file
@@ -0,0 +1,163 @@
|
||||
import { useState, useRef, useEffect } from 'react';
|
||||
import { IconButton } from '~/components/ui/IconButton';
|
||||
import { toast } from 'react-toastify';
|
||||
import { classNames } from '~/utils/classNames';
|
||||
|
||||
interface WebSearchProps {
|
||||
onSearchResult: (result: string) => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
interface WebSearchData {
|
||||
title: string;
|
||||
description: string;
|
||||
content: string;
|
||||
sourceUrl: string;
|
||||
}
|
||||
|
||||
interface WebSearchResponse {
|
||||
success: boolean;
|
||||
data?: WebSearchData;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function formatSearchResult(data: WebSearchData): string {
|
||||
const parts: string[] = [`[Web content from ${data.sourceUrl}]`];
|
||||
|
||||
if (data.title) {
|
||||
parts.push(`Title: ${data.title}`);
|
||||
}
|
||||
|
||||
if (data.description) {
|
||||
parts.push(`Description: ${data.description}`);
|
||||
}
|
||||
|
||||
parts.push('', data.content);
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
export function WebSearch({ onSearchResult, disabled = false }: WebSearchProps) {
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [isSearching, setIsSearching] = useState(false);
|
||||
const [url, setUrl] = useState('');
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (isOpen) {
|
||||
inputRef.current?.focus();
|
||||
}
|
||||
}, [isOpen]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOpen) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const handleClickOutside = (event: MouseEvent) => {
|
||||
if (containerRef.current && !containerRef.current.contains(event.target as Node)) {
|
||||
setIsOpen(false);
|
||||
}
|
||||
};
|
||||
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
|
||||
return () => document.removeEventListener('mousedown', handleClickOutside);
|
||||
}, [isOpen]);
|
||||
|
||||
const handleFetch = async () => {
|
||||
const trimmedUrl = url.trim();
|
||||
|
||||
if (!trimmedUrl) {
|
||||
return;
|
||||
}
|
||||
|
||||
setIsSearching(true);
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/web-search', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: trimmedUrl }),
|
||||
});
|
||||
|
||||
const result = (await response.json()) as WebSearchResponse;
|
||||
|
||||
if (!response.ok || !result.success || !result.data) {
|
||||
throw new Error(result.error || 'Failed to fetch URL content');
|
||||
}
|
||||
|
||||
onSearchResult(formatSearchResult(result.data));
|
||||
toast.success('URL content fetched');
|
||||
setUrl('');
|
||||
setIsOpen(false);
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : 'Failed to fetch URL');
|
||||
} finally {
|
||||
setIsSearching(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div ref={containerRef} className="relative">
|
||||
<IconButton
|
||||
title="Fetch URL content"
|
||||
disabled={disabled || isSearching}
|
||||
onClick={() => setIsOpen(!isOpen)}
|
||||
className="transition-all"
|
||||
>
|
||||
{isSearching ? (
|
||||
<div className="i-svg-spinners:90-ring-with-bg text-bolt-elements-loader-progress text-xl animate-spin" />
|
||||
) : (
|
||||
<div className="i-ph:globe text-xl" />
|
||||
)}
|
||||
</IconButton>
|
||||
{isOpen && (
|
||||
<div
|
||||
className={classNames(
|
||||
'absolute bottom-full left-0 mb-2 flex items-center gap-2',
|
||||
'rounded-lg border border-bolt-elements-borderColor bg-bolt-elements-background-depth-2 p-2 shadow-lg',
|
||||
)}
|
||||
>
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="url"
|
||||
value={url}
|
||||
onChange={(e) => setUrl(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter' && !isSearching) {
|
||||
handleFetch();
|
||||
}
|
||||
|
||||
if (e.key === 'Escape') {
|
||||
setIsOpen(false);
|
||||
}
|
||||
}}
|
||||
placeholder="https://example.com"
|
||||
disabled={isSearching}
|
||||
className={classNames(
|
||||
'w-[300px] px-3 py-1.5 text-sm rounded-md',
|
||||
'border border-bolt-elements-borderColor',
|
||||
'bg-bolt-elements-background-depth-1 text-bolt-elements-textPrimary',
|
||||
'placeholder-bolt-elements-textTertiary',
|
||||
'focus:outline-none focus:ring-2 focus:ring-bolt-elements-focus',
|
||||
)}
|
||||
/>
|
||||
<button
|
||||
onClick={handleFetch}
|
||||
disabled={isSearching || !url.trim()}
|
||||
className={classNames(
|
||||
'px-3 py-1.5 rounded-md text-sm font-medium whitespace-nowrap',
|
||||
'bg-bolt-elements-button-primary-background text-bolt-elements-button-primary-text',
|
||||
'hover:bg-bolt-elements-button-primary-backgroundHover',
|
||||
'disabled:opacity-50 disabled:cursor-not-allowed',
|
||||
)}
|
||||
>
|
||||
{isSearching ? 'Fetching...' : 'Fetch'}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
104
app/routes/api.web-search.ts
Normal file
104
app/routes/api.web-search.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import { json } from '@remix-run/cloudflare';
|
||||
import type { ActionFunctionArgs } from '@remix-run/cloudflare';
|
||||
import { isAllowedUrl } from '~/utils/url';
|
||||
|
||||
const MAX_CONTENT_LENGTH = 8000;
|
||||
|
||||
const FETCH_HEADERS = {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
};
|
||||
|
||||
function extractTitle(html: string): string {
|
||||
const match = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
||||
return match ? match[1].trim() : '';
|
||||
}
|
||||
|
||||
function extractMetaDescription(html: string): string {
|
||||
const match = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([^"']*)["'][^>]*>/i);
|
||||
|
||||
if (match) {
|
||||
return match[1].trim();
|
||||
}
|
||||
|
||||
// Try reverse attribute order
|
||||
const altMatch = html.match(/<meta[^>]*content=["']([^"']*)["'][^>]*name=["']description["'][^>]*>/i);
|
||||
|
||||
return altMatch ? altMatch[1].trim() : '';
|
||||
}
|
||||
|
||||
function extractTextContent(html: string): string {
|
||||
return html
|
||||
.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, ' ')
|
||||
.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, ' ')
|
||||
.replace(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/gi, ' ')
|
||||
.replace(/<header\b[^<]*(?:(?!<\/header>)<[^<]*)*<\/header>/gi, ' ')
|
||||
.replace(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function action({ request }: ActionFunctionArgs) {
|
||||
if (request.method !== 'POST') {
|
||||
return json({ error: 'Method not allowed' }, { status: 405 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { url } = (await request.json()) as { url?: string };
|
||||
|
||||
if (!url || typeof url !== 'string') {
|
||||
return json({ error: 'URL is required' }, { status: 400 });
|
||||
}
|
||||
|
||||
if (!isAllowedUrl(url)) {
|
||||
return json({ error: 'URL is not allowed. Only public HTTP/HTTPS URLs are accepted.' }, { status: 400 });
|
||||
}
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: FETCH_HEADERS,
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
return json({ error: `Failed to fetch URL: ${response.status} ${response.statusText}` }, { status: 502 });
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
|
||||
if (!contentType.includes('text/html') && !contentType.includes('text/plain')) {
|
||||
return json({ error: 'URL must point to an HTML or text page' }, { status: 400 });
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const title = extractTitle(html);
|
||||
const description = extractMetaDescription(html);
|
||||
const content = extractTextContent(html);
|
||||
|
||||
return json({
|
||||
success: true,
|
||||
data: {
|
||||
title,
|
||||
description,
|
||||
content: content.length > MAX_CONTENT_LENGTH ? content.slice(0, MAX_CONTENT_LENGTH) + '...' : content,
|
||||
sourceUrl: url,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
return json({ error: 'Request timed out after 10 seconds' }, { status: 504 });
|
||||
}
|
||||
|
||||
console.error('Web search error:', error);
|
||||
|
||||
return json({ error: error instanceof Error ? error.message : 'Failed to fetch URL' }, { status: 500 });
|
||||
}
|
||||
}
|
||||
42
app/utils/url.ts
Normal file
42
app/utils/url.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
/**
|
||||
* URL validation utilities with SSRF protection.
|
||||
*/
|
||||
|
||||
const PRIVATE_IP_PATTERNS = [
|
||||
/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, // Loopback
|
||||
/^10\.\d{1,3}\.\d{1,3}\.\d{1,3}$/, // Class A private
|
||||
/^172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3}$/, // Class B private
|
||||
/^192\.168\.\d{1,3}\.\d{1,3}$/, // Class C private
|
||||
/^169\.254\.\d{1,3}\.\d{1,3}$/, // Link-local
|
||||
/^0\.0\.0\.0$/, // Unspecified
|
||||
];
|
||||
|
||||
const BLOCKED_HOSTNAMES = new Set(['localhost', '[::1]', '0.0.0.0']);
|
||||
|
||||
export function isValidUrl(input: string): boolean {
|
||||
try {
|
||||
const url = new URL(input);
|
||||
return url.protocol === 'http:' || url.protocol === 'https:';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isAllowedUrl(input: string): boolean {
|
||||
if (!isValidUrl(input)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const url = new URL(input);
|
||||
const hostname = url.hostname.toLowerCase();
|
||||
|
||||
if (BLOCKED_HOSTNAMES.has(hostname)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (PRIVATE_IP_PATTERNS.some((pattern) => pattern.test(hostname))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user