pi-web-search 1.1.0 → 1.2.0

package/src/api.ts

@@ -1,17 +1,23 @@
-import type { ExtensionContext, AgentToolUpdateCallback } from "@mariozechner/pi-coding-agent";
-import { type Model } from "@mariozechner/pi-ai";
+import type { ExtensionContext, AgentToolUpdateCallback } from "@earendil-works/pi-coding-agent";
+import { type Model } from "@earendil-works/pi-ai";
 import { TextEncoder, TextDecoder } from "util";
 
 // --- Provider Configuration ---
 
+type ProviderKind = "google" | "openai" | "anthropic" | "unsupported";
+
+type GoogleRequestBuilder = (model: Model<any>, body: any) => { url: string; headers: Record<string, string>; body: any };
+
 type ProviderConfig = {
-    searchTool: string;
-    urlContextTool: string;
-    buildRequest: (model: Model<any>, body: any, projectId?: string) => { url: string; headers: Record<string, string>; body: any };
+    kind: ProviderKind;
+    searchTool?: string;
+    urlContextTool?: string;
+    buildRequest?: GoogleRequestBuilder;
 };
 
-const PROVIDERS: Record<string, ProviderConfig> = {
+const GOOGLE_PROVIDERS: Record<string, ProviderConfig> = {
     "google-generative-ai": {
+        kind: "google",
         searchTool: "google_search",
         urlContextTool: "url_context",
         buildRequest: (model, body) => ({
@@ -22,48 +28,21 @@ const PROVIDERS: Record<string, ProviderConfig> = {
             },
             body
         })
-    },
-    "google-gemini-cli": {
-        searchTool: "googleSearch",
-        urlContextTool: "urlContext",
-        buildRequest: (model, body, projectId) => ({
-            url: `${model.baseUrl}/v1internal:streamGenerateContent?alt=sse`,
-            headers: {
-                "Content-Type": "application/json",
-                "Accept": "text/event-stream",
-                "User-Agent": "google-cloud-sdk vscode_cloudshelleditor/0.1",
-                "X-Goog-Api-Client": "gl-node/22.17.0",
-                "Client-Metadata": JSON.stringify({ ideType: "IDE_UNSPECIFIED", platform: "PLATFORM_UNSPECIFIED", pluginType: "GEMINI" }),
-            },
-            body: { project: projectId, model: model.id, request: body }
-        })
-    },
-    "google-antigravity": {
-        searchTool: "googleSearch",
-        urlContextTool: "urlContext",
-        buildRequest: (model, body, projectId) => ({
-            url: `${model.baseUrl}/v1internal:streamGenerateContent?alt=sse`,
-            headers: {
-                "Content-Type": "application/json",
-                "Accept": "text/event-stream",
-                "User-Agent": "antigravity/1.15.8 darwin/arm64",
-                "X-Goog-Api-Client": "gl-node/22.17.0",
-                "Client-Metadata": JSON.stringify({ ideType: "IDE_UNSPECIFIED", platform: "PLATFORM_UNSPECIFIED", pluginType: "GEMINI" }),
-            },
-            body: {
-                project: projectId,
-                model: model.id,
-                request: body,
-                requestType: "agent",
-                userAgent: "antigravity",
-                requestId: `agent-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`,
-            }
-        })
     }
 };
 
+export function getProviderKind(model: Model<any>): ProviderKind {
+    if (GOOGLE_PROVIDERS[model.provider] || GOOGLE_PROVIDERS[model.api]) return "google";
+    if (model.api === "openai-responses") return "openai";
+    if (model.api === "anthropic-messages") return "anthropic";
+    return "unsupported";
+}
+
 export function getConfig(model: Model<any>): ProviderConfig {
-    return PROVIDERS[model.provider] || PROVIDERS[model.api] || PROVIDERS["google-generative-ai"];
+    const googleConfig = GOOGLE_PROVIDERS[model.provider] || GOOGLE_PROVIDERS[model.api];
+    if (googleConfig) return googleConfig;
+    const kind = getProviderKind(model);
+    return { kind };
 }
 
 // --- Auth Compatibility Layer ---
@@ -98,13 +77,387 @@ async function getAuth(ctx: ExtensionContext, model: Model<any>): Promise<Resolv
 
 // --- Streaming API Call ---
 
+export interface Source {
+    title: string;
+    url: string;
+}
+
+export interface SearchResultDetail {
+    title?: string;
+    url?: string;
+    query?: string;
+    source?: string;
+    pageAge?: string | null;
+    citedText?: string;
+    status?: string;
+    type?: string;
+    raw?: any;
+}
+
+export interface NativeSearchCallDetail {
+    id?: string;
+    provider: ProviderKind;
+    status?: string;
+    actionType?: string;
+    queries?: string[];
+    urls?: string[];
+    raw?: any;
+}
+
 export interface StreamResult {
     text: string;
+    sources?: Source[];
+    providerKind?: ProviderKind;
+    nativeSearchUsed?: boolean;
+    nativeSearchEvents?: string[];
+    nativeSearchCalls?: NativeSearchCallDetail[];
+    searchQueries?: string[];
+    searchResults?: SearchResultDetail[];
+    citations?: SearchResultDetail[];
     groundingMetadata?: any;
     urlContextMetadata?: any;
 }
 
-export async function callApiStream(
+type SseEvent = {
+    event: string;
+    data: any;
+};
+
+async function readSseEvents(
+    response: Response,
+    signal: AbortSignal | undefined,
+    onEvent: (event: SseEvent) => void | Promise<void>
+): Promise<void> {
+    if (!response.body) {
+        throw new Error("No response body");
+    }
+
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    let currentEventData = "";
+    let currentEventName = "";
+
+    const flushEvent = async () => {
+        if (!currentEventData) return;
+        const raw = currentEventData.trim();
+        currentEventData = "";
+        const eventName = currentEventName;
+        currentEventName = "";
+        if (!raw || raw === "[DONE]") return;
+
+        let data: any;
+        try {
+            data = JSON.parse(raw);
+        } catch {
+            return;
+        }
+        await onEvent({ event: eventName, data });
+    };
+
+    while (true) {
+        if (signal?.aborted) {
+            throw new Error("Request was aborted");
+        }
+
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+
+        for (const line of lines) {
+            if (line === "" || line === "\r") {
+                await flushEvent();
+                continue;
+            }
+
+            if (line.startsWith("data:")) {
+                const data = line.slice(5).trim();
+                currentEventData = currentEventData ? currentEventData + "\n" + data : data;
+            } else if (line.startsWith("event:")) {
+                currentEventName = line.slice(6).trim();
+            }
+        }
+    }
+
+    if (buffer.trim()) {
+        const line = buffer.trim();
+        if (line.startsWith("data:")) {
+            const data = line.slice(5).trim();
+            currentEventData = currentEventData ? currentEventData + "\n" + data : data;
+        }
+    }
+    await flushEvent();
+}
+
+function extractPromptFromGeminiBody(body: any): string {
+    const parts: string[] = [];
+    for (const content of body?.contents || []) {
+        for (const part of content?.parts || []) {
+            if (typeof part?.text === "string") {
+                parts.push(part.text);
+            } else if (part?.file_data?.file_uri) {
+                parts.push(String(part.file_data.file_uri));
+            }
+        }
+    }
+    return parts.join("\n\n").trim();
+}
+
+function trimTrailingSlash(value: string): string {
+    return value.replace(/\/+$/, "");
+}
+
+function resolveAnthropicMessagesUrl(baseUrl: string): string {
+    const base = trimTrailingSlash(baseUrl);
+    return base.endsWith("/v1") ? `${base}/messages` : `${base}/v1/messages`;
+}
+
+function pushUniqueSource(sources: Source[], source: Source): number {
+    const url = source.url || "";
+    const title = source.title || "Unknown";
+    const existingIndex = sources.findIndex((s) => s.url === url && s.title === title);
+    if (existingIndex >= 0) return existingIndex;
+    sources.push({ title, url });
+    return sources.length - 1;
+}
+
+function pushUniqueString(values: string[], value: string | undefined | null) {
+    if (!value || values.includes(value)) return;
+    values.push(value);
+}
+
+function pushUniqueSearchResult(results: SearchResultDetail[], result: SearchResultDetail) {
+    const key = `${result.url || ""}\t${result.title || ""}\t${result.query || ""}\t${result.citedText || ""}\t${result.type || ""}`;
+    const exists = results.some((item) => `${item.url || ""}\t${item.title || ""}\t${item.query || ""}\t${item.citedText || ""}\t${item.type || ""}` === key);
+    if (!exists) results.push(result);
+}
+
+function pushNativeSearchEvent(events: string[], event: string) {
+    if (!events.includes(event)) events.push(event);
+}
+
+function normalizeSearchUrl(url: string): string {
+    try {
+        const parsed = new URL(url);
+        parsed.hash = "";
+        if (!/(^|\.)youtube\.com$/i.test(parsed.hostname) && !/(^|\.)youtu\.be$/i.test(parsed.hostname)) {
+            const removableParams = ["ref", "referral_type", "openLinerExtension", "_clear", "lang", "api-mode"];
+            for (const name of removableParams) parsed.searchParams.delete(name);
+            for (const name of [...parsed.searchParams.keys()]) {
+                if (name.toLowerCase().startsWith("utm_")) parsed.searchParams.delete(name);
+            }
+        }
+        const query = parsed.searchParams.toString();
+        parsed.search = query ? `?${query}` : "";
+        return parsed.toString();
+    } catch {
+        return url;
+    }
+}
+
+function titleFromUrl(url: string): string {
+    try {
+        const parsed = new URL(url);
+        const lastSegment = parsed.pathname.split("/").filter(Boolean).pop();
+        return lastSegment || parsed.hostname || url;
+    } catch {
+        return url;
+    }
+}
+
+function extractOpenAIUrlCitation(annotation: any): { endIndex?: number; title: string; url: string } | undefined {
+    const nested = annotation?.url_citation || annotation?.urlCitation;
+    const url = annotation?.url || nested?.url;
+    if (!url || typeof url !== "string") return undefined;
+
+    const title = annotation?.title || nested?.title || titleFromUrl(url);
+    const endIndexValue = annotation?.end_index ?? annotation?.endIndex ?? nested?.end_index ?? nested?.endIndex;
+    return {
+        endIndex: typeof endIndexValue === "number" ? endIndexValue : undefined,
+        title,
+        url,
+    };
+}
+
+function mergeSearchResultMetadata(results: SearchResultDetail[], extras: SearchResultDetail[]) {
+    for (const extra of extras) {
+        if (!extra.url) continue;
+        const existing = results.find((item) => item.url === extra.url);
+        if (!existing) continue;
+        if (!existing.title && extra.title) existing.title = extra.title;
+        if (!existing.query && extra.query) existing.query = extra.query;
+        if (!existing.citedText && extra.citedText) existing.citedText = extra.citedText;
+        if (!existing.status && extra.status) existing.status = extra.status;
+        if (!existing.type && extra.type) existing.type = extra.type;
+        if (!existing.source && extra.source) existing.source = extra.source;
+    }
+}
+
+function isLikelyJunkSearchUrl(url: string | undefined): boolean {
+    if (!url) return true;
+    try {
+        const parsed = new URL(url);
+        const decodedPath = decodeURIComponent(parsed.pathname).toLowerCase();
+        const suspiciousSuffixes = [
+            ".gz", ".zip", ".tgz", ".tar", ".woff", ".woff2", ".ttf", ".otf", ".eot",
+            ".webm", ".mp4", ".mp3", ".wav", ".eps", ".sql", ".csv", ".xls", ".xlsx", ".ppt", ".pptx"
+        ];
+        if (suspiciousSuffixes.some((suffix) => decodedPath.endsWith(suffix))) return true;
+        if (decodedPath === "/%" || decodedPath.endsWith("/%")) return true;
+        return false;
+    } catch {
+        return false;
+    }
+}
+
+function sanitizeSearchResults(results: SearchResultDetail[]): SearchResultDetail[] {
+    const sanitized: SearchResultDetail[] = [];
+    for (const result of results) {
+        const normalizedUrl = result.url ? normalizeSearchUrl(result.url) : result.url;
+        const normalized = { ...result, url: normalizedUrl };
+        if (normalized.url && isLikelyJunkSearchUrl(normalized.url)) continue;
+        pushUniqueSearchResult(sanitized, normalized);
+    }
+    return sanitized;
+}
+
+function deriveSources(searchResults: SearchResultDetail[], citations: SearchResultDetail[] = []): Source[] {
+    const sources: Source[] = [];
+    for (const item of [...citations, ...searchResults]) {
+        if (!item.url) continue;
+        const url = normalizeSearchUrl(item.url);
+        if (isLikelyJunkSearchUrl(url)) continue;
+        pushUniqueSource(sources, {
+            title: item.title || titleFromUrl(url),
+            url,
+        });
+    }
+    return sources;
+}
+
+function isGoogleGroundingRedirect(url: string | undefined): boolean {
+    return !!url && /^https:\/\/vertexaisearch\.cloud\.google\.com\/grounding-api-redirect\//.test(url);
+}
+
+async function resolveGoogleGroundingRedirectUrls(searchResults: SearchResultDetail[], citations: SearchResultDetail[], signal?: AbortSignal) {
+    const redirectUrls = [...new Set([...searchResults, ...citations].map((item) => item.url).filter((url): url is string => isGoogleGroundingRedirect(url)))];
+    if (redirectUrls.length === 0) return;
+
+    const resolved = new Map<string, string>();
+    await Promise.all(redirectUrls.slice(0, 20).map(async (url) => {
+        try {
+            const response = await fetch(url, { method: "HEAD", redirect: "manual", signal });
+            const location = response.headers.get("location");
+            if (location) resolved.set(url, location);
+        } catch {
+            // Ignore redirect resolution failures and keep the original URL.
+        }
+    }));
+
+    if (resolved.size === 0) return;
+    for (const item of [...searchResults, ...citations]) {
+        if (!item.url) continue;
+        const canonicalUrl = resolved.get(item.url);
+        if (!canonicalUrl) continue;
+        item.url = canonicalUrl;
+        if (!item.title || item.title === "Unknown") item.title = titleFromUrl(canonicalUrl);
+    }
+}
+
+function applyIndexCitations(text: string, citations: Array<{ endIndex?: number; title: string; url: string }>): { text: string; sources: Source[] } {
+    const sources: Source[] = [];
+    const insertions = citations
+        .filter((c) => c.url && c.endIndex !== undefined)
+        .map((c) => ({
+            index: Math.max(0, Math.min(c.endIndex!, text.length)),
+            marker: `[${pushUniqueSource(sources, { title: c.title, url: c.url }) + 1}]`
+        }))
+        .sort((a, b) => b.index - a.index);
+
+    let result = text;
+    const seen = new Set<string>();
+    for (const insertion of insertions) {
+        const key = `${insertion.index}:${insertion.marker}`;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        result = result.slice(0, insertion.index) + insertion.marker + result.slice(insertion.index);
+    }
+
+    // Preserve sources that had no end index.
+    for (const citation of citations) {
+        if (citation.url) pushUniqueSource(sources, { title: citation.title, url: citation.url });
+    }
+
+    return { text: result, sources };
+}
+
+function applyTextCitations(text: string, citations: Array<{ citedText?: string; title: string; url: string }>): { text: string; sources: Source[] } {
+    const sources: Source[] = [];
+    const insertions: Array<{ index: number; marker: string }> = [];
+    const usedRanges = new Set<string>();
+
+    for (const citation of citations) {
+        if (!citation.url) continue;
+        const marker = `[${pushUniqueSource(sources, { title: citation.title, url: citation.url }) + 1}]`;
+        const citedText = citation.citedText?.trim();
+        if (!citedText) continue;
+        const index = text.indexOf(citedText);
+        if (index < 0) continue;
+        const end = index + citedText.length;
+        const key = `${end}:${marker}`;
+        if (usedRanges.has(key)) continue;
+        usedRanges.add(key);
+        insertions.push({ index: end, marker });
+    }
+
+    let result = text;
+    for (const insertion of insertions.sort((a, b) => b.index - a.index)) {
+        result = result.slice(0, insertion.index) + insertion.marker + result.slice(insertion.index);
+    }
+
+    return { text: result, sources };
+}
+
+function extractGoogleSearchDetails(groundingMetadata: any): { searchQueries: string[]; searchResults: SearchResultDetail[]; citations: SearchResultDetail[] } {
+    const searchQueries = groundingMetadata?.webSearchQueries || [];
+    const chunks = groundingMetadata?.groundingChunks || [];
+    const supports = groundingMetadata?.groundingSupports || [];
+    const searchResults: SearchResultDetail[] = [];
+    const citations: SearchResultDetail[] = [];
+
+    chunks.forEach((chunk: any, index: number) => {
+        if (!chunk?.web) return;
+        pushUniqueSearchResult(searchResults, {
+            title: chunk.web.title || "Unknown",
+            url: chunk.web.uri || "",
+            source: "google.groundingChunks",
+            type: "web",
+            raw: { index, ...chunk.web },
+        });
+    });
+
+    supports.forEach((support: any) => {
+        for (const index of support?.groundingChunkIndices || []) {
+            const web = chunks[index]?.web;
+            if (!web) continue;
+            pushUniqueSearchResult(citations, {
+                title: web.title || "Unknown",
+                url: web.uri || "",
+                citedText: support?.segment?.text,
+                source: "google.groundingSupports",
+                type: "citation",
+                raw: support,
+            });
+        }
+    });
+
+    return { searchQueries, searchResults, citations };
+}
+
+async function callGoogleStream(
     ctx: ExtensionContext,
     model: Model<any>,
     body: any,
@@ -112,36 +465,23 @@ export async function callApiStream(
     signal?: AbortSignal
 ): Promise<StreamResult> {
     const config = getConfig(model);
+    if (!config.buildRequest) {
+        throw new Error(`Unsupported Google provider: ${model.provider}`);
+    }
+
     const auth = await getAuth(ctx, model);
     if (!auth.ok) {
         throw new Error(auth.error || "Failed to get API key and headers");
     }
 
-    // Extract projectId from apiKey for internal Google APIs (gemini-cli, antigravity)
-    // These providers return apiKey as JSON: {projectId, token}
-    let projectId: string | undefined;
-    if (model.api !== "google-generative-ai" && auth.apiKey) {
-        try {
-            const parsed = JSON.parse(auth.apiKey);
-            projectId = parsed.projectId;
-        } catch {
-            // Not a JSON string, ignore
-        }
-    }
-
-    const req = config.buildRequest(model, body, projectId);
+    const req = config.buildRequest(model, body);
 
     // Handle auth
     if (auth.headers) {
         Object.assign(req.headers, auth.headers);
     }
     if (auth.apiKey) {
-        if (model.api === "google-generative-ai") {
-            req.headers["x-goog-api-key"] = auth.apiKey;
-        } else {
-            const parsed = JSON.parse(auth.apiKey);
-            req.headers["Authorization"] = `Bearer ${parsed.token}`;
-        }
+        req.headers["x-goog-api-key"] = auth.apiKey;
     }
 
     const response = await fetch(req.url, {
@@ -155,111 +495,430 @@ export async function callApiStream(
         throw new Error(`API error (${response.status}): ${await response.text()}`);
     }
 
-    if (!response.body) {
-        throw new Error("No response body");
-    }
-
-    // Parse SSE stream
-    const reader = response.body.getReader();
-    const decoder = new TextDecoder();
-    let buffer = "";
     let accumulatedText = "";
     let groundingMetadata: any;
     let urlContextMetadata: any;
-    let currentEventData = "";
-    let currentEventName = "";
 
-    while (true) {
-        if (signal?.aborted) {
-            throw new Error("Request was aborted");
+    await readSseEvents(response, signal, ({ data: chunk }) => {
+        if (chunk.error) {
+            const errorMsg = chunk.error.message || JSON.stringify(chunk.error);
+            throw new Error(`API error (${chunk.error.code || chunk.error.status || 'unknown'}): ${errorMsg}`);
         }
 
-        const { done, value } = await reader.read();
-        if (done) break;
-
-        buffer += decoder.decode(value, { stream: true });
-        const lines = buffer.split("\n");
-        buffer = lines.pop() || "";
-
-        for (const line of lines) {
-            // Empty line indicates end of an event
-            if (line === "" || line === "\r") {
-                if (currentEventData) {
-                    // Process the complete event
-                    let chunk: any;
-                    try {
-                        chunk = JSON.parse(currentEventData);
-                    } catch {
-                        currentEventData = "";
-                        currentEventName = "";
-                        continue;
-                    }
-
-                    // Check for error in chunk (API errors are sent via SSE stream)
-                    if (chunk.error) {
-                        const errorMsg = chunk.error.message || JSON.stringify(chunk.error);
-                        throw new Error(`API error (${chunk.error.code || chunk.error.status || 'unknown'}): ${errorMsg}`);
-                    }
-
-                    // Unwrap response for internal APIs
-                    const data = chunk.response || chunk;
-                    const candidate = data.candidates?.[0];
-
-                    if (candidate?.content?.parts) {
-                        for (const part of candidate.content.parts) {
-                            if (part.text) {
-                                accumulatedText += part.text;
-                                // Stream update
-                                onUpdate?.({
-                                    content: [{ type: "text", text: accumulatedText }],
-                                    details: { streaming: true }
-                                });
-                            }
-                        }
-                    }
-
-                    // Capture metadata from final chunk
-                    if (candidate?.groundingMetadata) {
-                        groundingMetadata = candidate.groundingMetadata;
-                    }
-                    // Handle both camelCase and snake_case
-                    if (candidate?.urlContextMetadata || candidate?.url_context_metadata) {
-                        urlContextMetadata = candidate.urlContextMetadata || candidate.url_context_metadata;
-                    }
+        // Unwrap response for internal APIs
+        const data = chunk.response || chunk;
+        const candidate = data.candidates?.[0];
+
+        if (candidate?.content?.parts) {
+            for (const part of candidate.content.parts) {
+                if (part.text) {
+                    accumulatedText += part.text;
+                    onUpdate?.({
+                        content: [{ type: "text", text: accumulatedText }],
+                        details: { streaming: true }
+                    });
                 }
-                currentEventData = "";
-                currentEventName = "";
-                continue;
             }
+        }
 
-            // Parse SSE field
-            if (line.startsWith("data:")) {
-                const data = line.slice(5).trim();
-                currentEventData = currentEventData ? currentEventData + "\n" + data : data;
-            } else if (line.startsWith("event:")) {
-                currentEventName = line.slice(6).trim();
-                // Check for error event type
-                if (currentEventName === "error") {
-                    // Next data line should contain error details
-                }
-            } else if (line.startsWith("id:")) {
-                // Event ID, can be ignored for now
-            } else if (line.startsWith(":")) {
-                // Comment line, ignore
-            }
+        // Capture metadata from final chunk
+        if (candidate?.groundingMetadata) {
+            groundingMetadata = candidate.groundingMetadata;
         }
-    }
+        // Handle both camelCase and snake_case
+        if (candidate?.urlContextMetadata || candidate?.url_context_metadata) {
+            urlContextMetadata = candidate.urlContextMetadata || candidate.url_context_metadata;
+        }
+    });
 
+    const searchDetails = extractGoogleSearchDetails(groundingMetadata);
+    await resolveGoogleGroundingRedirectUrls(searchDetails.searchResults, searchDetails.citations, signal);
+    const searchResults = sanitizeSearchResults(searchDetails.searchResults);
+    const citations = sanitizeSearchResults(searchDetails.citations);
     return {
         text: accumulatedText || "No answer available.",
+        sources: deriveSources(searchResults, citations),
+        providerKind: "google",
+        nativeSearchUsed: searchDetails.searchQueries.length > 0 || searchResults.length > 0,
+        nativeSearchEvents: searchDetails.searchQueries.length > 0 ? ["google.groundingMetadata.webSearchQueries"] : [],
+        searchQueries: searchDetails.searchQueries,
+        searchResults,
+        citations,
         groundingMetadata,
         urlContextMetadata
     };
 }
 
+async function callOpenAIStream(
+    ctx: ExtensionContext,
+    model: Model<any>,
+    prompt: string,
+    onUpdate?: AgentToolUpdateCallback,
+    signal?: AbortSignal
+): Promise<StreamResult> {
+    const auth = await getAuth(ctx, model);
+    if (!auth.ok) {
+        throw new Error(auth.error || "Failed to get API key and headers");
+    }
+
+    const headers: Record<string, string> = {
+        "Content-Type": "application/json",
+        "Accept": "text/event-stream",
+        ...(model.headers || {}),
+        ...(auth.headers || {}),
+    };
+    if (auth.apiKey && !headers.Authorization && !headers.authorization) {
+        headers.Authorization = `Bearer ${auth.apiKey}`;
+    }
+
+    const requestBody: any = {
+        model: model.id,
+        input: prompt,
+        tools: [{ type: "web_search" }],
+        include: ["web_search_call.action.sources", "web_search_call.results"],
+        stream: true,
+        store: false,
+    };
+    if (model.reasoning) {
+        requestBody.reasoning = { effort: "none" };
+    }
+
+    const response = await fetch(`${trimTrailingSlash(model.baseUrl)}/responses`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(requestBody),
+        signal
+    });
+
+    if (!response.ok) {
+        throw new Error(`OpenAI API error (${response.status}): ${await response.text()}`);
+    }
+
+    let accumulatedText = "";
+    const citations: Array<{ endIndex?: number; title: string; url: string }> = [];
+    const nativeSearchEvents: string[] = [];
+    const nativeSearchCalls: NativeSearchCallDetail[] = [];
+    const searchQueries: string[] = [];
+    const searchResults: SearchResultDetail[] = [];
+
+    const collectAnnotation = (annotation: any) => {
+        if (annotation?.type !== "url_citation") return;
+        const citation = extractOpenAIUrlCitation(annotation);
+        if (!citation) return;
+        citations.push(citation);
+    };
+
+    const collectWebSearchCall = (item: any) => {
+        if (item?.type !== "web_search_call") return;
+        const action = item.action || {};
+        const call: NativeSearchCallDetail = {
+            id: item.id,
+            provider: "openai",
+            status: item.status,
+            actionType: action.type,
+            raw: item,
+        };
+        if (Array.isArray(action.queries)) {
+            const queries = action.queries.filter((query: any): query is string => typeof query === "string");
+            call.queries = queries;
+            for (const query of queries) pushUniqueString(searchQueries, query);
+        } else if (typeof action.query === "string") {
+            call.queries = [action.query];
+            pushUniqueString(searchQueries, action.query);
+        }
+        if (Array.isArray(action.sources)) {
+            call.urls = action.sources.map((source: any) => source?.url).filter((url: any): url is string => typeof url === "string");
+            for (const source of action.sources) {
+                if (!source?.url) continue;
+                pushUniqueSearchResult(searchResults, {
+                    title: source.title || source.display_name || source.name || titleFromUrl(source.url),
+                    url: source.url,
+                    source: "openai.web_search_call.action.sources",
+                    type: source.type || "url",
+                    raw: source,
+                });
+            }
+        }
+        if (action.url) {
+            call.urls = [...(call.urls || []), action.url];
+            pushUniqueSearchResult(searchResults, {
+                title: titleFromUrl(action.url),
+                url: action.url,
+                source: `openai.web_search_call.action.${action.type}`,
+                type: action.type,
+                raw: action,
+            });
+        }
+        nativeSearchCalls.push(call);
+    };
+
+    const collectFromResponse = (response: any) => {
+        for (const item of response?.output || []) {
+            collectWebSearchCall(item);
+            if (item?.type !== "message") continue;
+            for (const content of item.content || []) {
+                if (content?.type !== "output_text") continue;
+                for (const annotation of content.annotations || []) collectAnnotation(annotation);
+            }
+        }
+    };
+
+    await readSseEvents(response, signal, ({ data: event }) => {
+        if (event.type === "response.output_text.delta") {
+            accumulatedText += event.delta || "";
+            onUpdate?.({
+                content: [{ type: "text", text: accumulatedText }],
+                details: { streaming: true }
+            });
+        } else if (event.type === "response.output_text.annotation.added") {
+            collectAnnotation(event.annotation);
+        } else if (event.type === "response.output_item.added" || event.type === "response.output_item.done") {
+            collectWebSearchCall(event.item);
+        } else if (event.type === "response.completed") {
+            collectFromResponse(event.response);
+        } else if (event.type === "response.web_search_call.in_progress" || event.type === "response.web_search_call.searching" || event.type === "response.web_search_call.completed") {
+            pushNativeSearchEvent(nativeSearchEvents, event.type);
+            const call = nativeSearchCalls.find((item) => item.id === event.item_id);
+            if (call) call.status = event.type.replace("response.web_search_call.", "");
+            else nativeSearchCalls.push({ id: event.item_id, provider: "openai", status: event.type.replace("response.web_search_call.", ""), raw: event });
+            if (event.type === "response.web_search_call.searching") {
+                onUpdate?.({
+                    content: [{ type: "text", text: accumulatedText || "Searching the web with OpenAI..." }],
+                    details: { streaming: true, searching: true }
+                });
+            }
+        } else if (event.type === "response.failed") {
+            const error = event.response?.error;
+            throw new Error(error?.message || JSON.stringify(error || event.response || event));
+        } else if (event.type === "error") {
+            throw new Error(event.message || JSON.stringify(event));
+        }
+    });
+
+    const cited = applyIndexCitations(accumulatedText || "No answer available.", citations);
+    const citationDetails = citations.map((citation) => ({
+        title: citation.title,
+        url: citation.url,
+        source: "openai.url_citation",
+        type: "citation",
+        raw: citation,
+    }));
+    for (const citation of citationDetails) pushUniqueSearchResult(searchResults, citation);
+    mergeSearchResultMetadata(searchResults, citationDetails);
+    const sanitizedSearchResults = sanitizeSearchResults(searchResults);
+    const sanitizedCitations = sanitizeSearchResults(citationDetails);
+    mergeSearchResultMetadata(sanitizedSearchResults, sanitizedCitations);
+    const derivedSources = deriveSources(sanitizedSearchResults, sanitizedCitations);
+
+    return {
+        text: cited.text,
+        sources: cited.sources.length ? cited.sources.map((source) => ({ ...source, url: normalizeSearchUrl(source.url) })).filter((source) => !isLikelyJunkSearchUrl(source.url)) : derivedSources,
+        providerKind: "openai",
+        nativeSearchUsed: nativeSearchEvents.length > 0 || nativeSearchCalls.length > 0 || sanitizedSearchResults.length > 0,
+        nativeSearchEvents,
+        nativeSearchCalls,
+        searchQueries,
+        searchResults: sanitizedSearchResults,
+        citations: sanitizedCitations,
+    };
+}
+
+async function callAnthropicStream(
+    ctx: ExtensionContext,
+    model: Model<any>,
+    prompt: string,
+    onUpdate?: AgentToolUpdateCallback,
+    signal?: AbortSignal
+): Promise<StreamResult> {
+    const auth = await getAuth(ctx, model);
+    if (!auth.ok) {
+        throw new Error(auth.error || "Failed to get API key and headers");
+    }
+
+    const isOAuth = !!auth.apiKey && auth.apiKey.includes("sk-ant-oat");
+    const headers: Record<string, string> = {
+        "Content-Type": "application/json",
+        "Accept": "text/event-stream",
+        "anthropic-version": "2023-06-01",
+        ...(model.headers || {}),
+        ...(auth.headers || {}),
+    };
+
+    if (auth.apiKey) {
+        if (isOAuth) {
+            if (!headers.Authorization && !headers.authorization) headers.Authorization = `Bearer ${auth.apiKey}`;
+            headers["anthropic-beta"] = headers["anthropic-beta"]
+                ? `${headers["anthropic-beta"]},claude-code-20250219,oauth-2025-04-20`
+                : "claude-code-20250219,oauth-2025-04-20";
+            headers["user-agent"] = headers["user-agent"] || "claude-cli/2.1.75";
+            headers["x-app"] = headers["x-app"] || "cli";
+        } else if (!headers["x-api-key"] && !headers["X-Api-Key"]) {
+            headers["x-api-key"] = auth.apiKey;
+        }
+    }
+
+    const maxTokens = Math.min(Math.max(1024, Math.floor(model.maxTokens / 3) || 4096), 8192);
+    const requestBody = {
+        model: model.id,
+        max_tokens: maxTokens,
+        messages: [{ role: "user", content: prompt }],
+        tools: [{ type: "web_search_20250305", name: "web_search", max_uses: 10 }],
+        stream: true,
+    };
+
+    const response = await fetch(resolveAnthropicMessagesUrl(model.baseUrl), {
+        method: "POST",
+        headers,
+        body: JSON.stringify(requestBody),
+        signal
+    });
+
+    if (!response.ok) {
+        throw new Error(`Anthropic API error (${response.status}): ${await response.text()}`);
+    }
+
+    let accumulatedText = "";
+    const citations: Array<{ citedText?: string; title: string; url: string }> = [];
+    const nativeSearchEvents: string[] = [];
+    const nativeSearchCalls: NativeSearchCallDetail[] = [];
+    const searchResults: SearchResultDetail[] = [];
+
+    const collectSource = (source: any, toolUseId?: string) => {
+        if (!source?.url) return;
+        const title = source.title || titleFromUrl(source.url);
+        citations.push({ title, url: source.url });
+        pushUniqueSearchResult(searchResults, {
+            title,
+            url: source.url,
+            pageAge: source.page_age ?? source.pageAge,
+            source: "anthropic.web_search_tool_result",
+            type: source.type || "web_search_result",
+            raw: { toolUseId, ...source },
+        });
+    };
+
+    await readSseEvents(response, signal, ({ data: event }) => {
+        if (event.type === "content_block_start") {
+            const block = event.content_block;
+            if (block?.type === "text" && block.text) {
+                accumulatedText += block.text;
+                onUpdate?.({ content: [{ type: "text", text: accumulatedText }], details: { streaming: true } });
+            } else if (block?.type === "server_tool_use" && block.name === "web_search") {
+                pushNativeSearchEvent(nativeSearchEvents, "anthropic.content_block_start.server_tool_use.web_search");
+                nativeSearchCalls.push({
+                    id: block.id,
+                    provider: "anthropic",
+                    status: "in_progress",
+                    actionType: block.name,
+                    queries: typeof block.input?.query === "string" ? [block.input.query] : undefined,
+                    raw: block,
+                });
+                onUpdate?.({
+                    content: [{ type: "text", text: accumulatedText || "Searching the web with Anthropic..." }],
+                    details: { streaming: true, searching: true }
+                });
+            } else if (block?.type === "web_search_tool_result") {
+                pushNativeSearchEvent(nativeSearchEvents, "anthropic.content_block_start.web_search_tool_result");
+                const call = nativeSearchCalls.find((item) => item.id === block.tool_use_id);
+                if (call) call.status = "completed";
+                else nativeSearchCalls.push({ id: block.tool_use_id, provider: "anthropic", status: "completed", actionType: "web_search", raw: block });
+                if (Array.isArray(block.content)) {
+                    for (const result of block.content) collectSource(result, block.tool_use_id);
+                } else if (block.content?.type === "web_search_tool_result_error") {
+                    pushUniqueSearchResult(searchResults, {
+                        status: block.content.error_code,
+                        source: "anthropic.web_search_tool_result_error",
+                        type: block.content.type,
+                        raw: block,
+                    });
+                }
+            }
+        } else if (event.type === "content_block_delta") {
+            const delta = event.delta;
+            if (delta?.type === "text_delta") {
+                accumulatedText += delta.text || "";
+                onUpdate?.({
+                    content: [{ type: "text", text: accumulatedText }],
+                    details: { streaming: true }
+                });
+            } else if (delta?.type === "citations_delta") {
+                const citation = delta.citation;
+                if (citation?.type === "web_search_result_location" && citation.url) {
+                    const detail = {
+                        citedText: citation.cited_text,
+                        title: citation.title || titleFromUrl(citation.url),
+                        url: citation.url,
+                        source: "anthropic.citations_delta",
+                        type: citation.type,
+                        raw: citation,
+                    };
+                    citations.push({ citedText: detail.citedText, title: detail.title, url: detail.url });
+                    pushUniqueSearchResult(searchResults, detail);
+                }
+            }
+        } else if (event.type === "error") {
+            throw new Error(event.error?.message || JSON.stringify(event.error || event));
+        }
+    });
+
+    const cited = applyTextCitations(accumulatedText || "No answer available.", citations);
+    const citationDetails = citations.map((citation) => ({
+        title: citation.title || titleFromUrl(citation.url),
+        url: citation.url,
+        citedText: citation.citedText,
+        source: "anthropic.citation",
+        type: "citation",
+        raw: citation,
+    }));
+    mergeSearchResultMetadata(searchResults, citationDetails);
+    const sanitizedSearchResults = sanitizeSearchResults(searchResults);
+    const sanitizedCitations = sanitizeSearchResults(citationDetails);
+    mergeSearchResultMetadata(sanitizedSearchResults, sanitizedCitations);
+    const derivedSources = deriveSources(sanitizedSearchResults, sanitizedCitations);
+
+    return {
+        text: cited.text,
+        sources: cited.sources.length ? cited.sources.map((source) => ({ ...source, url: normalizeSearchUrl(source.url) })).filter((source) => !isLikelyJunkSearchUrl(source.url)) : derivedSources,
+        providerKind: "anthropic",
+        nativeSearchUsed: nativeSearchEvents.length > 0 || nativeSearchCalls.length > 0 || sanitizedSearchResults.length > 0,
+        nativeSearchEvents,
+        nativeSearchCalls,
+        searchQueries: nativeSearchCalls.flatMap((call) => call.queries || []),
+        searchResults: sanitizedSearchResults,
+        citations: sanitizedCitations,
+    };
+}
+
+export async function callApiStream(
+    ctx: ExtensionContext,
+    model: Model<any>,
+    body: any,
+    onUpdate?: AgentToolUpdateCallback,
+    signal?: AbortSignal
+): Promise<StreamResult> {
+    const kind = getProviderKind(model);
+    if (kind === "google") {
+        return callGoogleStream(ctx, model, body, onUpdate, signal);
+    }
+
+    const prompt = extractPromptFromGeminiBody(body);
+    if (!prompt) {
+        throw new Error("No prompt text found in request body");
+    }
+
+    if (kind === "openai") {
+        return callOpenAIStream(ctx, model, prompt, onUpdate, signal);
+    }
+    if (kind === "anthropic") {
+        return callAnthropicStream(ctx, model, prompt, onUpdate, signal);
+    }
+
+    throw new Error(`Unsupported provider for web search: ${model.provider} (${model.api})`);
+}
+
 // --- Citation Processing (byte-safe) ---
 
-export function applyCitations(text: string, groundingMetadata: any): { text: string; sources: { title: string; url: string }[] } {
+export function applyCitations(text: string, groundingMetadata: any): { text: string; sources: Source[] } {
     const chunks = groundingMetadata?.groundingChunks || [];
     const supports = groundingMetadata?.groundingSupports || [];