pi-web-kit 0.2.3 → 0.3.0

package/src/install-telemetry.ts

@@ -1,17 +1,35 @@
 import { readFileSync } from "node:fs";
-import { mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { reportInstallTelemetry as report } from "@mocito/install-telemetry";
 import { getAgentDir } from "@earendil-works/pi-coding-agent";
 
 const PACKAGE_NAME = "pi-web-kit";
-const INSTALL_TELEMETRY_URL = "https://mocito.dev/api/report-install";
-const INSTALL_TELEMETRY_TIMEOUT_MS = 5000;
+const INSTALL_TELEMETRY_ENDPOINT = "https://mocito.dev/api/report-install";
+const CI_ENVIRONMENT_VARIABLES = [
+  "APPVEYOR",
+  "BITBUCKET_BUILD_NUMBER",
+  "BUILDKITE",
+  "CIRCLECI",
+  "CODESPACES",
+  "DRONE",
+  "GITHUB_ACTIONS",
+  "GITLAB_CI",
+  "JENKINS_URL",
+  "NETLIFY",
+  "TEAMCITY_VERSION",
+  "TF_BUILD",
+  "TRAVIS",
+  "VERCEL",
+];
 
-type InstallTelemetryState = { lastReportedVersion?: string };
+interface PiSettingsDocument {
+  enableInstallTelemetry?: unknown;
+}
 
 function readJsonFile(path: string): unknown {
   try {
-    return JSON.parse(readFileSync(path, "utf8"));
+    return JSON.parse(readFileSync(path, "utf8")) as unknown;
   } catch {
     return {};
   }
@@ -22,48 +40,38 @@ function isTruthyEnvFlag(value: string | undefined): boolean {
   return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes";
 }
 
-function isInstallTelemetryEnabled(): boolean {
-  if (isTruthyEnvFlag(process.env.PI_OFFLINE)) return false;
-  if (isTruthyEnvFlag(process.env.CI) || isTruthyEnvFlag(process.env.GITHUB_ACTIONS)) return false;
-  if (process.env.PI_TELEMETRY !== undefined) return isTruthyEnvFlag(process.env.PI_TELEMETRY);
-  const settings = readJsonFile(join(getAgentDir(), "settings.json")) as { enableInstallTelemetry?: unknown };
-  return settings.enableInstallTelemetry !== false;
+function isPresentEnvFlag(value: string | undefined): boolean {
+  if (!value) return false;
+  const normalized = value.toLowerCase();
+  return normalized !== "0" && normalized !== "false" && normalized !== "no";
 }
 
-function getPackageVersion(): string {
-  try {
-    const packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { version?: unknown };
-    return typeof packageJson.version === "string" && packageJson.version.length > 0 ? packageJson.version : "0.0.0";
-  } catch {
-    return "0.0.0";
-  }
+export function isInstallTelemetryEnabled(env: NodeJS.ProcessEnv = process.env, settingsPath = join(getAgentDir(), "settings.json")): boolean {
+  if (isTruthyEnvFlag(env.CI)) return false;
+  if (CI_ENVIRONMENT_VARIABLES.some((name) => isPresentEnvFlag(env[name]))) return false;
+  if (isTruthyEnvFlag(env.PI_OFFLINE)) return false;
+
+  const settings = readJsonFile(settingsPath) as PiSettingsDocument;
+  if (settings.enableInstallTelemetry === false) return false;
+  if (env.PI_TELEMETRY !== undefined) return isTruthyEnvFlag(env.PI_TELEMETRY);
+  return true;
 }
 
-function getInstallTelemetryUserAgent(version: string): string {
-  const runtimeVersions = process.versions as NodeJS.ProcessVersions & { bun?: string };
-  const runtime = runtimeVersions.bun ? `bun/${runtimeVersions.bun}` : `node/${process.version}`;
-  return `${PACKAGE_NAME}/${version} (${process.platform}; ${runtime}; ${process.arch})`;
+function getPackageVersion(): string {
+  const packageJson = readJsonFile(fileURLToPath(new URL("../package.json", import.meta.url))) as { version?: unknown };
+  return typeof packageJson.version === "string" && packageJson.version.length > 0 ? packageJson.version : "0.0.0";
 }
 
-export async function reportInstallTelemetry(): Promise<void> {
+export function reportInstallTelemetry(): void {
   try {
-    if (!isInstallTelemetryEnabled()) return;
-
-    const version = getPackageVersion();
-    const telemetryDir = join(getAgentDir(), "extensions");
-    const statePath = join(telemetryDir, "pi-web-kit-install.json");
-    const state = readJsonFile(statePath) as InstallTelemetryState;
-    if (state.lastReportedVersion === version) return;
-
-    await mkdir(telemetryDir, { recursive: true });
-    await writeFile(statePath, `${JSON.stringify({ lastReportedVersion: version }, null, 2)}\n`);
-
-    const params = new URLSearchParams({ tool: PACKAGE_NAME, version });
-    await fetch(`${INSTALL_TELEMETRY_URL}?${params.toString()}`, {
-      headers: { "User-Agent": getInstallTelemetryUserAgent(version) },
-      signal: AbortSignal.timeout(INSTALL_TELEMETRY_TIMEOUT_MS),
-    });
+    void report({
+      endpoint: INSTALL_TELEMETRY_ENDPOINT,
+      tool: PACKAGE_NAME,
+      version: getPackageVersion(),
+      statePath: join(getAgentDir(), "extensions", "pi-web-kit-install.json"),
+      enabled: isInstallTelemetryEnabled(),
+    }).catch(() => undefined);
   } catch {
-    // Best-effort install telemetry: ignore settings, filesystem, and network failures.
+    // Best-effort telemetry: ignore local policy and filesystem failures.
   }
 }

package/src/limits.ts

@@ -1,8 +1,19 @@
+import type { SearchProviderName } from "./types.js";
+
 export const MAX_QUERY_COUNT = 5;
 export const MAX_URL_COUNT = 10;
 export const MAX_URL_LENGTH = 2_048;
 export const MAX_NUM_RESULTS = 20;
 export const DEFAULT_NUM_RESULTS = 10;
+export const DEFAULT_SEARCH_CONTEXT_TOKENS = 8_192;
+export const MAX_SEARCH_CONTEXT_TOKENS = 10_000;
+export const SEARCH_RESULT_LIMITS: Readonly<Record<SearchProviderName, number | undefined>> = {
+  exa: 100,
+  brave: 50,
+  firecrawl: 100,
+  tinyfish: undefined,
+};
+export const TINYFISH_MAX_PAGE = 10;
 export const MAX_OFFSET = 10_000_000;
 export const MAX_LIMIT = 100_000;
 export const DEFAULT_FETCH_LIMIT = 30_000;
@@ -12,3 +23,31 @@ export const FETCH_CACHE_MAX_ENTRIES = 100;
 export const FETCH_CACHE_MAX_BYTES = 20 * 1024 * 1024;
 export const FETCH_CACHE_TTL_MS = 30 * 60 * 1000;
 export const FETCH_CONCURRENCY = 3;
+
+export function capSearchResultLimit(provider: SearchProviderName, requested: number): number {
+  return Math.min(requested, SEARCH_RESULT_LIMITS[provider] ?? requested);
+}
+
+export function applySearchContextBudget(results: Array<{ content?: string; [key: string]: unknown }>, maxCharacters: number) {
+  let remaining = maxCharacters;
+  let remainingContentResults = results.filter((result) => typeof result.content === "string").length;
+  let contextCharacters = 0;
+  let omittedContextCharacters = 0;
+  const bounded = results.map((result) => {
+    if (typeof result.content !== "string") return result;
+    const share = Math.max(0, Math.floor(remaining / remainingContentResults));
+    const content = safePrefix(result.content, share);
+    remaining -= content.length;
+    remainingContentResults--;
+    contextCharacters += content.length;
+    omittedContextCharacters += result.content.length - content.length;
+    return { ...result, content };
+  });
+  return { results: bounded, contextCharacters, omittedContextCharacters };
+}
+
+export function safePrefix(value: string, maxChars: number): string {
+  let end = Math.min(value.length, maxChars);
+  if (end > 0 && /[\uD800-\uDBFF]/.test(value[end - 1])) end--;
+  return value.slice(0, end);
+}

package/src/providers/brave.ts

@@ -1,4 +1,4 @@
-import { asSnippet, requestJson } from "../http.js";
+import { asSnippet, asText, requestJson } from "../http.js";
 import type { SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
 
@@ -7,24 +7,51 @@ export class BraveProvider implements SearchProvider {
   constructor(config: WebKitConfig) { this.key = requireKey(config, "brave"); }
 
   async search(input: SearchInput, signal?: AbortSignal) {
-    const url = new URL("https://api.search.brave.com/res/v1/llm/context");
-    url.searchParams.set("q", input.query);
+    const params: Record<string, string | number | boolean | string[]> = { q: boundedQuery(input.query) };
     if (input.numResults) {
-      const count = String(Math.max(1, Math.min(input.numResults, 20)));
-      url.searchParams.set("count", count);
-      url.searchParams.set("maximum_number_of_urls", String(input.maxUrls ?? count));
+      params.count = input.numResults;
+      params.maximum_number_of_urls = input.numResults;
+    }
+    if (input.contextTokens) params.maximum_number_of_tokens = Math.max(1_024, Math.min(input.contextTokens, 32_768));
+    if (typeof input.country === "string") params.country = input.country;
+    if (typeof input.searchLang === "string") params.search_lang = input.searchLang;
+    if (typeof input.safesearch === "string") params.safesearch = input.safesearch;
+    if (typeof input.freshness === "string") params.freshness = input.freshness;
+    if (typeof input.spellcheck === "boolean") params.spellcheck = input.spellcheck;
+    if (typeof input.contextThresholdMode === "string") params.context_threshold_mode = input.contextThresholdMode;
+    if (typeof input.maxSnippets === "number") params.maximum_number_of_snippets = input.maxSnippets;
+    if (typeof input.maxTokensPerUrl === "number") params.maximum_number_of_tokens_per_url = input.maxTokensPerUrl;
+    if (typeof input.maxSnippetsPerUrl === "number") params.maximum_number_of_snippets_per_url = input.maxSnippetsPerUrl;
+    if (typeof input.goggles === "string" || Array.isArray(input.goggles)) params.goggles = input.goggles;
+
+    const url = new URL("https://api.search.brave.com/res/v1/llm/context");
+    for (const [name, value] of Object.entries(params)) {
+      if (Array.isArray(value)) for (const item of value) url.searchParams.append(name, item);
+      else url.searchParams.set(name, String(value));
     }
-    if (typeof input.country === "string") url.searchParams.set("country", input.country);
-    if (typeof input.searchLang === "string") url.searchParams.set("search_lang", input.searchLang);
-    if (typeof input.uiLang === "string") url.searchParams.set("ui_lang", input.uiLang);
-    if (typeof input.safesearch === "string") url.searchParams.set("safesearch", input.safesearch);
-    if (typeof input.freshness === "string") url.searchParams.set("freshness", input.freshness);
-    const data = await requestJson<any>(url.toString(), { headers: { "X-Subscription-Token": this.key, accept: "application/json" }, signal, timeoutMs: 30000 });
+    const usePost = input.goggles != null || url.toString().length > 2_000;
+    const data = await requestJson<any>(usePost ? `${url.origin}${url.pathname}` : url.toString(), {
+      method: usePost ? "POST" : undefined,
+      headers: { "X-Subscription-Token": this.key, accept: "application/json", ...(usePost ? { "content-type": "application/json" } : {}) },
+      body: usePost ? JSON.stringify(params) : undefined,
+      signal,
+      timeoutMs: 30000,
+    });
     const sources = data.sources ?? data.web?.results ?? [];
     const snippets = data.grounding?.generic ?? [];
-    const rows = sources.length ? sources : snippets;
-    return { provider: "brave" as const, query: input.query, results: rows.map((r: any, i: number) => ({
-      title: r.title ?? r.name, url: r.url, snippet: asSnippet(r.snippets ?? r.description ?? snippets[i]?.snippets), siteName: r.site_name ?? r.source, position: i + 1,
-    })).filter((r: any) => r.url) };
+    const sourceRows = Array.isArray(sources)
+      ? sources
+      : Object.entries(sources).map(([url, source]: [string, any]) => ({ ...source, url }));
+    const rows = snippets.length ? snippets : sourceRows;
+    return { provider: "brave" as const, query: input.query, results: rows.map((r: any, i: number) => {
+      const content = asText(r.snippets ?? r.snippet ?? r.description ?? snippets[i]?.snippets);
+      return {
+        title: r.title ?? r.name, url: r.url, snippet: asSnippet(content), content, contentFormat: content ? "markdown" as const : undefined, siteName: r.site_name ?? r.source, position: i + 1,
+      };
+    }).filter((r: any) => r.url) };
   }
 }
+
+function boundedQuery(query: string): string {
+  return query.trim().split(/\s+/).slice(0, 50).join(" ").slice(0, 400);
+}

package/src/providers/exa.ts

@@ -1,4 +1,4 @@
-import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { asSnippet, asText, normalizeUrls, requestJson, withoutContent } from "../http.js";
 import { DEFAULT_NUM_RESULTS } from "../limits.js";
 import { urlsMatch } from "../urls.js";
 import type { ExaCodeInput, ExaCodeResult, FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
@@ -14,10 +14,15 @@ export class ExaProvider implements SearchProvider, FetchProvider {
   }
 
   async search(input: SearchInput, signal?: AbortSignal) {
+    const textCharacters = input.contextTokens ? Math.min(100_000, Math.max(1, Math.floor(input.contextTokens * 4 / (input.numResults ?? DEFAULT_NUM_RESULTS)))) : undefined;
     const body = {
       query: input.query,
       numResults: input.numResults ?? DEFAULT_NUM_RESULTS,
-      contents: input.contents ?? { highlights: true },
+      contents: input.contents ?? {
+        highlights: input.purpose ? { query: input.purpose } : true,
+        ...(textCharacters ? { text: { maxCharacters: textCharacters } } : {}),
+        ...(typeof input.maxAgeHours === "number" ? { maxAgeHours: input.maxAgeHours } : {}),
+      },
       includeDomains: input.includeDomains,
       excludeDomains: input.excludeDomains,
       startPublishedDate: input.startPublishedDate,
@@ -37,13 +42,18 @@ export class ExaProvider implements SearchProvider, FetchProvider {
     return {
       provider: "exa" as const,
       query: input.query,
-      results: (data.results ?? []).map((r: any, i: number) => ({
-        title: r.title,
-        url: r.url,
-        snippet: asSnippet(r.highlights ?? r.summary ?? r.text),
-        siteName: r.author ?? r.publishedDate,
-        position: i + 1,
-      })).filter((r: any) => r.url),
+      results: (data.results ?? []).map((r: any, i: number) => {
+        const content = asText(r.text ?? r.highlights ?? r.summary);
+        return {
+          title: r.title,
+          url: r.url,
+          snippet: asSnippet(content),
+          content,
+          contentFormat: content ? "markdown" as const : undefined,
+          siteName: r.author ?? r.publishedDate,
+          position: i + 1,
+        };
+      }).filter((r: any) => r.url),
     };
   }
 
@@ -53,7 +63,14 @@ export class ExaProvider implements SearchProvider, FetchProvider {
     const data = await requestJson<any>("https://api.exa.ai/contents", {
       method: "POST",
       headers: this.headers(),
-      body: JSON.stringify({ urls, text: true, highlights: false }),
+      body: JSON.stringify({
+        urls,
+        text: { maxCharacters: 100_000 },
+        highlights: false,
+        maxAgeHours: input.refresh === true
+          ? 0
+          : input.maxAgeHours ?? (typeof input.maxAgeMs === "number" ? Math.floor(input.maxAgeMs / 3_600_000) : undefined),
+      }),
       signal,
       timeoutMs: 45000,
     });
@@ -61,7 +78,7 @@ export class ExaProvider implements SearchProvider, FetchProvider {
     const primary: WebFetchResult = { provider: "exa", results: urls.map((url, i) => {
       const r: any = list.find((item: any) => urlsMatch(item.url, url)) ?? list[i];
       if (!r) return { url, error: "No content returned by Exa contents endpoint." };
-      return { url: r.url ?? url, title: r.title, content: r.text ?? r.summary ?? "", format: "markdown" as const, metadata: r };
+      return { url: r.url ?? url, title: r.title, content: r.text ?? r.summary ?? "", format: "markdown" as const, metadata: withoutContent(r) };
     }) };
     return applyExaFetchFallbacks(this.config, input, urls, primary, signal);
   }

package/src/providers/firecrawl.ts

@@ -1,4 +1,4 @@
-import { asSnippet, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
+import { asSnippet, asText, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
 import { DEFAULT_NUM_RESULTS, FETCH_CONCURRENCY } from "../limits.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
@@ -20,13 +20,22 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
         excludeDomains: input.excludeDomains,
         categories: input.categories,
         tbs: input.tbs,
-        scrapeOptions: input.scrapeOptions ?? (input.scrape === true ? { formats: ["markdown"] } : undefined),
+        scrapeOptions: input.scrapeOptions ?? (input.scrape === true || input.contextTokens ? {
+          formats: ["markdown"],
+          onlyMainContent: true,
+          maxAge: input.maxAge,
+        } : undefined),
       }),
     });
     const list = data.data?.web ?? data.web ?? data.data ?? [];
-    return { provider: "firecrawl" as const, query: input.query, results: list.map((r: any, i: number) => ({
-      title: r.title, url: r.url, snippet: asSnippet(r.description ?? r.markdown ?? r.content), siteName: r.siteName, position: i + 1,
-    })).filter((r: any) => r.url) };
+    return { provider: "firecrawl" as const, query: input.query, results: list.map((r: any, i: number) => {
+      const content = asText(r.markdown ?? r.content ?? r.highlights);
+      return {
+        title: r.title, url: r.url, snippet: asSnippet(r.highlights ?? r.description ?? content), content,
+        contentFormat: content ? (r.markdown ? "markdown" as const : "text" as const) : undefined,
+        siteName: r.siteName, position: i + 1,
+      };
+    }).filter((r: any) => r.url) };
   }
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
@@ -44,7 +53,7 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
             waitFor: input.waitFor,
             mobile: input.mobile,
             location: input.location,
-            maxAge: input.maxAge,
+            maxAge: input.refresh === true ? 0 : input.maxAge ?? input.maxAgeMs,
           }),
         });
         const d = data.data ?? data;

package/src/providers/index.ts

@@ -2,7 +2,6 @@ import { validateFetchProvider, validateSearchProvider } from "../config.js";
 import type { FetchProvider, SearchProvider, WebKitConfig } from "../types.js";
 import { BraveProvider } from "./brave.js";
 import { Context7Provider } from "./context7.js";
-import { ExaMcpProvider } from "./exa-mcp.js";
 import { ExaProvider } from "./exa.js";
 import { FirecrawlProvider } from "./firecrawl.js";
 import { MarkdownNewProvider } from "./markdown-new.js";
@@ -11,7 +10,6 @@ import { TinyFishProvider } from "./tinyfish.js";
 export function createSearchProvider(config: WebKitConfig): SearchProvider {
   validateSearchProvider(config.provider_search);
   switch (config.provider_search) {
-    case "exa_mcp": return new ExaMcpProvider(config);
     case "exa": return new ExaProvider(config);
     case "tinyfish": return new TinyFishProvider(config);
     case "brave": return new BraveProvider(config);
@@ -22,7 +20,6 @@ export function createSearchProvider(config: WebKitConfig): SearchProvider {
 export function createFetchProvider(config: WebKitConfig): FetchProvider {
   validateFetchProvider(config.provider_fetch);
   switch (config.provider_fetch) {
-    case "exa_mcp": return new ExaMcpProvider(config);
     case "exa": return new ExaProvider(config);
     case "tinyfish": return new TinyFishProvider(config);
     case "markdown_new": return new MarkdownNewProvider(config);

package/src/providers/markdown-new.ts

@@ -15,14 +15,22 @@ export class MarkdownNewProvider implements FetchProvider {
           body: JSON.stringify({
             url,
             method: input.method ?? this.config.markdownNew.method,
-            retainImages: input.retainImages ?? this.config.markdownNew.retainImages,
+            retain_images: input.retainImages ?? this.config.markdownNew.retainImages,
           }),
           signal,
           timeoutMs: 45_000,
         });
         const text = await res.text();
         if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
-        return { url, content: text, format: "markdown" as const };
+        return {
+          url,
+          content: text,
+          format: "markdown" as const,
+          metadata: {
+            estimatedTokens: numberHeader(res.headers.get("x-markdown-tokens")),
+            conversionMethod: res.headers.get("x-markdown-method") ?? undefined,
+          },
+        };
       } catch (e) {
         return { url, error: e instanceof Error ? e.message : String(e) };
       }
@@ -30,3 +38,9 @@ export class MarkdownNewProvider implements FetchProvider {
     return { provider: "markdown_new" as const, results };
   }
 }
+
+function numberHeader(value: string | null): number | undefined {
+  if (value == null) return undefined;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}

package/src/providers/tinyfish.ts

@@ -1,6 +1,7 @@
-import { asSnippet, normalizeUrls, requestJson } from "../http.js";
-import { MAX_URL_COUNT } from "../limits.js";
-import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
+import { asSnippet, normalizeUrls, requestJson, withoutContent } from "../http.js";
+import { DEFAULT_NUM_RESULTS, MAX_URL_COUNT, TINYFISH_MAX_PAGE } from "../limits.js";
+import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig, WebSearchResult } from "../types.js";
+import { canonicalWebUrl, urlsMatch } from "../urls.js";
 import { requireKey } from "../config.js";
 
 export class TinyFishProvider implements SearchProvider, FetchProvider {
@@ -8,15 +9,81 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
   constructor(config: WebKitConfig) { this.key = requireKey(config, "tinyfish"); }
 
   async search(input: SearchInput, signal?: AbortSignal) {
-    const url = new URL("https://api.search.tinyfish.ai/");
-    url.searchParams.set("query", input.query);
-    if (input.numResults) url.searchParams.set("limit", String(input.numResults));
-    if (typeof input.page === "number") url.searchParams.set("page", String(Math.min(input.page, 10)));
-    const data = await requestJson<any>(url.toString(), { headers: { "X-API-Key": this.key }, signal, timeoutMs: 10000 });
-    const list = data.results ?? data.data ?? data.web ?? [];
-    return { provider: "tinyfish" as const, query: input.query, results: list.map((r: any, i: number) => ({
-      title: r.title, url: r.url ?? r.link, snippet: asSnippet(r.snippet ?? r.description ?? r.text), siteName: r.siteName ?? r.source, position: r.position ?? i + 1,
-    })).filter((r: any) => r.url) };
+    const desired = input.numResults ?? DEFAULT_NUM_RESULTS;
+    const firstPage = typeof input.page === "number" ? Math.max(0, Math.min(input.page, TINYFISH_MAX_PAGE)) : 0;
+    const research = input.domainType === "research_paper";
+    const dates = orderedStrings(input.afterDate, input.beforeDate);
+    const years = orderedNumbers(input.pubYearMin, input.pubYearMax);
+    const results: WebSearchResult["results"] = [];
+    const seen = new Set<string>();
+    let lastPage = firstPage - 1;
+
+    for (let page = firstPage; page <= TINYFISH_MAX_PAGE && results.length < desired; page++) {
+      lastPage = page;
+      const url = new URL("https://api.search.tinyfish.ai/");
+      url.searchParams.set("query", input.query);
+      url.searchParams.set("page", String(page));
+      setString(url, "purpose", input.purpose);
+      setString(url, "location", input.location);
+      setString(url, "language", input.language);
+      setDomains(url, "include_domains", input.includeDomains);
+      setDomains(url, "exclude_domains", input.excludeDomains);
+      setString(url, "domain_type", input.domainType);
+      if (research) {
+        setNumber(url, "pub_year_min", years[0]);
+        setNumber(url, "pub_year_max", years[1]);
+      } else if (typeof input.recencyMinutes === "number") {
+        setNumber(url, "recency_minutes", input.recencyMinutes);
+      } else {
+        setString(url, "after_date", dates[0]);
+        setString(url, "before_date", dates[1]);
+      }
+      const data = await requestJson<any>(url.toString(), { headers: { "X-API-Key": this.key }, signal, timeoutMs: 10000 });
+      const list = data.results ?? data.data ?? data.web ?? [];
+      if (!Array.isArray(list) || list.length === 0) break;
+      for (const r of list) {
+        const resultUrl = r.url ?? r.link;
+        const key = typeof resultUrl === "string" ? canonicalResultUrl(resultUrl) : undefined;
+        if (!key || seen.has(key)) continue;
+        seen.add(key);
+        results.push({
+          title: r.title,
+          url: resultUrl,
+          snippet: asSnippet(r.snippet ?? r.description ?? r.text),
+          siteName: r.siteName ?? r.source,
+          position: results.length + 1,
+        });
+        if (results.length === desired) break;
+      }
+      if (typeof data.total_results === "number" && results.length >= data.total_results) break;
+    }
+
+    if (input.contextTokens && results.length) {
+      const contextCharacters = input.contextTokens * 4;
+      let extractedCharacters = 0;
+      for (let i = 0; i < results.length && extractedCharacters < contextCharacters;) {
+        // ponytail: budget for about 512 useful tokens per page; fetch another batch when pages are short.
+        const batchSize = Math.min(MAX_URL_COUNT, Math.max(1, Math.ceil((contextCharacters - extractedCharacters) / 2_048)));
+        const batch = results.slice(i, i + batchSize);
+        const fetched = await this.fetch({ urls: batch.map((item) => item.url), format: "markdown", purpose: input.purpose }, signal);
+        for (const item of batch) {
+          const page = fetched.results.find((candidate) => urlsMatch(candidate.url, item.url) || urlsMatch(candidate.metadata?.requestedUrl, item.url));
+          if (page?.content) {
+            item.content = page.content;
+            item.contentFormat = "markdown";
+            extractedCharacters += page.content.length;
+          }
+        }
+        i += batch.length;
+      }
+    }
+
+    return {
+      provider: "tinyfish" as const,
+      query: input.query,
+      effectiveResultLimit: lastPage === TINYFISH_MAX_PAGE && results.length < desired ? results.length : desired,
+      results,
+    };
   }
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
@@ -25,20 +92,73 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
     const data = await requestJson<any>("https://api.fetch.tinyfish.ai", {
       method: "POST",
       headers: { "content-type": "application/json", "X-API-Key": this.key },
-      body: JSON.stringify({ urls, format: input.format ?? "markdown", links: input.links, image_links: input.imageLinks }),
+      body: JSON.stringify({
+        urls,
+        purpose: input.purpose,
+        format: input.format ?? "markdown",
+        links: input.links,
+        image_links: input.imageLinks,
+        ttl: input.refresh === true
+          ? 0
+          : input.ttl ?? (typeof input.maxAgeMs === "number" ? Math.floor(input.maxAgeMs / 1_000) : undefined),
+        per_url_timeout_ms: input.perUrlTimeoutMs,
+        include_selectors: input.includeSelectors,
+        exclude_selectors: input.excludeSelectors,
+      }),
       signal,
       timeoutMs: 150000,
     });
     const list = data.results ?? data.data ?? data.pages ?? [];
-    return { provider: "tinyfish" as const, results: urls.map((url, i) => {
-      const r = list.find((x: any) => (x.url ?? x.source_url) === url) ?? list[i];
-      if (!r) return { url, error: "No content returned by TinyFish." };
+    const errors = data.errors ?? [];
+    return { provider: "tinyfish" as const, results: urls.map((url) => {
+      const r = list.find((x: any) => urlsMatch(x.url ?? x.source_url, url));
+      const failure = errors.find((x: any) => urlsMatch(x.url, url));
+      if (!r) return { url, error: failure ? `${failure.error}${failure.status ? ` (${failure.status})` : ""}` : "No content returned by TinyFish." };
       const content = r.text ?? r.content ?? r.markdown ?? r.html;
-      return { url, content: stringifyContent(content), format: input.format ?? r.format ?? "markdown", title: r.title, metadata: r, error: r.error };
+      return {
+        url: r.final_url ?? r.url ?? url,
+        content: stringifyContent(content),
+        format: input.format ?? r.format ?? "markdown",
+        title: r.title,
+        metadata: { ...withoutContent(r), requestedUrl: url, finalUrl: r.final_url },
+        error: r.error,
+      };
     }) };
   }
 }
 
+function setString(url: URL, name: string, value: unknown) {
+  if (typeof value === "string" && value) url.searchParams.set(name, value);
+}
+
+function setNumber(url: URL, name: string, value: unknown) {
+  if (typeof value === "number") url.searchParams.set(name, String(value));
+}
+
+function setDomains(url: URL, name: string, value: unknown) {
+  if (Array.isArray(value) && value.length) url.searchParams.set(name, value.join(","));
+  else setString(url, name, value);
+}
+
+function orderedStrings(a: unknown, b: unknown): [string | undefined, string | undefined] {
+  const values = [a, b].filter((value): value is string => typeof value === "string").sort();
+  return [values[0], values[1]];
+}
+
+function orderedNumbers(a: unknown, b: unknown): [number | undefined, number | undefined] {
+  const values = [a, b].filter((value): value is number => typeof value === "number").sort((x, y) => x - y);
+  return [values[0], values[1]];
+}
+
 function stringifyContent(value: unknown): string | undefined {
   return typeof value === "string" ? value : JSON.stringify(value, null, 2);
 }
+
+function canonicalResultUrl(value: string): string {
+  try {
+    const canonical = canonicalWebUrl(value);
+    return canonical.endsWith("/") ? canonical.slice(0, -1) : canonical;
+  } catch {
+    return value;
+  }
+}

package/src/types.ts

@@ -1,5 +1,5 @@
-export type SearchProviderName = "exa_mcp" | "exa" | "tinyfish" | "brave" | "firecrawl";
-export type FetchProviderName = "exa_mcp" | "exa" | "tinyfish" | "markdown_new" | "firecrawl";
+export type SearchProviderName = "exa" | "tinyfish" | "brave" | "firecrawl";
+export type FetchProviderName = "exa" | "tinyfish" | "markdown_new" | "firecrawl";
 export type FetchFormat = "markdown" | "html" | "json";
 
 export interface WebKitConfig {
@@ -12,13 +12,16 @@ export interface WebKitConfig {
 export interface SearchInput {
   query: string;
   numResults?: number;
+  contextTokens?: number;
+  purpose?: string;
   [key: string]: unknown;
 }
 
 export interface WebSearchResult {
   provider: SearchProviderName;
   query: string;
-  results: Array<{ title?: string; url: string; snippet?: string; siteName?: string; position?: number }>;
+  effectiveResultLimit?: number;
+  results: Array<{ title?: string; url: string; snippet?: string; content?: string; contentFormat?: "markdown" | "text"; siteName?: string; position?: number }>;
 }
 
 export interface FetchInput {
@@ -27,9 +30,15 @@ export interface FetchInput {
   offset?: number;
   limit?: number;
   refresh?: boolean;
+  maxAgeMs?: number;
   format?: FetchFormat;
   links?: boolean;
   imageLinks?: boolean;
+  purpose?: string;
+  ttl?: number;
+  perUrlTimeoutMs?: number;
+  includeSelectors?: string[];
+  excludeSelectors?: string[];
   [key: string]: unknown;
 }