pi-web-kit 0.1.0

package/src/http.ts

@@ -0,0 +1,56 @@
+import { DEFAULT_TIMEOUT_MS } from "./limits.js";
+import { normalizeUrlInput } from "./urls.js";
+
+const MAX_CHARS = 50_000;
+const MAX_LINES = 2_000;
+
+export function truncateText(text: string, maxChars = MAX_CHARS): string {
+  const lines = text.split(/\r?\n/);
+  const lineLimited = lines.length > MAX_LINES ? lines.slice(0, MAX_LINES).join("\n") + "\n[truncated: line limit]" : text;
+  return lineLimited.length > maxChars ? lineLimited.slice(0, maxChars) + "\n[truncated: size limit]" : lineLimited;
+}
+
+export async function fetchWithTimeout(url: string, init: RequestInit & { timeoutMs?: number } = {}): Promise<Response> {
+  const controller = new AbortController();
+  const timeoutMs = init.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const timer = setTimeout(() => controller.abort(new Error(`Request timed out after ${timeoutMs}ms`)), timeoutMs);
+  const signal = init.signal ? AbortSignal.any([init.signal, controller.signal]) : controller.signal;
+  const { timeoutMs: _timeoutMs, ...rest } = init;
+  try {
+    return await fetch(url, { ...rest, signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function requestJson<T>(url: string, init: RequestInit & { timeoutMs?: number } = {}): Promise<T> {
+  const res = await fetchWithTimeout(url, init);
+  const text = await res.text();
+  if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
+  if (!text) return undefined as T;
+  return JSON.parse(text) as T;
+}
+
+export function asSnippet(value: unknown): string | undefined {
+  if (value == null) return undefined;
+  if (Array.isArray(value)) return value.filter(Boolean).join("\n").slice(0, 1000) || undefined;
+  if (typeof value === "string") return value.slice(0, 1000) || undefined;
+  return JSON.stringify(value).slice(0, 1000);
+}
+
+export function normalizeUrls(input: { url?: string; urls?: string[] }, maxCount?: number): string[] {
+  return normalizeUrlInput(input, maxCount);
+}
+
+export async function mapConcurrent<T, R>(items: T[], concurrency: number, mapper: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (next < items.length) {
+      const index = next++;
+      results[index] = await mapper(items[index], index);
+    }
+  });
+  await Promise.all(workers);
+  return results;
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./types.js";
+export * from "./config.js";
+export * from "./providers/index.js";

package/src/limits.ts

@@ -0,0 +1,14 @@
+export const MAX_QUERY_COUNT = 5;
+export const MAX_URL_COUNT = 10;
+export const MAX_URL_LENGTH = 2_048;
+export const MAX_NUM_RESULTS = 20;
+export const DEFAULT_NUM_RESULTS = 10;
+export const MAX_OFFSET = 10_000_000;
+export const MAX_LIMIT = 100_000;
+export const DEFAULT_FETCH_LIMIT = 30_000;
+export const MULTI_FETCH_LIMIT = 8_000;
+export const DEFAULT_TIMEOUT_MS = 30_000;
+export const FETCH_CACHE_MAX_ENTRIES = 100;
+export const FETCH_CACHE_MAX_BYTES = 20 * 1024 * 1024;
+export const FETCH_CACHE_TTL_MS = 30 * 60 * 1000;
+export const FETCH_CONCURRENCY = 3;

package/src/providers/brave.ts

@@ -0,0 +1,30 @@
+import { asSnippet, requestJson } from "../http.js";
+import type { SearchInput, SearchProvider, WebKitConfig } from "../types.js";
+import { requireKey } from "../config.js";
+
+export class BraveProvider implements SearchProvider {
+  private key: string;
+  constructor(config: WebKitConfig) { this.key = requireKey(config, "brave"); }
+
+  async search(input: SearchInput, signal?: AbortSignal) {
+    const url = new URL("https://api.search.brave.com/res/v1/llm/context");
+    url.searchParams.set("q", input.query);
+    if (input.numResults) {
+      const count = String(Math.max(1, Math.min(input.numResults, 20)));
+      url.searchParams.set("count", count);
+      url.searchParams.set("maximum_number_of_urls", String(input.maxUrls ?? count));
+    }
+    if (typeof input.country === "string") url.searchParams.set("country", input.country);
+    if (typeof input.searchLang === "string") url.searchParams.set("search_lang", input.searchLang);
+    if (typeof input.uiLang === "string") url.searchParams.set("ui_lang", input.uiLang);
+    if (typeof input.safesearch === "string") url.searchParams.set("safesearch", input.safesearch);
+    if (typeof input.freshness === "string") url.searchParams.set("freshness", input.freshness);
+    const data = await requestJson<any>(url.toString(), { headers: { "X-Subscription-Token": this.key, accept: "application/json" }, signal, timeoutMs: 30000 });
+    const sources = data.sources ?? data.web?.results ?? [];
+    const snippets = data.grounding?.generic ?? [];
+    const rows = sources.length ? sources : snippets;
+    return { provider: "brave" as const, query: input.query, results: rows.map((r: any, i: number) => ({
+      title: r.title ?? r.name, url: r.url, snippet: asSnippet(r.snippets ?? r.description ?? snippets[i]?.snippets), siteName: r.site_name ?? r.source, position: i + 1,
+    })).filter((r: any) => r.url) };
+  }
+}

package/src/providers/exa-mcp.ts

@@ -0,0 +1,91 @@
+import { asSnippet, fetchWithTimeout, normalizeUrls } from "../http.js";
+import { urlsMatch } from "../urls.js";
+import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
+import { applyExaFetchFallbacks } from "./fallback.js";
+
+let nextId = 1;
+
+export class ExaMcpProvider implements SearchProvider, FetchProvider {
+  private sessionId?: string;
+  constructor(private config: WebKitConfig) {}
+
+  async search(input: SearchInput, signal?: AbortSignal) {
+    const result = await this.callTool("web_search_exa", { query: input.query, numResults: input.numResults ?? 10 }, signal);
+    return { provider: "exa_mcp" as const, query: input.query, results: normalizeSearch(result, input.query) };
+  }
+
+  async fetch(input: FetchInput, signal?: AbortSignal) {
+    const urls = normalizeUrls(input);
+    const result = await this.callTool("web_fetch_exa", { urls }, signal);
+    const pages = normalizeFetch(result, urls);
+    return applyExaFetchFallbacks(this.config, input, urls, { provider: "exa_mcp" as const, results: pages }, signal);
+  }
+
+  private async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<any> {
+    await this.initialize(signal);
+    return this.rpc("tools/call", { name, arguments: args }, signal);
+  }
+
+  private async initialize(signal?: AbortSignal) {
+    if (this.sessionId) return;
+    const data = await this.rpcRaw("initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "pi-web-kit", version: "0.1.0" } }, signal);
+    this.sessionId = data.sessionId ?? data.payload?.sessionId;
+    if (!this.sessionId) throw new Error("Exa MCP initialize did not return an mcp-session-id response header.");
+    await this.rpcRaw("notifications/initialized", undefined, signal, true);
+  }
+
+  private async rpc(method: string, params: unknown, signal?: AbortSignal) {
+    const data = await this.rpcRaw(method, params, signal);
+    if (data.payload?.error) throw new Error(data.payload.error.message ?? JSON.stringify(data.payload.error));
+    return data.payload?.result ?? data.payload;
+  }
+
+  private async rpcRaw(method: string, params: unknown, signal?: AbortSignal, notification = false): Promise<{ payload?: any; sessionId?: string }> {
+    const headers: Record<string, string> = { "content-type": "application/json", accept: "application/json, text/event-stream" };
+    if (this.sessionId) headers["mcp-session-id"] = this.sessionId;
+    if (this.config.apiKeys.exa) headers["x-api-key"] = this.config.apiKeys.exa;
+    const body = notification ? { jsonrpc: "2.0", method, params } : { jsonrpc: "2.0", id: nextId++, method, params };
+    const res = await fetchWithTimeout("https://mcp.exa.ai/mcp", { method: "POST", headers, body: JSON.stringify(body), signal, timeoutMs: 45_000 });
+    const text = await res.text();
+    if (!res.ok) throw new Error(`Exa MCP ${method} failed: ${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
+    if (notification) return { sessionId: this.sessionId };
+    const sessionId = res.headers.get("mcp-session-id") ?? undefined;
+    const payload = parseMcpResponse(text, res.headers.get("content-type") ?? "");
+    return { payload, sessionId };
+  }
+}
+
+function parseMcpResponse(text: string, contentType: string): any {
+  if (!text.trim()) return undefined;
+  if (contentType.includes("text/event-stream") || text.startsWith("event:") || text.startsWith("data:")) {
+    const data = text.split(/\r?\n/).filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).join("\n");
+    return data ? JSON.parse(data) : undefined;
+  }
+  return JSON.parse(text);
+}
+
+function textFromContent(result: any): string {
+  const content = result?.content ?? result;
+  if (Array.isArray(content)) return content.map((c) => typeof c === "string" ? c : c.text ?? JSON.stringify(c)).join("\n");
+  return typeof content === "string" ? content : JSON.stringify(content);
+}
+
+function normalizeSearch(result: any, _query: string) {
+  const structured = result?.structuredContent ?? result?.result ?? result;
+  const list = structured.results ?? structured.data ?? structured.items;
+  if (Array.isArray(list)) return list.map((r: any, i: number) => ({ title: r.title, url: r.url, snippet: asSnippet(r.snippet ?? r.text ?? r.summary ?? r.highlights), siteName: r.siteName, position: i + 1 })).filter((r: any) => r.url);
+  const text = textFromContent(result);
+  const urls = [...text.matchAll(/https?:\/\/[^\s)\]}>"']+/g)].map((m) => m[0]);
+  return [...new Set(urls)].map((url, i) => ({ url, snippet: i === 0 ? text.slice(0, 1000) : undefined, position: i + 1 }));
+}
+
+function normalizeFetch(result: any, urls: string[]) {
+  const structured = result?.structuredContent ?? result?.result ?? result;
+  const list = structured.results ?? structured.data ?? structured.pages;
+  if (Array.isArray(list)) return urls.map((url, i) => {
+    const r = list.find((x: any) => urlsMatch(x.url, url)) ?? list[i];
+    return r ? { url, title: r.title, content: r.markdown ?? r.text ?? r.content ?? r.html, format: "markdown" as const, metadata: r, error: r.error } : { url, error: "No content returned by Exa MCP." };
+  });
+  const text = textFromContent(result);
+  return urls.length <= 1 ? [{ url: urls[0] ?? "", content: text, format: "markdown" as const }] : urls.map((url) => ({ url, content: text, format: "markdown" as const }));
+}

package/src/providers/exa.ts

@@ -0,0 +1,63 @@
+import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { urlsMatch } from "../urls.js";
+import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
+import { requireKey } from "../config.js";
+import { applyExaFetchFallbacks } from "./fallback.js";
+
+export class ExaProvider implements SearchProvider, FetchProvider {
+  private key: string;
+  constructor(private config: WebKitConfig) { this.key = requireKey(config, "exa"); }
+
+  async search(input: SearchInput, signal?: AbortSignal) {
+    const body = {
+      query: input.query,
+      numResults: input.numResults ?? 10,
+      contents: input.contents ?? { highlights: true },
+      includeDomains: input.includeDomains,
+      excludeDomains: input.excludeDomains,
+      startPublishedDate: input.startPublishedDate,
+      endPublishedDate: input.endPublishedDate,
+      startCrawlDate: input.startCrawlDate,
+      endCrawlDate: input.endCrawlDate,
+      type: input.type,
+      category: input.category,
+    };
+    const data = await requestJson<any>("https://api.exa.ai/search", {
+      method: "POST",
+      headers: { "content-type": "application/json", "x-api-key": this.key },
+      body: JSON.stringify(body),
+      signal,
+      timeoutMs: 30000,
+    });
+    return {
+      provider: "exa" as const,
+      query: input.query,
+      results: (data.results ?? []).map((r: any, i: number) => ({
+        title: r.title,
+        url: r.url,
+        snippet: asSnippet(r.highlights ?? r.summary ?? r.text),
+        siteName: r.author ?? r.publishedDate,
+        position: i + 1,
+      })).filter((r: any) => r.url),
+    };
+  }
+
+  async fetch(input: FetchInput, signal?: AbortSignal): Promise<WebFetchResult> {
+    const urls = normalizeUrls(input);
+    if (urls.length === 0) return { provider: "exa", results: [] };
+    const data = await requestJson<any>("https://api.exa.ai/contents", {
+      method: "POST",
+      headers: { "content-type": "application/json", "x-api-key": this.key },
+      body: JSON.stringify({ urls, text: true, highlights: false }),
+      signal,
+      timeoutMs: 45000,
+    });
+    const list = data.results ?? [];
+    const primary: WebFetchResult = { provider: "exa", results: urls.map((url, i) => {
+      const r: any = list.find((item: any) => urlsMatch(item.url, url)) ?? list[i];
+      if (!r) return { url, error: "No content returned by Exa contents endpoint." };
+      return { url: r.url ?? url, title: r.title, content: r.text ?? r.summary ?? "", format: "markdown" as const, metadata: r };
+    }) };
+    return applyExaFetchFallbacks(this.config, input, urls, primary, signal);
+  }
+}

package/src/providers/fallback.ts

@@ -0,0 +1,82 @@
+import type { FetchInput, FetchProvider, FetchProviderName, WebFetchResult, WebKitConfig } from "../types.js";
+import { urlsMatch } from "../urls.js";
+import { FirecrawlProvider } from "./firecrawl.js";
+import { MarkdownNewProvider } from "./markdown-new.js";
+import { TinyFishProvider } from "./tinyfish.js";
+
+type FetchPage = WebFetchResult["results"][number];
+
+export function isMissingExaCrawlResult(page: FetchPage | undefined): boolean {
+  if (!page) return true;
+  const message = [page.error, page.content].filter((v): v is string => typeof v === "string").join("\n");
+  if (!message.trim()) return true;
+  return /\b(no crawl results?|not crawled|crawl(?:ed)? content (?:is )?(?:not )?(?:available|found)|no content returned by exa|no results? found)\b/i.test(message);
+}
+
+export async function applyExaFetchFallbacks(
+  config: WebKitConfig,
+  input: FetchInput,
+  urls: string[],
+  primary: WebFetchResult,
+  signal?: AbortSignal,
+): Promise<WebFetchResult> {
+  const results = urls.map((url, i) => primary.results.find((item) => urlsMatch(item.url, url)) ?? primary.results[i] ?? { url, error: "No content returned by Exa." });
+  let missing = urls.filter((url, i) => isMissingExaCrawlResult(results[i]));
+  if (missing.length === 0) return { ...primary, results };
+
+  for (const { name, provider } of fallbackProviders(config)) {
+    if (missing.length === 0) break;
+    let fetched: WebFetchResult;
+    try {
+      fetched = await provider.fetch({ ...input, url: undefined, urls: missing }, signal);
+    } catch {
+      continue;
+    }
+    const mapped = mapFetchResults(missing, fetched);
+    missing = missing.filter((url) => {
+      const item = mapped.get(url);
+      if (!hasUsableContent(item)) return true;
+      const index = urls.findIndex((candidate) => candidate === url);
+      results[index] = {
+        ...item,
+        url,
+        metadata: {
+          ...(isRecord(item?.metadata) ? item.metadata : {}),
+          fallbackProvider: name,
+          fallbackFrom: primary.provider,
+          fetchedUrl: item?.url,
+        },
+      };
+      return false;
+    });
+  }
+
+  return { ...primary, results };
+}
+
+function fallbackProviders(config: WebKitConfig): Array<{ name: FetchProviderName; provider: FetchProvider }> {
+  const providers: Array<{ name: FetchProviderName; provider: FetchProvider }> = [];
+  if (config.apiKeys.tinyfish) providers.push({ name: "tinyfish", provider: new TinyFishProvider(config) });
+  if (config.apiKeys.firecrawl) providers.push({ name: "firecrawl", provider: new FirecrawlProvider(config) });
+  providers.push({ name: "markdown_new", provider: new MarkdownNewProvider(config) });
+  return providers;
+}
+
+function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, FetchPage> {
+  const out = new Map<string, FetchPage>();
+  const remaining = [...(fetched.results ?? [])];
+  for (const url of requested) {
+    const index = remaining.findIndex((item) => urlsMatch(item.url, url));
+    if (index >= 0) out.set(url, remaining.splice(index, 1)[0]);
+  }
+  requested.forEach((url, index) => { if (!out.has(url) && fetched.results?.[index]) out.set(url, fetched.results[index]); });
+  return out;
+}
+
+function hasUsableContent(item: FetchPage | undefined): item is FetchPage {
+  return !!item && !item.error && typeof item.content === "string" && item.content.trim().length > 0;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === "object" && !Array.isArray(value);
+}

package/src/providers/firecrawl.ts

@@ -0,0 +1,60 @@
+import { asSnippet, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
+import { FETCH_CONCURRENCY } from "../limits.js";
+import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
+import { requireKey } from "../config.js";
+
+export class FirecrawlProvider implements SearchProvider, FetchProvider {
+  private key: string;
+  constructor(config: WebKitConfig) { this.key = requireKey(config, "firecrawl"); }
+  private headers() { return { "content-type": "application/json", authorization: `Bearer ${this.key}` }; }
+
+  async search(input: SearchInput, signal?: AbortSignal) {
+    const data = await requestJson<any>("https://api.firecrawl.dev/v2/search", {
+      method: "POST", headers: this.headers(), signal, timeoutMs: 45000,
+      body: JSON.stringify({
+        query: input.query,
+        limit: input.numResults ?? 10,
+        location: input.location,
+        country: input.country,
+        includeDomains: input.includeDomains,
+        excludeDomains: input.excludeDomains,
+        categories: input.categories,
+        tbs: input.tbs,
+        scrapeOptions: input.scrapeOptions ?? (input.scrape === true ? { formats: ["markdown"] } : undefined),
+      }),
+    });
+    const list = data.data?.web ?? data.web ?? data.data ?? [];
+    return { provider: "firecrawl" as const, query: input.query, results: list.map((r: any, i: number) => ({
+      title: r.title, url: r.url, snippet: asSnippet(r.description ?? r.markdown ?? r.content), siteName: r.siteName, position: i + 1,
+    })).filter((r: any) => r.url) };
+  }
+
+  async fetch(input: FetchInput, signal?: AbortSignal) {
+    const urls = normalizeUrls(input);
+    const results = await mapConcurrent(urls, FETCH_CONCURRENCY, async (url) => {
+      try {
+        const format = input.format ?? "markdown";
+        const formats = [format];
+        const data = await requestJson<any>("https://api.firecrawl.dev/v2/scrape", {
+          method: "POST", headers: this.headers(), signal, timeoutMs: 90000,
+          body: JSON.stringify({
+            url,
+            formats,
+            onlyMainContent: input.onlyMainContent ?? true,
+            waitFor: input.waitFor,
+            mobile: input.mobile,
+            location: input.location,
+            maxAge: input.maxAge,
+          }),
+        });
+        const d = data.data ?? data;
+        const selected = format === "html" ? d.html : format === "json" ? d.json : d.markdown;
+        const content = typeof selected === "string" ? selected : selected == null ? undefined : JSON.stringify(selected, null, 2);
+        return { url: d.url ?? url, content, format, title: d.metadata?.title, metadata: d.metadata ?? d };
+      } catch (e) {
+        return { url, error: e instanceof Error ? e.message : String(e) };
+      }
+    });
+    return { provider: "firecrawl" as const, results };
+  }
+}

package/src/providers/index.ts

@@ -0,0 +1,30 @@
+import { validateFetchProvider, validateSearchProvider } from "../config.js";
+import type { FetchProvider, SearchProvider, WebKitConfig } from "../types.js";
+import { BraveProvider } from "./brave.js";
+import { ExaMcpProvider } from "./exa-mcp.js";
+import { ExaProvider } from "./exa.js";
+import { FirecrawlProvider } from "./firecrawl.js";
+import { MarkdownNewProvider } from "./markdown-new.js";
+import { TinyFishProvider } from "./tinyfish.js";
+
+export function createSearchProvider(config: WebKitConfig): SearchProvider {
+  validateSearchProvider(config.provider_search);
+  switch (config.provider_search) {
+    case "exa_mcp": return new ExaMcpProvider(config);
+    case "exa": return new ExaProvider(config);
+    case "tinyfish": return new TinyFishProvider(config);
+    case "brave": return new BraveProvider(config);
+    case "firecrawl": return new FirecrawlProvider(config);
+  }
+}
+
+export function createFetchProvider(config: WebKitConfig): FetchProvider {
+  validateFetchProvider(config.provider_fetch);
+  switch (config.provider_fetch) {
+    case "exa_mcp": return new ExaMcpProvider(config);
+    case "exa": return new ExaProvider(config);
+    case "tinyfish": return new TinyFishProvider(config);
+    case "markdown_new": return new MarkdownNewProvider(config);
+    case "firecrawl": return new FirecrawlProvider(config);
+  }
+}

package/src/providers/markdown-new.ts

@@ -0,0 +1,32 @@
+import { FETCH_CONCURRENCY } from "../limits.js";
+import { fetchWithTimeout, mapConcurrent, normalizeUrls } from "../http.js";
+import type { FetchInput, FetchProvider, WebKitConfig } from "../types.js";
+
+export class MarkdownNewProvider implements FetchProvider {
+  constructor(private config: WebKitConfig) {}
+
+  async fetch(input: FetchInput, signal?: AbortSignal) {
+    const urls = normalizeUrls(input);
+    const results = await mapConcurrent(urls, FETCH_CONCURRENCY, async (url) => {
+      try {
+        const res = await fetchWithTimeout("https://markdown.new/", {
+          method: "POST",
+          headers: { "content-type": "application/json", accept: "text/markdown, text/plain, */*" },
+          body: JSON.stringify({
+            url,
+            method: input.method ?? this.config.markdownNew.method,
+            retainImages: input.retainImages ?? this.config.markdownNew.retainImages,
+          }),
+          signal,
+          timeoutMs: 45_000,
+        });
+        const text = await res.text();
+        if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
+        return { url, content: text, format: "markdown" as const };
+      } catch (e) {
+        return { url, error: e instanceof Error ? e.message : String(e) };
+      }
+    });
+    return { provider: "markdown_new" as const, results };
+  }
+}

package/src/providers/tinyfish.ts

@@ -0,0 +1,39 @@
+import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
+import { requireKey } from "../config.js";
+
+export class TinyFishProvider implements SearchProvider, FetchProvider {
+  private key: string;
+  constructor(config: WebKitConfig) { this.key = requireKey(config, "tinyfish"); }
+
+  async search(input: SearchInput, signal?: AbortSignal) {
+    const url = new URL("https://api.search.tinyfish.ai/");
+    url.searchParams.set("query", input.query);
+    if (input.numResults) url.searchParams.set("limit", String(input.numResults));
+    if (typeof input.page === "number") url.searchParams.set("page", String(Math.min(input.page, 10)));
+    const data = await requestJson<any>(url.toString(), { headers: { "X-API-Key": this.key }, signal, timeoutMs: 10000 });
+    const list = data.results ?? data.data ?? data.web ?? [];
+    return { provider: "tinyfish" as const, query: input.query, results: list.map((r: any, i: number) => ({
+      title: r.title, url: r.url ?? r.link, snippet: asSnippet(r.snippet ?? r.description ?? r.text), siteName: r.siteName ?? r.source, position: r.position ?? i + 1,
+    })).filter((r: any) => r.url) };
+  }
+
+  async fetch(input: FetchInput, signal?: AbortSignal) {
+    const urls = normalizeUrls(input);
+    if (urls.length > 10) throw new Error("TinyFish fetch supports a maximum of 10 URLs per request.");
+    const data = await requestJson<any>("https://api.fetch.tinyfish.ai", {
+      method: "POST",
+      headers: { "content-type": "application/json", "X-API-Key": this.key },
+      body: JSON.stringify({ urls, format: input.format ?? "markdown", links: input.links, image_links: input.imageLinks }),
+      signal,
+      timeoutMs: 150000,
+    });
+    const list = data.results ?? data.data ?? data.pages ?? [];
+    return { provider: "tinyfish" as const, results: urls.map((url, i) => {
+      const r = list.find((x: any) => (x.url ?? x.source_url) === url) ?? list[i];
+      if (!r) return { url, error: "No content returned by TinyFish." };
+      const content = r.text ?? r.content ?? r.markdown ?? r.html;
+      return { url, content: typeof content === "string" ? content : JSON.stringify(content, null, 2), format: input.format ?? r.format ?? "markdown", title: r.title, metadata: r, error: r.error };
+    }) };
+  }
+}

package/src/types.ts

@@ -0,0 +1,47 @@
+export type SearchProviderName = "exa_mcp" | "exa" | "tinyfish" | "brave" | "firecrawl";
+export type FetchProviderName = "exa_mcp" | "exa" | "tinyfish" | "markdown_new" | "firecrawl";
+export type FetchFormat = "markdown" | "html" | "json";
+
+export interface WebKitConfig {
+  provider_search: SearchProviderName;
+  provider_fetch: FetchProviderName;
+  apiKeys: Partial<Record<"exa" | "tinyfish" | "brave" | "firecrawl", string>>;
+  markdownNew: { method: "auto" | "ai" | "browser"; retainImages: boolean };
+}
+
+export interface SearchInput {
+  query: string;
+  numResults?: number;
+  [key: string]: unknown;
+}
+
+export interface WebSearchResult {
+  provider: SearchProviderName;
+  query: string;
+  results: Array<{ title?: string; url: string; snippet?: string; siteName?: string; position?: number }>;
+}
+
+export interface FetchInput {
+  url?: string;
+  urls?: string[];
+  offset?: number;
+  limit?: number;
+  refresh?: boolean;
+  format?: FetchFormat;
+  links?: boolean;
+  imageLinks?: boolean;
+  [key: string]: unknown;
+}
+
+export interface WebFetchResult {
+  provider: FetchProviderName;
+  results: Array<{ url: string; content?: string; format?: FetchFormat; title?: string; metadata?: Record<string, unknown>; error?: string }>;
+}
+
+export interface SearchProvider {
+  search(input: SearchInput, signal?: AbortSignal): Promise<WebSearchResult>;
+}
+
+export interface FetchProvider {
+  fetch(input: FetchInput, signal?: AbortSignal): Promise<WebFetchResult>;
+}

package/src/urls.ts

@@ -0,0 +1,40 @@
+import { MAX_URL_COUNT, MAX_URL_LENGTH } from "./limits.js";
+
+export function normalizeWebUrl(value: string): string {
+  if (value.length > MAX_URL_LENGTH) throw new Error(`URL length must be <= ${MAX_URL_LENGTH} characters.`);
+  let parsed: URL;
+  try {
+    parsed = new URL(value.trim());
+  } catch {
+    throw new Error(`Malformed URL: ${value}`);
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error(`URL scheme must be http or https: ${value}`);
+  if (parsed.username || parsed.password) throw new Error(`URL credentials are not allowed: ${value}`);
+  parsed.hash = "";
+  return parsed.toString();
+}
+
+export function canonicalWebUrl(value: string): string {
+  const parsed = new URL(normalizeWebUrl(value));
+  parsed.hostname = parsed.hostname.toLowerCase();
+  if ((parsed.protocol === "https:" && parsed.port === "443") || (parsed.protocol === "http:" && parsed.port === "80")) parsed.port = "";
+  return parsed.toString();
+}
+
+export function normalizeUrlInput(input: { url?: string; urls?: string[] }, maxCount = MAX_URL_COUNT): string[] {
+  const raw = [...(Array.isArray(input.urls) ? input.urls : []), ...(input.url ? [input.url] : [])].map((url) => String(url).trim()).filter(Boolean);
+  const unique = [...new Set(raw.map(normalizeWebUrl))];
+  if (unique.length > maxCount) throw new Error(`Too many URLs: maximum is ${maxCount}.`);
+  return unique;
+}
+
+export function urlsMatch(a: string | undefined, b: string | undefined): boolean {
+  if (!a || !b) return false;
+  try {
+    if (canonicalWebUrl(a) === canonicalWebUrl(b)) return true;
+    const trimSlash = (s: string) => s.endsWith("/") ? s.slice(0, -1) : s;
+    return trimSlash(canonicalWebUrl(a)) === trimSlash(canonicalWebUrl(b));
+  } catch {
+    return a === b;
+  }
+}