pi-web-kit 0.2.4 → 0.4.0

package/src/providers/tinyfish.ts

@@ -1,6 +1,7 @@
-import { asSnippet, normalizeUrls, requestJson } from "../http.js";
-import { MAX_URL_COUNT } from "../limits.js";
-import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
+import { asSnippet, normalizeUrls, requestJson, withoutContent } from "../http.js";
+import { DEFAULT_NUM_RESULTS, MAX_URL_COUNT, TINYFISH_MAX_PAGE } from "../limits.js";
+import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig, WebSearchResult } from "../types.js";
+import { canonicalWebUrl, urlsMatch } from "../urls.js";
 import { requireKey } from "../config.js";
 
 export class TinyFishProvider implements SearchProvider, FetchProvider {
@@ -8,15 +9,81 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
   constructor(config: WebKitConfig) { this.key = requireKey(config, "tinyfish"); }
 
   async search(input: SearchInput, signal?: AbortSignal) {
-    const url = new URL("https://api.search.tinyfish.ai/");
-    url.searchParams.set("query", input.query);
-    if (input.numResults) url.searchParams.set("limit", String(input.numResults));
-    if (typeof input.page === "number") url.searchParams.set("page", String(Math.min(input.page, 10)));
-    const data = await requestJson<any>(url.toString(), { headers: { "X-API-Key": this.key }, signal, timeoutMs: 10000 });
-    const list = data.results ?? data.data ?? data.web ?? [];
-    return { provider: "tinyfish" as const, query: input.query, results: list.map((r: any, i: number) => ({
-      title: r.title, url: r.url ?? r.link, snippet: asSnippet(r.snippet ?? r.description ?? r.text), siteName: r.siteName ?? r.source, position: r.position ?? i + 1,
-    })).filter((r: any) => r.url) };
+    const desired = input.numResults ?? DEFAULT_NUM_RESULTS;
+    const firstPage = typeof input.page === "number" ? Math.max(0, Math.min(input.page, TINYFISH_MAX_PAGE)) : 0;
+    const research = input.domainType === "research_paper";
+    const dates = orderedStrings(input.afterDate, input.beforeDate);
+    const years = orderedNumbers(input.pubYearMin, input.pubYearMax);
+    const results: WebSearchResult["results"] = [];
+    const seen = new Set<string>();
+    let lastPage = firstPage - 1;
+
+    for (let page = firstPage; page <= TINYFISH_MAX_PAGE && results.length < desired; page++) {
+      lastPage = page;
+      const url = new URL("https://api.search.tinyfish.ai/");
+      url.searchParams.set("query", input.query);
+      url.searchParams.set("page", String(page));
+      setString(url, "purpose", input.purpose);
+      setString(url, "location", input.location);
+      setString(url, "language", input.language);
+      setDomains(url, "include_domains", input.includeDomains);
+      setDomains(url, "exclude_domains", input.excludeDomains);
+      setString(url, "domain_type", input.domainType);
+      if (research) {
+        setNumber(url, "pub_year_min", years[0]);
+        setNumber(url, "pub_year_max", years[1]);
+      } else if (typeof input.recencyMinutes === "number") {
+        setNumber(url, "recency_minutes", input.recencyMinutes);
+      } else {
+        setString(url, "after_date", dates[0]);
+        setString(url, "before_date", dates[1]);
+      }
+      const data = await requestJson<any>(url.toString(), { headers: { "X-API-Key": this.key }, signal, timeoutMs: 10000 });
+      const list = data.results ?? data.data ?? data.web ?? [];
+      if (!Array.isArray(list) || list.length === 0) break;
+      for (const r of list) {
+        const resultUrl = r.url ?? r.link;
+        const key = typeof resultUrl === "string" ? canonicalResultUrl(resultUrl) : undefined;
+        if (!key || seen.has(key)) continue;
+        seen.add(key);
+        results.push({
+          title: r.title,
+          url: resultUrl,
+          snippet: asSnippet(r.snippet ?? r.description ?? r.text),
+          siteName: r.siteName ?? r.source,
+          position: results.length + 1,
+        });
+        if (results.length === desired) break;
+      }
+      if (typeof data.total_results === "number" && results.length >= data.total_results) break;
+    }
+
+    if (input.contextTokens && results.length) {
+      const contextCharacters = input.contextTokens * 4;
+      let extractedCharacters = 0;
+      for (let i = 0; i < results.length && extractedCharacters < contextCharacters;) {
+        // ponytail: budget for about 512 useful tokens per page; fetch another batch when pages are short.
+        const batchSize = Math.min(MAX_URL_COUNT, Math.max(1, Math.ceil((contextCharacters - extractedCharacters) / 2_048)));
+        const batch = results.slice(i, i + batchSize);
+        const fetched = await this.fetch({ urls: batch.map((item) => item.url), format: "markdown", purpose: input.purpose }, signal);
+        for (const item of batch) {
+          const page = fetched.results.find((candidate) => urlsMatch(candidate.url, item.url) || urlsMatch(candidate.metadata?.requestedUrl, item.url));
+          if (page?.content) {
+            item.content = page.content;
+            item.contentFormat = "markdown";
+            extractedCharacters += page.content.length;
+          }
+        }
+        i += batch.length;
+      }
+    }
+
+    return {
+      provider: "tinyfish" as const,
+      query: input.query,
+      effectiveResultLimit: lastPage === TINYFISH_MAX_PAGE && results.length < desired ? results.length : desired,
+      results,
+    };
   }
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
@@ -25,20 +92,73 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
     const data = await requestJson<any>("https://api.fetch.tinyfish.ai", {
       method: "POST",
       headers: { "content-type": "application/json", "X-API-Key": this.key },
-      body: JSON.stringify({ urls, format: input.format ?? "markdown", links: input.links, image_links: input.imageLinks }),
+      body: JSON.stringify({
+        urls,
+        purpose: input.purpose,
+        format: input.format ?? "markdown",
+        links: input.links,
+        image_links: input.imageLinks,
+        ttl: input.refresh === true
+          ? 0
+          : input.ttl ?? (typeof input.maxAgeMs === "number" ? Math.floor(input.maxAgeMs / 1_000) : undefined),
+        per_url_timeout_ms: input.perUrlTimeoutMs,
+        include_selectors: input.includeSelectors,
+        exclude_selectors: input.excludeSelectors,
+      }),
       signal,
       timeoutMs: 150000,
     });
     const list = data.results ?? data.data ?? data.pages ?? [];
-    return { provider: "tinyfish" as const, results: urls.map((url, i) => {
-      const r = list.find((x: any) => (x.url ?? x.source_url) === url) ?? list[i];
-      if (!r) return { url, error: "No content returned by TinyFish." };
+    const errors = data.errors ?? [];
+    return { provider: "tinyfish" as const, results: urls.map((url) => {
+      const r = list.find((x: any) => urlsMatch(x.url ?? x.source_url, url));
+      const failure = errors.find((x: any) => urlsMatch(x.url, url));
+      if (!r) return { url, error: failure ? `${failure.error}${failure.status ? ` (${failure.status})` : ""}` : "No content returned by TinyFish." };
       const content = r.text ?? r.content ?? r.markdown ?? r.html;
-      return { url, content: stringifyContent(content), format: input.format ?? r.format ?? "markdown", title: r.title, metadata: r, error: r.error };
+      return {
+        url: r.final_url ?? r.url ?? url,
+        content: stringifyContent(content),
+        format: input.format ?? r.format ?? "markdown",
+        title: r.title,
+        metadata: { ...withoutContent(r), requestedUrl: url, finalUrl: r.final_url },
+        error: r.error,
+      };
     }) };
   }
 }
 
+function setString(url: URL, name: string, value: unknown) {
+  if (typeof value === "string" && value) url.searchParams.set(name, value);
+}
+
+function setNumber(url: URL, name: string, value: unknown) {
+  if (typeof value === "number") url.searchParams.set(name, String(value));
+}
+
+function setDomains(url: URL, name: string, value: unknown) {
+  if (Array.isArray(value) && value.length) url.searchParams.set(name, value.join(","));
+  else setString(url, name, value);
+}
+
+function orderedStrings(a: unknown, b: unknown): [string | undefined, string | undefined] {
+  const values = [a, b].filter((value): value is string => typeof value === "string").sort();
+  return [values[0], values[1]];
+}
+
+function orderedNumbers(a: unknown, b: unknown): [number | undefined, number | undefined] {
+  const values = [a, b].filter((value): value is number => typeof value === "number").sort((x, y) => x - y);
+  return [values[0], values[1]];
+}
+
 function stringifyContent(value: unknown): string | undefined {
   return typeof value === "string" ? value : JSON.stringify(value, null, 2);
 }
+
+function canonicalResultUrl(value: string): string {
+  try {
+    const canonical = canonicalWebUrl(value);
+    return canonical.endsWith("/") ? canonical.slice(0, -1) : canonical;
+  } catch {
+    return value;
+  }
+}

package/src/types.ts

@@ -1,5 +1,5 @@
-export type SearchProviderName = "exa_mcp" | "exa" | "tinyfish" | "brave" | "firecrawl";
-export type FetchProviderName = "exa_mcp" | "exa" | "tinyfish" | "markdown_new" | "firecrawl";
+export type SearchProviderName = "exa" | "tinyfish" | "brave" | "firecrawl";
+export type FetchProviderName = "exa" | "tinyfish" | "markdown_new" | "firecrawl";
 export type FetchFormat = "markdown" | "html" | "json";
 
 export interface WebKitConfig {
@@ -12,13 +12,16 @@ export interface WebKitConfig {
 export interface SearchInput {
   query: string;
   numResults?: number;
+  contextTokens?: number;
+  purpose?: string;
   [key: string]: unknown;
 }
 
 export interface WebSearchResult {
   provider: SearchProviderName;
   query: string;
-  results: Array<{ title?: string; url: string; snippet?: string; siteName?: string; position?: number }>;
+  effectiveResultLimit?: number;
+  results: Array<{ title?: string; url: string; snippet?: string; content?: string; contentFormat?: "markdown" | "text"; siteName?: string; position?: number }>;
 }
 
 export interface FetchInput {
@@ -27,9 +30,15 @@ export interface FetchInput {
   offset?: number;
   limit?: number;
   refresh?: boolean;
+  maxAgeMs?: number;
   format?: FetchFormat;
   links?: boolean;
   imageLinks?: boolean;
+  purpose?: string;
+  ttl?: number;
+  perUrlTimeoutMs?: number;
+  includeSelectors?: string[];
+  excludeSelectors?: string[];
   [key: string]: unknown;
 }
 

package/src/urls.ts

@@ -6,10 +6,10 @@ export function normalizeWebUrl(value: string): string {
   try {
     parsed = new URL(value.trim());
   } catch {
-    throw new Error(`Malformed URL: ${value}`);
+    throw new Error("Malformed URL.");
   }
-  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error(`URL scheme must be http or https: ${value}`);
-  if (parsed.username || parsed.password) throw new Error(`URL credentials are not allowed: ${value}`);
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("URL scheme must be http or https.");
+  if (parsed.username || parsed.password) throw new Error("URL credentials are not allowed.");
   parsed.hash = "";
   return parsed.toString();
 }

package/src/providers/exa-mcp.ts

@@ -1,102 +0,0 @@
-import { asSnippet, fetchWithTimeout, normalizeUrls } from "../http.js";
-import { DEFAULT_NUM_RESULTS } from "../limits.js";
-import { urlsMatch } from "../urls.js";
-import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
-import { applyExaFetchFallbacks } from "./fallback.js";
-
-export class ExaMcpProvider implements SearchProvider, FetchProvider {
-  private sessionId?: string;
-  private nextId = 1;
-  constructor(private config: WebKitConfig) {}
-
-  async search(input: SearchInput, signal?: AbortSignal) {
-    const result = await this.callTool("web_search_exa", { query: input.query, numResults: input.numResults ?? DEFAULT_NUM_RESULTS }, signal);
-    return { provider: "exa_mcp" as const, query: input.query, results: normalizeSearch(result) };
-  }
-
-  async fetch(input: FetchInput, signal?: AbortSignal) {
-    const urls = normalizeUrls(input);
-    const result = await this.callTool("web_fetch_exa", { urls }, signal);
-    const pages = normalizeFetch(result, urls);
-    return applyExaFetchFallbacks(this.config, input, urls, { provider: "exa_mcp" as const, results: pages }, signal);
-  }
-
-  private async callTool(name: string, args: Record<string, unknown>, signal?: AbortSignal): Promise<any> {
-    await this.initialize(signal);
-    return this.rpc("tools/call", { name, arguments: args }, signal);
-  }
-
-  private async initialize(signal?: AbortSignal) {
-    if (this.sessionId) return;
-    const data = await this.rpcRaw("initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "pi-web-kit", version: "0.1.0" } }, signal);
-    this.sessionId = data.sessionId ?? data.payload?.sessionId;
-    if (!this.sessionId) throw new Error("Exa MCP initialize did not return an mcp-session-id response header.");
-    await this.rpcRaw("notifications/initialized", undefined, signal, true);
-  }
-
-  private async rpc(method: string, params: unknown, signal?: AbortSignal) {
-    const data = await this.rpcRaw(method, params, signal);
-    if (data.payload?.error) throw new Error(data.payload.error.message ?? JSON.stringify(data.payload.error));
-    return data.payload?.result ?? data.payload;
-  }
-
-  private async rpcRaw(method: string, params: unknown, signal?: AbortSignal, notification = false): Promise<{ payload?: any; sessionId?: string }> {
-    const headers: Record<string, string> = { "content-type": "application/json", accept: "application/json, text/event-stream" };
-    if (this.sessionId) headers["mcp-session-id"] = this.sessionId;
-    if (this.config.apiKeys.exa) headers["x-api-key"] = this.config.apiKeys.exa;
-    const body = notification ? { jsonrpc: "2.0", method, params } : { jsonrpc: "2.0", id: this.nextId++, method, params };
-    const res = await fetchWithTimeout("https://mcp.exa.ai/mcp", { method: "POST", headers, body: JSON.stringify(body), signal, timeoutMs: 45_000 });
-    const text = await res.text();
-    if (!res.ok) throw new Error(`Exa MCP ${method} failed: ${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
-    if (notification) return { sessionId: this.sessionId };
-    const sessionId = res.headers.get("mcp-session-id") ?? undefined;
-    const payload = parseMcpResponse(text, res.headers.get("content-type") ?? "");
-    return { payload, sessionId };
-  }
-}
-
-function parseMcpResponse(text: string, contentType: string): any {
-  if (!text.trim()) return undefined;
-  if (contentType.includes("text/event-stream") || text.startsWith("event:") || text.startsWith("data:")) {
-    const data = text.split(/\r?\n/).filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).join("\n");
-    return data ? JSON.parse(data) : undefined;
-  }
-  return JSON.parse(text);
-}
-
-function textFromContent(result: any): string {
-  const content = result?.content ?? result;
-  if (Array.isArray(content)) return content.map((c) => typeof c === "string" ? c : c.text ?? JSON.stringify(c)).join("\n");
-  return typeof content === "string" ? content : JSON.stringify(content);
-}
-
-function normalizeSearch(result: any) {
-  const structured = result?.structuredContent ?? result?.result ?? result;
-  const list = structured.results ?? structured.data ?? structured.items;
-  if (Array.isArray(list)) return list.map(toSearchResult).filter((r: any) => r.url);
-  const text = textFromContent(result);
-  const urls = [...text.matchAll(/https?:\/\/[^\s)\]}>"']+/g)].map((m) => m[0]);
-  return [...new Set(urls)].map((url, i) => ({ url, snippet: i === 0 ? text.slice(0, 1000) : undefined, position: i + 1 }));
-}
-
-function toSearchResult(r: any, index: number) {
-  return {
-    title: r.title,
-    url: r.url,
-    snippet: asSnippet(r.snippet ?? r.text ?? r.summary ?? r.highlights),
-    siteName: r.siteName,
-    position: index + 1,
-  };
-}
-
-function normalizeFetch(result: any, urls: string[]) {
-  const structured = result?.structuredContent ?? result?.result ?? result;
-  const list = structured.results ?? structured.data ?? structured.pages;
-  if (Array.isArray(list)) return urls.map((url, i) => {
-    const r = list.find((x: any) => urlsMatch(x.url, url)) ?? list[i];
-    if (!r) return { url, error: "No content returned by Exa MCP." };
-    return { url, title: r.title, content: r.markdown ?? r.text ?? r.content ?? r.html, format: "markdown" as const, metadata: r, error: r.error };
-  });
-  const text = textFromContent(result);
-  return urls.length <= 1 ? [{ url: urls[0] ?? "", content: text, format: "markdown" as const }] : urls.map((url) => ({ url, content: text, format: "markdown" as const }));
-}