pi-web-kit 0.1.0

package/extensions/pi-web-kit/index.ts

@@ -0,0 +1,408 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+import { fetchCache, type CachedPage } from "../../src/cache.js";
+import { resolveConfig } from "../../src/config.js";
+import { DEFAULT_FETCH_LIMIT, DEFAULT_NUM_RESULTS, MAX_LIMIT, MAX_NUM_RESULTS, MAX_OFFSET, MAX_QUERY_COUNT, MAX_URL_COUNT, MULTI_FETCH_LIMIT } from "../../src/limits.js";
+import { truncateText } from "../../src/http.js";
+import { createFetchProvider, createSearchProvider } from "../../src/providers/index.js";
+import type { FetchProviderName, SearchProviderName, WebFetchResult } from "../../src/types.js";
+import { canonicalWebUrl, normalizeUrlInput, urlsMatch } from "../../src/urls.js";
+
+export default function (pi: ExtensionAPI) {
+  pi.registerFlag("web-provider-search", {
+    description: "Temporary pi-web-kit search provider override (exa_mcp, exa, tinyfish, brave, firecrawl)",
+    type: "string",
+  });
+  pi.registerFlag("web-provider-fetch", {
+    description: "Temporary pi-web-kit fetch provider override (exa_mcp, exa, tinyfish, markdown_new, firecrawl)",
+    type: "string",
+  });
+
+  const startupConfig = resolveConfig({
+    providerSearch: pi.getFlag("web-provider-search"),
+    providerFetch: pi.getFlag("web-provider-fetch"),
+  });
+
+  pi.registerTool({
+    name: "web_search",
+    label: "Web Search",
+    description: buildSearchDescription(startupConfig.provider_search),
+    promptSnippet: "Find current or external web information.",
+    promptGuidelines: ["Use web_search to find current or external web information."],
+    parameters: buildSearchSchema(startupConfig.provider_search),
+    async execute(_toolCallId, rawParams, signal, onUpdate, ctx) {
+      const params = rawParams as Record<string, any>;
+      const queries = normalizeQueries(params);
+      const numResults = parseInteger(params.numResults, DEFAULT_NUM_RESULTS, "numResults", 1, MAX_NUM_RESULTS);
+      if (queries.length === 0) throw new Error("web_search requires query or queries.");
+
+      const config = resolveConfig({ providerSearch: pi.getFlag("web-provider-search") }, ctx.cwd);
+      assertProviderUnchanged("web_search", startupConfig.provider_search, config.provider_search);
+      const provider = createSearchProvider(config);
+      const grouped = [];
+      const progress = createProgress("search", config.provider_search, queries);
+      emitProgress(onUpdate, progress);
+      for (const query of queries) {
+        markProgressCurrent(progress, query);
+        emitProgress(onUpdate, progress);
+        const result = await provider.search({ ...params, query, numResults }, signal);
+        grouped.push({ query, results: result.results });
+        markProgressDone(progress, query, `${result.results.length} results`);
+        emitProgress(onUpdate, progress);
+      }
+      const result = { provider: config.provider_search, queries: grouped };
+      const text = truncateText(JSON.stringify(result, null, 2));
+      return { content: [{ type: "text", text }], details: boundedDetails(result) };
+    },
+    renderCall(args, theme) {
+      return new Text(renderWebCall("search", args as Record<string, any>, theme), 0, 0);
+    },
+    renderResult(result, options, theme, context) {
+      return renderWebResult("search", result, options, theme, context);
+    },
+  });
+
+  pi.registerTool({
+    name: "web_fetch",
+    label: "Web Fetch",
+    description: buildFetchDescription(startupConfig.provider_fetch),
+    promptSnippet: "Read page content from URL(s), with offset/limit for long pages.",
+    promptGuidelines: ["Use web_fetch when the user provides URLs or asks to read page content."],
+    parameters: buildFetchSchema(startupConfig.provider_fetch),
+    async execute(_toolCallId, rawParams, signal, onUpdate, ctx) {
+      const params = rawParams as Record<string, any>;
+      const urls = normalizeUrls(params);
+      if (urls.length === 0) throw new Error("web_fetch requires url or urls.");
+      if (urls.length > 1 && params.offset != null && params.offset !== 0) {
+        throw new Error("web_fetch offset range reads require a single url, not urls.");
+      }
+
+      const config = resolveConfig({ providerFetch: pi.getFlag("web-provider-fetch") }, ctx.cwd);
+      assertProviderUnchanged("web_fetch", startupConfig.provider_fetch, config.provider_fetch);
+      const progress = createProgress("fetch", config.provider_fetch, urls);
+      const result = await fetchWithCache(config.provider_fetch, params, urls, signal, config, (event) => {
+        if (event.status === "current") markProgressCurrent(progress, event.url);
+        else if (event.status === "done") markProgressDone(progress, event.url, event.note);
+        else if (event.status === "error") markProgressError(progress, event.url, event.error);
+        emitProgress(onUpdate, progress);
+      });
+      const text = truncateText(JSON.stringify(result, null, 2));
+      return { content: [{ type: "text", text }], details: boundedDetails(result) };
+    },
+    renderCall(args, theme) {
+      return new Text(renderWebCall("fetch", args as Record<string, any>, theme), 0, 0);
+    },
+    renderResult(result, options, theme, context) {
+      return renderWebResult("fetch", result, options, theme, context);
+    },
+  });
+}
+
+type ProgressKind = "search" | "fetch";
+type ProgressItem = { label: string; status: "pending" | "current" | "done" | "error"; note?: string; error?: string };
+type WebProgress = { kind: ProgressKind; provider: string; total: number; completed: number; items: ProgressItem[] };
+type FetchProgressEvent = { status: "current" | "done" | "error"; url: string; note?: string; error?: string };
+
+const int = (description: string, min: number, max?: number) => Type.Integer({ description, minimum: min, ...(max == null ? {} : { maximum: max }) });
+
+export function buildSearchSchema(provider: SearchProviderName) {
+  const props: Record<string, any> = {
+    query: Type.Optional(Type.String({ description: "Single search query" })),
+    queries: Type.Optional(Type.Array(Type.String(), { description: `Multiple related search queries (max ${MAX_QUERY_COUNT})`, maxItems: MAX_QUERY_COUNT })),
+    numResults: Type.Optional(int("Results per query", 1, MAX_NUM_RESULTS)),
+  };
+  if (provider === "exa") Object.assign(props, {
+    includeDomains: Type.Optional(Type.Array(Type.String())),
+    excludeDomains: Type.Optional(Type.Array(Type.String())),
+    startPublishedDate: Type.Optional(Type.String()),
+    endPublishedDate: Type.Optional(Type.String()),
+    startCrawlDate: Type.Optional(Type.String()),
+    endCrawlDate: Type.Optional(Type.String()),
+    type: Type.Optional(Type.String()),
+    category: Type.Optional(Type.String()),
+  });
+  if (provider === "tinyfish") Object.assign(props, { page: Type.Optional(int("Result page", 1, 10)) });
+  if (provider === "brave") Object.assign(props, {
+    country: Type.Optional(Type.String()), searchLang: Type.Optional(Type.String()), uiLang: Type.Optional(Type.String()), safesearch: Type.Optional(Type.String()), freshness: Type.Optional(Type.String()), maxUrls: Type.Optional(int("Maximum URLs", 1, MAX_NUM_RESULTS)),
+  });
+  if (provider === "firecrawl") Object.assign(props, {
+    location: Type.Optional(Type.String()), country: Type.Optional(Type.String()), includeDomains: Type.Optional(Type.Array(Type.String())), excludeDomains: Type.Optional(Type.Array(Type.String())), categories: Type.Optional(Type.Array(Type.String())), tbs: Type.Optional(Type.String()), scrape: Type.Optional(Type.Boolean({ description: "Enable default markdown scrape-on-search" })), scrapeOptions: Type.Optional(Type.Object({}, { additionalProperties: true, description: "Firecrawl scrapeOptions for search." })),
+  });
+  return Type.Object(props, { additionalProperties: false });
+}
+
+export function buildFetchSchema(provider: FetchProviderName) {
+  const props: Record<string, any> = {
+    url: Type.Optional(Type.String({ description: "Single URL", maxLength: 2048 })),
+    urls: Type.Optional(Type.Array(Type.String({ maxLength: 2048 }), { description: `Multiple URLs (max ${MAX_URL_COUNT})`, maxItems: MAX_URL_COUNT })),
+    offset: Type.Optional(int("Character offset for cached/ranged reads", 0, MAX_OFFSET)),
+    limit: Type.Optional(int("Maximum characters to return", 1, MAX_LIMIT)),
+    refresh: Type.Optional(Type.Boolean({ description: "Refetch even if cached" })),
+  };
+  if (provider === "tinyfish") Object.assign(props, { format: Type.Optional(Type.Union([Type.Literal("markdown"), Type.Literal("html"), Type.Literal("json")])), links: Type.Optional(Type.Boolean()), imageLinks: Type.Optional(Type.Boolean()) });
+  if (provider === "markdown_new") Object.assign(props, { method: Type.Optional(Type.Union([Type.Literal("auto"), Type.Literal("ai"), Type.Literal("browser")])), retainImages: Type.Optional(Type.Boolean()) });
+  if (provider === "firecrawl") Object.assign(props, {
+    format: Type.Optional(Type.Union([Type.Literal("markdown"), Type.Literal("html"), Type.Literal("json")])),
+    onlyMainContent: Type.Optional(Type.Boolean()), waitFor: Type.Optional(int("Milliseconds to wait", 0, 60_000)), mobile: Type.Optional(Type.Boolean()), location: Type.Optional(Type.String()), maxAge: Type.Optional(int("Maximum cached page age", 0)),
+  });
+  return Type.Object(props, { additionalProperties: false });
+}
+
+function buildSearchDescription(provider: SearchProviderName): string {
+  return `Search the web with startup provider '${provider}'. Use query or queries; returns compact results grouped by query. Restart/reload pi after provider changes.`;
+}
+
+function buildFetchDescription(provider: FetchProviderName): string {
+  return `Fetch URL content with startup provider '${provider}'. Results are cached by URL/options; use offset/limit to read long pages in chunks. Restart/reload pi after provider changes.`;
+}
+
+function normalizeQueries(params: Record<string, any>): string[] {
+  const raw = Array.isArray(params.queries) ? params.queries : params.query ? [params.query] : [];
+  const queries = [...new Set(raw.map((q) => String(q).trim()).filter(Boolean))];
+  if (queries.length > MAX_QUERY_COUNT) throw new Error(`Too many queries: maximum is ${MAX_QUERY_COUNT}.`);
+  return queries;
+}
+
+function normalizeUrls(params: Record<string, any>): string[] {
+  return normalizeUrlInput({ url: params.url, urls: params.urls }, MAX_URL_COUNT);
+}
+
+export async function fetchWithCache(providerName: FetchProviderName, params: Record<string, any>, urls: string[], signal: AbortSignal | undefined, config: any, onProgress?: (event: FetchProgressEvent) => void) {
+  const provider = createFetchProvider(config);
+  const offset = parseInteger(params.offset, 0, "offset", 0, MAX_OFFSET);
+  const defaultLimit = urls.length > 1 ? MULTI_FETCH_LIMIT : DEFAULT_FETCH_LIMIT;
+  const limit = parseInteger(params.limit, defaultLimit, "limit", 1, MAX_LIMIT);
+  const refresh = params.refresh === true;
+
+  const pages = new Map<string, { page?: CachedPage; cached: boolean; refreshed: boolean; error?: string }>();
+  const missing: string[] = [];
+  for (const url of urls) {
+    const cacheKey = buildCacheKey(providerName, url, params, config);
+    const cached = fetchCache.get(cacheKey);
+    if (cached && !refresh) {
+      pages.set(url, { page: cached, cached: true, refreshed: false });
+      onProgress?.({ status: "done", url, note: "cached" });
+    } else {
+      missing.push(url);
+    }
+  }
+
+  if (missing.length > 0) {
+    for (const url of missing) onProgress?.({ status: "current", url });
+    const fetched = await provider.fetch({ ...params, url: undefined, urls: missing }, signal);
+    const mapped = mapFetchResults(missing, fetched);
+    for (const requestedUrl of missing) {
+      const item = mapped.get(requestedUrl);
+      if (!item || item.error) {
+        const error = item?.error ?? "No content returned.";
+        pages.set(requestedUrl, { error, cached: false, refreshed: refresh });
+        onProgress?.({ status: "error", url: requestedUrl, error });
+        continue;
+      }
+      const cacheKey = buildCacheKey(providerName, requestedUrl, params, config);
+      const page = fetchCache.set(cacheKey, {
+        provider: providerName,
+        cacheKey,
+        requestedUrl,
+        url: item.url || requestedUrl,
+        title: item.title,
+        content: item.content ?? "",
+        format: item.format ?? "markdown",
+        metadata: item.metadata,
+        fetchedAt: Date.now(),
+      });
+      pages.set(requestedUrl, { page, cached: false, refreshed: refresh });
+      onProgress?.({ status: "done", url: requestedUrl, note: `${page.content.length} chars` });
+    }
+  }
+
+  return { provider: providerName, results: urls.map((url) => {
+    const entry = pages.get(url);
+    if (!entry?.page) return { url, error: entry?.error ?? "No content returned." };
+    return pageSlice(entry.page, offset, limit, entry.cached, entry.refreshed);
+  }) };
+}
+
+function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, WebFetchResult["results"][number]> {
+  const out = new Map<string, WebFetchResult["results"][number]>();
+  const remaining = [...(fetched.results ?? [])];
+  for (const url of requested) {
+    const index = remaining.findIndex((item) => urlsMatch(item.url, url));
+    if (index >= 0) out.set(url, remaining.splice(index, 1)[0]);
+  }
+  requested.forEach((url, index) => { if (!out.has(url) && fetched.results?.[index]) out.set(url, fetched.results[index]); });
+  return out;
+}
+
+export function pageSlice(page: CachedPage, offset: number, limit: number, cached: boolean, refreshed: boolean) {
+  const total = page.content.length;
+  const content = page.content.slice(offset, offset + limit);
+  return { url: page.requestedUrl ?? page.url, fetchedUrl: page.url, title: page.title, content, format: page.format, cached, refreshed, cacheKey: page.cacheKey, range: { offset, limit, returned: content.length, total, truncated: offset > 0 || offset + content.length < total, hasPrevious: offset > 0, hasNext: offset + content.length < total } };
+}
+
+export function buildCacheKey(provider: FetchProviderName, url: string, params: Record<string, any>, config?: any): string {
+  const canonical = canonicalWebUrl(url);
+  const affecting: Record<string, unknown> = {};
+  for (const key of Object.keys(params).sort()) {
+    if (["url", "urls", "offset", "limit", "refresh"].includes(key)) continue;
+    affecting[key] = params[key];
+  }
+  for (const [key, value] of Object.entries(fetchConfigDefaults(provider, config))) if (affecting[key] === undefined) affecting[key] = value;
+  const scope = providerScope(provider, config);
+  return `${provider}\0${scope}\0${JSON.stringify(affecting)}\0${canonical}`;
+}
+
+function parseInteger(value: unknown, defaultValue: number, name: string, min: number, max: number): number {
+  if (value == null) return defaultValue;
+  if (typeof value !== "number" || !Number.isInteger(value) || !Number.isFinite(value) || value < min || value > max) throw new Error(`${name} must be a finite integer between ${min} and ${max}.`);
+  return value;
+}
+
+function fetchConfigDefaults(provider: FetchProviderName, config?: any): Record<string, unknown> {
+  if (provider === "markdown_new") return { method: config?.markdownNew?.method ?? "auto", retainImages: config?.markdownNew?.retainImages ?? false };
+  if (provider === "firecrawl") return { onlyMainContent: true, format: "markdown" };
+  return {};
+}
+
+function providerScope(provider: FetchProviderName, config?: any): string {
+  const key = provider === "exa" || provider === "exa_mcp" ? config?.apiKeys?.exa : provider === "tinyfish" ? config?.apiKeys?.tinyfish : provider === "firecrawl" ? config?.apiKeys?.firecrawl : undefined;
+  return key ? `key:${String(key).slice(0, 8)}` : "default";
+}
+
+function assertProviderUnchanged(tool: string, startup: string, runtime: string) {
+  if (startup !== runtime) throw new Error(`${tool} was registered for provider '${startup}' but runtime config resolved '${runtime}'. Restart or reload pi after changing pi-web-kit provider config so tool schemas match the active provider.`);
+}
+
+function boundedDetails(value: any): unknown {
+  if (value?.queries && Array.isArray(value.queries)) return { provider: value.provider, queries: value.queries.map((q: any) => ({ query: q.query, resultCount: (q.results ?? []).length, results: (q.results ?? []).map((r: any) => ({ title: r.title, url: r.url, siteName: r.siteName, position: r.position })) })) };
+  if (value?.results && Array.isArray(value.results)) return { provider: value.provider, results: value.results.map((r: any) => ({ url: r.url, fetchedUrl: r.fetchedUrl, title: r.title, format: r.format, cached: r.cached, refreshed: r.refreshed, cacheKey: r.cacheKey, range: r.range, error: r.error })) };
+  return value;
+}
+
+function createProgress(kind: ProgressKind, provider: string, labels: string[]): WebProgress {
+  return { kind, provider, total: labels.length, completed: 0, items: labels.map((label) => ({ label, status: "pending" })) };
+}
+
+function markProgressCurrent(progress: WebProgress, label: string) {
+  const item = progress.items.find((i) => i.label === label);
+  if (item && item.status === "pending") item.status = "current";
+}
+
+function markProgressDone(progress: WebProgress, label: string, note?: string) {
+  const item = progress.items.find((i) => i.label === label);
+  if (!item) return;
+  item.status = "done";
+  item.note = note;
+  progress.completed = progress.items.filter((i) => i.status === "done" || i.status === "error").length;
+}
+
+function markProgressError(progress: WebProgress, label: string, error?: string) {
+  const item = progress.items.find((i) => i.label === label);
+  if (!item) return;
+  item.status = "error";
+  item.error = error;
+  progress.completed = progress.items.filter((i) => i.status === "done" || i.status === "error").length;
+}
+
+function emitProgress(onUpdate: ((patch: any) => void) | undefined, progress: WebProgress) {
+  const verb = progress.kind === "search" ? "Searching web" : "Fetching pages";
+  onUpdate?.({
+    content: [{ type: "text", text: `${verb}: ${progress.completed}/${progress.total}` }],
+    details: { progress: cloneProgress(progress) },
+  });
+}
+
+function cloneProgress(progress: WebProgress): WebProgress {
+  return { ...progress, items: progress.items.map((item) => ({ ...item })) };
+}
+
+function renderWebCall(kind: ProgressKind, args: Record<string, any>, theme: any): string {
+  const title = kind === "search" ? "web_search" : "web_fetch";
+  const labels = kind === "search" ? normalizeLabels(args.query, args.queries) : normalizeLabels(args.url, args.urls);
+  const summary = labels.length > 1 ? `${labels.length} ${kind === "search" ? "queries" : "URLs"}` : (labels[0] ?? "…");
+  return `${theme.fg("toolTitle", theme.bold(title))} ${theme.fg("accent", truncateMiddle(summary, 96))}`;
+}
+
+function renderWebResult(kind: ProgressKind, result: any, { expanded, isPartial }: any, theme: any, context: any) {
+  const progress = result.details?.progress as WebProgress | undefined;
+  if (isPartial && progress) {
+    startSpinner(context);
+    return new Text(renderProgress(progress, theme, spinnerFrame(context), expanded), 0, 0);
+  }
+  stopSpinner(context);
+
+  const details = result.details as any;
+  if (kind === "search" && details?.queries) {
+    const total = details.queries.reduce((sum: number, q: any) => sum + (q.resultCount ?? q.results?.length ?? 0), 0);
+    let text = `${theme.fg("success", "✅ Web search complete")} ${theme.fg("muted", `${details.queries.length}/${details.queries.length}`)}\n   results: ${total} total`;
+    if (expanded) for (const q of details.queries) text += `\n   ✓ ${theme.fg("accent", quote(q.query))} ${theme.fg("muted", `${q.resultCount ?? q.results?.length ?? 0} results`)}`;
+    return new Text(text, 0, 0);
+  }
+  if (kind === "fetch" && details?.results) {
+    const ok = details.results.filter((r: any) => !r.error).length;
+    const failed = details.results.length - ok;
+    let text = failed > 0 ? `${theme.fg("warning", "⚠️ Fetch complete")} ${ok}/${details.results.length} succeeded` : `${theme.fg("success", details.results.length === 1 ? "✅ Page fetched" : "✅ Pages fetched")} ${theme.fg("muted", `${ok}/${details.results.length}`)}`;
+    if (expanded) for (const r of details.results) text += `\n   ${r.error ? theme.fg("error", "✕") : theme.fg("success", "✓")} ${theme.fg("accent", truncateMiddle(r.url, 100))}${r.error ? theme.fg("error", ` ${r.error}`) : theme.fg("muted", ` ${r.range?.returned ?? 0}/${r.range?.total ?? 0} chars${r.cached ? " cached" : ""}`)}`;
+    return new Text(text, 0, 0);
+  }
+  const content = result.content?.find?.((c: any) => c.type === "text")?.text ?? "";
+  return new Text(content, 0, 0);
+}
+
+function renderProgress(progress: WebProgress, theme: any, spinner: string, expanded: boolean): string {
+  const isSearch = progress.kind === "search";
+  const verb = isSearch ? "Searching web" : progress.total === 1 ? "Fetching page" : "Fetching pages";
+  let text = `${isSearch ? "🔎" : "🌐"} ${verb}${progress.total > 1 ? `  ${progressBar(progress.completed, progress.total)} ${progress.completed}/${progress.total}` : "…"}`;
+  const visible = expanded ? progress.items : progress.items.slice(0, 6);
+  for (const item of visible) {
+    const icon = item.status === "done" ? theme.fg("success", "✓") : item.status === "error" ? theme.fg("error", "✕") : item.status === "current" ? theme.fg("warning", spinner) : theme.fg("muted", "·");
+    const note = item.error ? theme.fg("error", ` ${item.error}`) : item.note ? theme.fg("muted", ` ${item.note}`) : "";
+    text += `\n   ${icon} ${theme.fg(item.status === "pending" ? "muted" : "accent", quote(truncateMiddle(item.label, 100)))}${note}`;
+  }
+  if (!expanded && progress.items.length > visible.length) text += `\n   ${theme.fg("muted", `… +${progress.items.length - visible.length} more`)}`;
+  return text;
+}
+
+function startSpinner(context: any) {
+  if (context.state?.spinnerTimer) return;
+  context.state.spinnerIndex = context.state.spinnerIndex ?? 0;
+  context.state.spinnerTimer = setInterval(() => {
+    context.state.spinnerIndex = ((context.state.spinnerIndex ?? 0) + 1) % SPINNER.length;
+    context.invalidate?.();
+  }, 140);
+}
+
+function stopSpinner(context: any) {
+  if (!context.state?.spinnerTimer) return;
+  clearInterval(context.state.spinnerTimer);
+  context.state.spinnerTimer = undefined;
+}
+
+const SPINNER = ["◐", "◓", "◑", "◒"];
+
+function spinnerFrame(context: any): string {
+  return SPINNER[context.state?.spinnerIndex ?? 0] ?? SPINNER[0];
+}
+
+function progressBar(completed: number, total: number): string {
+  const width = 16;
+  const filled = total <= 0 ? 0 : Math.round((completed / total) * width);
+  return `[${"█".repeat(filled)}${"░".repeat(width - filled)}]`;
+}
+
+function normalizeLabels(single: unknown, multiple: unknown): string[] {
+  const raw = Array.isArray(multiple) ? multiple : single ? [single] : [];
+  return raw.map((v) => String(v).trim()).filter(Boolean);
+}
+
+function quote(value: string): string {
+  return `"${value}"`;
+}
+
+function truncateMiddle(value: string, max: number): string {
+  if (value.length <= max) return value;
+  const head = Math.ceil((max - 1) / 2);
+  const tail = Math.floor((max - 1) / 2);
+  return `${value.slice(0, head)}…${value.slice(value.length - tail)}`;
+}

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "pi-web-kit",
+  "version": "0.1.0",
+  "description": "Context-efficient web search and fetch tools for Pi.",
+  "type": "module",
+  "license": "MIT",
+  "author": "jvm",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jvm/pi-web-kit.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jvm/pi-web-kit/issues"
+  },
+  "homepage": "https://github.com/jvm/pi-web-kit#readme",
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "pi",
+    "web-search",
+    "web-fetch",
+    "exa",
+    "firecrawl"
+  ],
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "pi": {
+    "extensions": [
+      "./extensions/pi-web-kit"
+    ]
+  },
+  "files": [
+    "extensions",
+    "src",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md",
+    "SECURITY.md",
+    "CONTRIBUTING.md",
+    "CODE_OF_CONDUCT.md"
+  ],
+  "scripts": {
+    "check": "tsc --noEmit",
+    "typecheck": "tsc --noEmit",
+    "test": "node --import tsx --test tests/*.test.mjs",
+    "pack:dry-run": "npm pack --dry-run"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.2",
+    "tsx": "^4.21.0",
+    "typescript": "^6.0.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20.6.0"
+  }
+}

package/src/cache.ts

@@ -0,0 +1,76 @@
+import { FETCH_CACHE_MAX_BYTES, FETCH_CACHE_MAX_ENTRIES, FETCH_CACHE_TTL_MS } from "./limits.js";
+
+export type CachedPage = {
+  provider: string;
+  cacheKey: string;
+  requestedUrl: string;
+  url: string;
+  title?: string;
+  content: string;
+  format: string;
+  metadata?: Record<string, unknown>;
+  fetchedAt: number;
+  bytes: number;
+};
+
+type Entry = { page: CachedPage; lastAccessed: number };
+
+export class FetchCache {
+  private entries = new Map<string, Entry>();
+  private totalBytes = 0;
+
+  constructor(private opts = { maxEntries: FETCH_CACHE_MAX_ENTRIES, maxBytes: FETCH_CACHE_MAX_BYTES, ttlMs: FETCH_CACHE_TTL_MS }) {}
+
+  get totalCachedBytes() { return this.totalBytes; }
+  get size() { this.evictExpired(); return this.entries.size; }
+
+  get(key: string, now = Date.now()): CachedPage | undefined {
+    const entry = this.entries.get(key);
+    if (!entry) return undefined;
+    if (now - entry.page.fetchedAt > this.opts.ttlMs) {
+      this.delete(key);
+      return undefined;
+    }
+    entry.lastAccessed = now;
+    this.entries.delete(key);
+    this.entries.set(key, entry);
+    return entry.page;
+  }
+
+  set(key: string, page: Omit<CachedPage, "bytes"> & { bytes?: number }, now = Date.now()): CachedPage {
+    const bytes = page.bytes ?? byteLength(page.content);
+    const stored = { ...page, bytes };
+    this.delete(key);
+    this.entries.set(key, { page: stored, lastAccessed: now });
+    this.totalBytes += bytes;
+    this.evict(now);
+    return stored;
+  }
+
+  clear() { this.entries.clear(); this.totalBytes = 0; }
+
+  private delete(key: string) {
+    const old = this.entries.get(key);
+    if (old) this.totalBytes -= old.page.bytes;
+    this.entries.delete(key);
+  }
+
+  private evict(now = Date.now()) {
+    this.evictExpired(now);
+    while (this.entries.size > this.opts.maxEntries || this.totalBytes > this.opts.maxBytes) {
+      const oldest = this.entries.keys().next().value;
+      if (!oldest) break;
+      this.delete(oldest);
+    }
+  }
+
+  private evictExpired(now = Date.now()) {
+    for (const [key, entry] of this.entries) if (now - entry.page.fetchedAt > this.opts.ttlMs) this.delete(key);
+  }
+}
+
+export function byteLength(text: string): number {
+  return Buffer.byteLength(text, "utf8");
+}
+
+export const fetchCache = new FetchCache();

package/src/config.ts

@@ -0,0 +1,71 @@
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { FetchProviderName, SearchProviderName, WebKitConfig } from "./types.js";
+
+const SEARCH = ["exa_mcp", "exa", "tinyfish", "brave", "firecrawl"] as const;
+const FETCH = ["exa_mcp", "exa", "tinyfish", "markdown_new", "firecrawl"] as const;
+
+type PartialConfig = Partial<Omit<WebKitConfig, "apiKeys" | "markdownNew">> & {
+  apiKeys?: Partial<WebKitConfig["apiKeys"]>;
+  markdownNew?: Partial<WebKitConfig["markdownNew"]>;
+};
+
+export function resolveConfig(flags: { providerSearch?: unknown; providerFetch?: unknown } = {}, cwd = process.cwd(), env = process.env): WebKitConfig {
+  let cfg: WebKitConfig = {
+    provider_search: "exa_mcp",
+    provider_fetch: "exa_mcp",
+    apiKeys: {},
+    markdownNew: { method: "auto", retainImages: false },
+  };
+  cfg = merge(cfg, {
+    provider_search: env.PI_WEB_KIT_PROVIDER_SEARCH as SearchProviderName | undefined,
+    provider_fetch: env.PI_WEB_KIT_PROVIDER_FETCH as FetchProviderName | undefined,
+    apiKeys: {
+      exa: env.EXA_API_KEY,
+      tinyfish: env.TINYFISH_API_KEY,
+      brave: env.BRAVE_SEARCH_API_KEY,
+      firecrawl: env.FIRECRAWL_API_KEY,
+    },
+  });
+  const home = env.HOME ?? homedir();
+  for (const path of [join(home, ".pi/agent/pi-web-kit.json"), join(cwd, ".pi-web-kit.json")]) {
+    if (existsSync(path)) cfg = merge(cfg, JSON.parse(readFileSync(path, "utf8")) as PartialConfig);
+  }
+  cfg = merge(cfg, {
+    provider_search: flags.providerSearch as SearchProviderName | undefined,
+    provider_fetch: flags.providerFetch as FetchProviderName | undefined,
+  });
+  validateSearchProvider(cfg.provider_search);
+  validateFetchProvider(cfg.provider_fetch);
+  return cfg;
+}
+
+function merge(base: WebKitConfig, patch: PartialConfig): WebKitConfig {
+  return {
+    provider_search: patch.provider_search ?? base.provider_search,
+    provider_fetch: patch.provider_fetch ?? base.provider_fetch,
+    apiKeys: clean({ ...base.apiKeys, ...patch.apiKeys }),
+    markdownNew: { ...base.markdownNew, ...patch.markdownNew },
+  };
+}
+
+function clean<T extends Record<string, unknown>>(obj: T): T {
+  for (const k of Object.keys(obj)) if (obj[k] == null || obj[k] === "") delete obj[k];
+  return obj;
+}
+
+export function validateSearchProvider(name: string): asserts name is SearchProviderName {
+  if (!SEARCH.includes(name as SearchProviderName)) throw new Error(`Unknown search provider '${name}'. Expected one of: ${SEARCH.join(", ")}.`);
+}
+
+export function validateFetchProvider(name: string): asserts name is FetchProviderName {
+  if (!FETCH.includes(name as FetchProviderName)) throw new Error(`Unknown fetch provider '${name}'. Expected one of: ${FETCH.join(", ")}.`);
+}
+
+export function requireKey(config: WebKitConfig, provider: "exa" | "tinyfish" | "brave" | "firecrawl"): string {
+  const key = config.apiKeys[provider];
+  const envName = provider === "exa" ? "EXA_API_KEY" : provider === "tinyfish" ? "TINYFISH_API_KEY" : provider === "brave" ? "BRAVE_SEARCH_API_KEY" : "FIRECRAWL_API_KEY";
+  if (!key) throw new Error(`${provider} provider requires ${envName} or apiKeys.${provider} in .pi-web-kit.json / ~/.pi/agent/pi-web-kit.json.`);
+  return key;
+}