pi-web-kit 0.1.1 → 0.1.2

package/CHANGELOG.md

@@ -6,6 +6,21 @@ This project follows the spirit of [Keep a Changelog](https://keepachangelog.com
 
 ## [Unreleased]
 
+## [0.1.2] - 2026-05-09
+
+### Changed
+
+- Deduplicated `mapFetchResults` between extension and fallback provider; extension now imports the shared implementation.
+- Replaced `existsSync` + `readFileSync` with a single `readFileSync` in a try/catch to eliminate TOCTOU race in config loading.
+- Replaced nested ternary for `requireKey` env-var names with a lookup object.
+- Replaced magic number `10` with `DEFAULT_NUM_RESULTS` in Exa, Exa MCP, and Firecrawl providers; replaced `10` with `MAX_URL_COUNT` in TinyFish.
+- Fixed `urlsMatch` to compute `canonicalWebUrl` once per URL instead of twice.
+- Moved `nextId` JSON-RPC counter from module scope to `ExaMcpProvider` instance to avoid ID conflicts across concurrent instances.
+- Removed unused `_query` parameter from internal `normalizeSearch` in Exa MCP provider.
+- Replaced O(n) completed-count recomputation in progress tracking with O(1) increment.
+- Cached `buildCacheKey` results in `fetchWithCache` to avoid redundant URL parsing per fetched page.
+- Replaced nested ternary chains in `providerScope` and `renderProgress` with lookup objects.
+
 ## [0.1.1] - 2026-05-09
 
 ### Fixed

package/extensions/index.ts

@@ -6,8 +6,9 @@ import { resolveConfig } from "../src/config.js";
 import { DEFAULT_FETCH_LIMIT, DEFAULT_NUM_RESULTS, MAX_LIMIT, MAX_NUM_RESULTS, MAX_OFFSET, MAX_QUERY_COUNT, MAX_URL_COUNT, MULTI_FETCH_LIMIT } from "../src/limits.js";
 import { truncateText } from "../src/http.js";
 import { createFetchProvider, createSearchProvider } from "../src/providers/index.js";
+import { mapFetchResults } from "../src/providers/fallback.js";
 import type { FetchProviderName, SearchProviderName, WebFetchResult } from "../src/types.js";
-import { canonicalWebUrl, normalizeUrlInput, urlsMatch } from "../src/urls.js";
+import { canonicalWebUrl, normalizeUrlInput } from "../src/urls.js";
 
 export default function (pi: ExtensionAPI) {
   pi.registerFlag("web-provider-search", {
@@ -176,9 +177,11 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
   const refresh = params.refresh === true;
 
   const pages = new Map<string, { page?: CachedPage; cached: boolean; refreshed: boolean; error?: string }>();
+  const cacheKeys = new Map<string, string>();
   const missing: string[] = [];
   for (const url of urls) {
     const cacheKey = buildCacheKey(providerName, url, params, config);
+    cacheKeys.set(url, cacheKey);
     const cached = fetchCache.get(cacheKey);
     if (cached && !refresh) {
       pages.set(url, { page: cached, cached: true, refreshed: false });
@@ -200,7 +203,7 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
         onProgress?.({ status: "error", url: requestedUrl, error });
         continue;
       }
-      const cacheKey = buildCacheKey(providerName, requestedUrl, params, config);
+      const cacheKey = cacheKeys.get(requestedUrl)!;
       const page = fetchCache.set(cacheKey, {
         provider: providerName,
         cacheKey,
@@ -224,16 +227,6 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
   }) };
 }
 
-function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, WebFetchResult["results"][number]> {
-  const out = new Map<string, WebFetchResult["results"][number]>();
-  const remaining = [...(fetched.results ?? [])];
-  for (const url of requested) {
-    const index = remaining.findIndex((item) => urlsMatch(item.url, url));
-    if (index >= 0) out.set(url, remaining.splice(index, 1)[0]);
-  }
-  requested.forEach((url, index) => { if (!out.has(url) && fetched.results?.[index]) out.set(url, fetched.results[index]); });
-  return out;
-}
 
 export function pageSlice(page: CachedPage, offset: number, limit: number, cached: boolean, refreshed: boolean) {
   const total = page.content.length;
@@ -266,7 +259,13 @@ function fetchConfigDefaults(provider: FetchProviderName, config?: any): Record<
 }
 
 function providerScope(provider: FetchProviderName, config?: any): string {
-  const key = provider === "exa" || provider === "exa_mcp" ? config?.apiKeys?.exa : provider === "tinyfish" ? config?.apiKeys?.tinyfish : provider === "firecrawl" ? config?.apiKeys?.firecrawl : undefined;
+  const keyMap: Partial<Record<FetchProviderName, string | undefined>> = {
+    exa: config?.apiKeys?.exa,
+    exa_mcp: config?.apiKeys?.exa,
+    tinyfish: config?.apiKeys?.tinyfish,
+    firecrawl: config?.apiKeys?.firecrawl,
+  };
+  const key = keyMap[provider];
   return key ? `key:${String(key).slice(0, 8)}` : "default";
 }
 
@@ -294,7 +293,7 @@ function markProgressDone(progress: WebProgress, label: string, note?: string) {
   if (!item) return;
   item.status = "done";
   item.note = note;
-  progress.completed = progress.items.filter((i) => i.status === "done" || i.status === "error").length;
+  progress.completed++;
 }
 
 function markProgressError(progress: WebProgress, label: string, error?: string) {
@@ -302,7 +301,7 @@ function markProgressError(progress: WebProgress, label: string, error?: string)
   if (!item) return;
   item.status = "error";
   item.error = error;
-  progress.completed = progress.items.filter((i) => i.status === "done" || i.status === "error").length;
+  progress.completed++;
 }
 
 function emitProgress(onUpdate: ((patch: any) => void) | undefined, progress: WebProgress) {
@@ -355,8 +354,14 @@ function renderProgress(progress: WebProgress, theme: any, spinner: string, expa
   const verb = isSearch ? "Searching web" : progress.total === 1 ? "Fetching page" : "Fetching pages";
   let text = `${isSearch ? "🔎" : "🌐"} ${verb}${progress.total > 1 ? `  ${progressBar(progress.completed, progress.total)} ${progress.completed}/${progress.total}` : "…"}`;
   const visible = expanded ? progress.items : progress.items.slice(0, 6);
+  const iconMap: Record<string, string> = {
+    done: theme.fg("success", "✓"),
+    error: theme.fg("error", "✕"),
+    current: theme.fg("warning", spinner),
+    pending: theme.fg("muted", "·"),
+  };
   for (const item of visible) {
-    const icon = item.status === "done" ? theme.fg("success", "✓") : item.status === "error" ? theme.fg("error", "✕") : item.status === "current" ? theme.fg("warning", spinner) : theme.fg("muted", "·");
+    const icon = iconMap[item.status] ?? theme.fg("muted", "·");
     const note = item.error ? theme.fg("error", ` ${item.error}`) : item.note ? theme.fg("muted", ` ${item.note}`) : "";
     text += `\n   ${icon} ${theme.fg(item.status === "pending" ? "muted" : "accent", quote(truncateMiddle(item.label, 100)))}${note}`;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-kit",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Context-efficient web search and fetch tools for Pi.",
   "type": "module",
   "license": "MIT",

package/src/config.ts

@@ -1,4 +1,4 @@
-import { existsSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import type { FetchProviderName, SearchProviderName, WebKitConfig } from "./types.js";
@@ -30,7 +30,7 @@ export function resolveConfig(flags: { providerSearch?: unknown; providerFetch?:
   });
   const home = env.HOME ?? homedir();
   for (const path of [join(home, ".pi/agent/pi-web-kit.json"), join(cwd, ".pi-web-kit.json")]) {
-    if (existsSync(path)) cfg = merge(cfg, JSON.parse(readFileSync(path, "utf8")) as PartialConfig);
+    try { cfg = merge(cfg, JSON.parse(readFileSync(path, "utf8")) as PartialConfig); } catch (e: any) { if (e?.code !== "ENOENT") throw e; }
   }
   cfg = merge(cfg, {
     provider_search: flags.providerSearch as SearchProviderName | undefined,
@@ -63,9 +63,11 @@ export function validateFetchProvider(name: string): asserts name is FetchProvid
   if (!FETCH.includes(name as FetchProviderName)) throw new Error(`Unknown fetch provider '${name}'. Expected one of: ${FETCH.join(", ")}.`);
 }
 
+const PROVIDER_ENV_NAMES = { exa: "EXA_API_KEY", tinyfish: "TINYFISH_API_KEY", brave: "BRAVE_SEARCH_API_KEY", firecrawl: "FIRECRAWL_API_KEY" } as const;
+
 export function requireKey(config: WebKitConfig, provider: "exa" | "tinyfish" | "brave" | "firecrawl"): string {
   const key = config.apiKeys[provider];
-  const envName = provider === "exa" ? "EXA_API_KEY" : provider === "tinyfish" ? "TINYFISH_API_KEY" : provider === "brave" ? "BRAVE_SEARCH_API_KEY" : "FIRECRAWL_API_KEY";
+  const envName = PROVIDER_ENV_NAMES[provider];
   if (!key) throw new Error(`${provider} provider requires ${envName} or apiKeys.${provider} in .pi-web-kit.json / ~/.pi/agent/pi-web-kit.json.`);
   return key;
 }

package/src/providers/exa-mcp.ts

@@ -1,17 +1,17 @@
 import { asSnippet, fetchWithTimeout, normalizeUrls } from "../http.js";
+import { DEFAULT_NUM_RESULTS } from "../limits.js";
 import { urlsMatch } from "../urls.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { applyExaFetchFallbacks } from "./fallback.js";
 
-let nextId = 1;
-
 export class ExaMcpProvider implements SearchProvider, FetchProvider {
   private sessionId?: string;
+  private nextId = 1;
   constructor(private config: WebKitConfig) {}
 
   async search(input: SearchInput, signal?: AbortSignal) {
-    const result = await this.callTool("web_search_exa", { query: input.query, numResults: input.numResults ?? 10 }, signal);
-    return { provider: "exa_mcp" as const, query: input.query, results: normalizeSearch(result, input.query) };
+    const result = await this.callTool("web_search_exa", { query: input.query, numResults: input.numResults ?? DEFAULT_NUM_RESULTS }, signal);
+    return { provider: "exa_mcp" as const, query: input.query, results: normalizeSearch(result) };
   }
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
@@ -44,7 +44,7 @@ export class ExaMcpProvider implements SearchProvider, FetchProvider {
     const headers: Record<string, string> = { "content-type": "application/json", accept: "application/json, text/event-stream" };
     if (this.sessionId) headers["mcp-session-id"] = this.sessionId;
     if (this.config.apiKeys.exa) headers["x-api-key"] = this.config.apiKeys.exa;
-    const body = notification ? { jsonrpc: "2.0", method, params } : { jsonrpc: "2.0", id: nextId++, method, params };
+    const body = notification ? { jsonrpc: "2.0", method, params } : { jsonrpc: "2.0", id: this.nextId++, method, params };
     const res = await fetchWithTimeout("https://mcp.exa.ai/mcp", { method: "POST", headers, body: JSON.stringify(body), signal, timeoutMs: 45_000 });
     const text = await res.text();
     if (!res.ok) throw new Error(`Exa MCP ${method} failed: ${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
@@ -70,7 +70,7 @@ function textFromContent(result: any): string {
   return typeof content === "string" ? content : JSON.stringify(content);
 }
 
-function normalizeSearch(result: any, _query: string) {
+function normalizeSearch(result: any) {
   const structured = result?.structuredContent ?? result?.result ?? result;
   const list = structured.results ?? structured.data ?? structured.items;
   if (Array.isArray(list)) return list.map((r: any, i: number) => ({ title: r.title, url: r.url, snippet: asSnippet(r.snippet ?? r.text ?? r.summary ?? r.highlights), siteName: r.siteName, position: i + 1 })).filter((r: any) => r.url);

package/src/providers/exa.ts

@@ -1,4 +1,5 @@
 import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { DEFAULT_NUM_RESULTS } from "../limits.js";
 import { urlsMatch } from "../urls.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
@@ -11,7 +12,7 @@ export class ExaProvider implements SearchProvider, FetchProvider {
   async search(input: SearchInput, signal?: AbortSignal) {
     const body = {
       query: input.query,
-      numResults: input.numResults ?? 10,
+      numResults: input.numResults ?? DEFAULT_NUM_RESULTS,
       contents: input.contents ?? { highlights: true },
       includeDomains: input.includeDomains,
       excludeDomains: input.excludeDomains,

package/src/providers/fallback.ts

@@ -62,13 +62,14 @@ function fallbackProviders(config: WebKitConfig): Array<{ name: FetchProviderNam
   return providers;
 }
 
-function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, FetchPage> {
+export function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, FetchPage> {
   const out = new Map<string, FetchPage>();
   const remaining = [...(fetched.results ?? [])];
   for (const url of requested) {
     const index = remaining.findIndex((item) => urlsMatch(item.url, url));
     if (index >= 0) out.set(url, remaining.splice(index, 1)[0]);
   }
+  // providers may not echo back the canonical URL; fall back to positional match
   requested.forEach((url, index) => { if (!out.has(url) && fetched.results?.[index]) out.set(url, fetched.results[index]); });
   return out;
 }

package/src/providers/firecrawl.ts

@@ -1,5 +1,5 @@
 import { asSnippet, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
-import { FETCH_CONCURRENCY } from "../limits.js";
+import { DEFAULT_NUM_RESULTS, FETCH_CONCURRENCY } from "../limits.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
 
@@ -13,7 +13,7 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
       method: "POST", headers: this.headers(), signal, timeoutMs: 45000,
       body: JSON.stringify({
         query: input.query,
-        limit: input.numResults ?? 10,
+        limit: input.numResults ?? DEFAULT_NUM_RESULTS,
         location: input.location,
         country: input.country,
         includeDomains: input.includeDomains,

package/src/providers/tinyfish.ts

@@ -1,4 +1,5 @@
 import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { MAX_URL_COUNT } from "../limits.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
 
@@ -20,7 +21,7 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
     const urls = normalizeUrls(input);
-    if (urls.length > 10) throw new Error("TinyFish fetch supports a maximum of 10 URLs per request.");
+    if (urls.length > MAX_URL_COUNT) throw new Error(`TinyFish fetch supports a maximum of ${MAX_URL_COUNT} URLs per request.`);
     const data = await requestJson<any>("https://api.fetch.tinyfish.ai", {
       method: "POST",
       headers: { "content-type": "application/json", "X-API-Key": this.key },

package/src/urls.ts

@@ -31,9 +31,11 @@ export function normalizeUrlInput(input: { url?: string; urls?: string[] }, maxC
 export function urlsMatch(a: string | undefined, b: string | undefined): boolean {
   if (!a || !b) return false;
   try {
-    if (canonicalWebUrl(a) === canonicalWebUrl(b)) return true;
+    const ca = canonicalWebUrl(a);
+    const cb = canonicalWebUrl(b);
+    if (ca === cb) return true;
     const trimSlash = (s: string) => s.endsWith("/") ? s.slice(0, -1) : s;
-    return trimSlash(canonicalWebUrl(a)) === trimSlash(canonicalWebUrl(b));
+    return trimSlash(ca) === trimSlash(cb);
   } catch {
     return a === b;
   }