pi-web-kit 0.1.1 → 0.1.3

package/CHANGELOG.md

@@ -6,6 +6,27 @@ This project follows the spirit of [Keep a Changelog](https://keepachangelog.com
 
 ## [Unreleased]
 
+## [0.1.3] - 2026-05-10
+
+### Added
+
+- Install/update telemetry ping to `mocito.dev`, gated by Pi telemetry/offline settings and deduplicated per package version.
+
+## [0.1.2] - 2026-05-09
+
+### Changed
+
+- Deduplicated `mapFetchResults` between extension and fallback provider; extension now imports the shared implementation.
+- Replaced `existsSync` + `readFileSync` with a single `readFileSync` in a try/catch to eliminate TOCTOU race in config loading.
+- Replaced nested ternary for `requireKey` env-var names with a lookup object.
+- Replaced magic number `10` with `DEFAULT_NUM_RESULTS` in Exa, Exa MCP, and Firecrawl providers; replaced `10` with `MAX_URL_COUNT` in TinyFish.
+- Fixed `urlsMatch` to compute `canonicalWebUrl` once per URL instead of twice.
+- Moved `nextId` JSON-RPC counter from module scope to `ExaMcpProvider` instance to avoid ID conflicts across concurrent instances.
+- Removed unused `_query` parameter from internal `normalizeSearch` in Exa MCP provider.
+- Replaced O(n) completed-count recomputation in progress tracking with O(1) increment.
+- Cached `buildCacheKey` results in `fetchWithCache` to avoid redundant URL parsing per fetched page.
+- Replaced nested ternary chains in `providerScope` and `renderProgress` with lookup objects.
+
 ## [0.1.1] - 2026-05-09
 
 ### Fixed

package/README.md

@@ -98,6 +98,8 @@ Resolution order: defaults < environment variables < global config < project con
 ### Environment variables
 
 ```bash
+PI_OFFLINE=1        # disables install/update telemetry
+PI_TELEMETRY=0      # disables install/update telemetry
 PI_WEB_KIT_PROVIDER_SEARCH=exa_mcp|exa|tinyfish|brave|firecrawl
 PI_WEB_KIT_PROVIDER_FETCH=exa_mcp|exa|tinyfish|markdown_new|firecrawl
 EXA_API_KEY=...

package/SECURITY.md

@@ -23,4 +23,6 @@ The maintainer will acknowledge reports as soon as practical and coordinate disc
 
 `pi-web-kit` sends search queries and fetched URLs to the configured third-party provider. Page content returned by providers is cached in memory for the lifetime of the Pi process, subject to TTL and size limits. API keys are read from environment variables or local config files and are used only for provider requests. Do not commit API keys, tokens, or config files containing secrets.
 
+On startup, the extension sends a best-effort install/update telemetry ping to `mocito.dev` once per package version unless Pi telemetry is disabled or offline mode is enabled. The ping includes only the package name, version, and parsed platform/runtime/architecture from its User-Agent; it does not include prompts, queries, fetched URLs, file paths, config values, or API keys.
+
 The extension validates URLs before fetch calls and only accepts `http:` and `https:` URLs without embedded credentials. This validation reduces accidental misuse but does not sandbox provider responses or the local Pi process.

package/extensions/index.ts

@@ -1,4 +1,7 @@
-import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { readFileSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { type ExtensionAPI, getAgentDir } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { Type } from "typebox";
 import { fetchCache, type CachedPage } from "../src/cache.js";
@@ -6,10 +9,18 @@ import { resolveConfig } from "../src/config.js";
 import { DEFAULT_FETCH_LIMIT, DEFAULT_NUM_RESULTS, MAX_LIMIT, MAX_NUM_RESULTS, MAX_OFFSET, MAX_QUERY_COUNT, MAX_URL_COUNT, MULTI_FETCH_LIMIT } from "../src/limits.js";
 import { truncateText } from "../src/http.js";
 import { createFetchProvider, createSearchProvider } from "../src/providers/index.js";
+import { mapFetchResults } from "../src/providers/fallback.js";
 import type { FetchProviderName, SearchProviderName, WebFetchResult } from "../src/types.js";
-import { canonicalWebUrl, normalizeUrlInput, urlsMatch } from "../src/urls.js";
+import { canonicalWebUrl, normalizeUrlInput } from "../src/urls.js";
+
+const PACKAGE_NAME = "pi-web-kit";
+const INSTALL_TELEMETRY_URL = "https://mocito.dev/api/report-install";
+const INSTALL_TELEMETRY_TIMEOUT_MS = 5000;
+
+type InstallTelemetryState = { lastReportedVersion?: string };
 
 export default function (pi: ExtensionAPI) {
+  void reportInstallTelemetry();
   pi.registerFlag("web-provider-search", {
     description: "Temporary pi-web-kit search provider override (exa_mcp, exa, tinyfish, brave, firecrawl)",
     type: "string",
@@ -157,6 +168,64 @@ function buildFetchDescription(provider: FetchProviderName): string {
   return `Fetch URL content with startup provider '${provider}'. Results are cached by URL/options; use offset/limit to read long pages in chunks. Restart/reload pi after provider changes.`;
 }
 
+function readJsonFile(path: string): unknown {
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch {
+    return {};
+  }
+}
+
+function isTruthyEnvFlag(value: string | undefined): boolean {
+  if (!value) return false;
+  return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes";
+}
+
+function isInstallTelemetryEnabled(): boolean {
+  if (isTruthyEnvFlag(process.env.PI_OFFLINE)) return false;
+  if (process.env.PI_TELEMETRY !== undefined) return isTruthyEnvFlag(process.env.PI_TELEMETRY);
+  const settings = readJsonFile(join(getAgentDir(), "settings.json")) as { enableInstallTelemetry?: unknown };
+  return settings.enableInstallTelemetry !== false;
+}
+
+function getPackageVersion(): string {
+  try {
+    const packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { version?: unknown };
+    return typeof packageJson.version === "string" && packageJson.version.length > 0 ? packageJson.version : "0.0.0";
+  } catch {
+    return "0.0.0";
+  }
+}
+
+function getInstallTelemetryUserAgent(version: string): string {
+  const runtimeVersions = process.versions as NodeJS.ProcessVersions & { bun?: string };
+  const runtime = runtimeVersions.bun ? `bun/${runtimeVersions.bun}` : `node/${process.version}`;
+  return `${PACKAGE_NAME}/${version} (${process.platform}; ${runtime}; ${process.arch})`;
+}
+
+export async function reportInstallTelemetry(): Promise<void> {
+  try {
+    if (!isInstallTelemetryEnabled()) return;
+
+    const version = getPackageVersion();
+    const telemetryDir = join(getAgentDir(), "extensions");
+    const statePath = join(telemetryDir, "pi-web-kit-install.json");
+    const state = readJsonFile(statePath) as InstallTelemetryState;
+    if (state.lastReportedVersion === version) return;
+
+    await mkdir(telemetryDir, { recursive: true });
+    await writeFile(statePath, `${JSON.stringify({ lastReportedVersion: version }, null, 2)}\n`);
+
+    const params = new URLSearchParams({ tool: PACKAGE_NAME, version });
+    await fetch(`${INSTALL_TELEMETRY_URL}?${params.toString()}`, {
+      headers: { "User-Agent": getInstallTelemetryUserAgent(version) },
+      signal: AbortSignal.timeout(INSTALL_TELEMETRY_TIMEOUT_MS),
+    });
+  } catch {
+    // Best-effort install telemetry: ignore settings, filesystem, and network failures.
+  }
+}
+
 function normalizeQueries(params: Record<string, any>): string[] {
   const raw = Array.isArray(params.queries) ? params.queries : params.query ? [params.query] : [];
   const queries = [...new Set(raw.map((q) => String(q).trim()).filter(Boolean))];
@@ -176,9 +245,11 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
   const refresh = params.refresh === true;
 
   const pages = new Map<string, { page?: CachedPage; cached: boolean; refreshed: boolean; error?: string }>();
+  const cacheKeys = new Map<string, string>();
   const missing: string[] = [];
   for (const url of urls) {
     const cacheKey = buildCacheKey(providerName, url, params, config);
+    cacheKeys.set(url, cacheKey);
     const cached = fetchCache.get(cacheKey);
     if (cached && !refresh) {
       pages.set(url, { page: cached, cached: true, refreshed: false });
@@ -200,7 +271,7 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
         onProgress?.({ status: "error", url: requestedUrl, error });
         continue;
       }
-      const cacheKey = buildCacheKey(providerName, requestedUrl, params, config);
+      const cacheKey = cacheKeys.get(requestedUrl)!;
       const page = fetchCache.set(cacheKey, {
         provider: providerName,
         cacheKey,
@@ -224,16 +295,6 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
   }) };
 }
 
-function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, WebFetchResult["results"][number]> {
-  const out = new Map<string, WebFetchResult["results"][number]>();
-  const remaining = [...(fetched.results ?? [])];
-  for (const url of requested) {
-    const index = remaining.findIndex((item) => urlsMatch(item.url, url));
-    if (index >= 0) out.set(url, remaining.splice(index, 1)[0]);
-  }
-  requested.forEach((url, index) => { if (!out.has(url) && fetched.results?.[index]) out.set(url, fetched.results[index]); });
-  return out;
-}
 
 export function pageSlice(page: CachedPage, offset: number, limit: number, cached: boolean, refreshed: boolean) {
   const total = page.content.length;
@@ -266,7 +327,13 @@ function fetchConfigDefaults(provider: FetchProviderName, config?: any): Record<
 }
 
 function providerScope(provider: FetchProviderName, config?: any): string {
-  const key = provider === "exa" || provider === "exa_mcp" ? config?.apiKeys?.exa : provider === "tinyfish" ? config?.apiKeys?.tinyfish : provider === "firecrawl" ? config?.apiKeys?.firecrawl : undefined;
+  const keyMap: Partial<Record<FetchProviderName, string | undefined>> = {
+    exa: config?.apiKeys?.exa,
+    exa_mcp: config?.apiKeys?.exa,
+    tinyfish: config?.apiKeys?.tinyfish,
+    firecrawl: config?.apiKeys?.firecrawl,
+  };
+  const key = keyMap[provider];
   return key ? `key:${String(key).slice(0, 8)}` : "default";
 }
 
@@ -294,7 +361,7 @@ function markProgressDone(progress: WebProgress, label: string, note?: string) {
   if (!item) return;
   item.status = "done";
   item.note = note;
-  progress.completed = progress.items.filter((i) => i.status === "done" || i.status === "error").length;
+  progress.completed++;
 }
 
 function markProgressError(progress: WebProgress, label: string, error?: string) {
@@ -302,7 +369,7 @@ function markProgressError(progress: WebProgress, label: string, error?: string)
   if (!item) return;
   item.status = "error";
   item.error = error;
-  progress.completed = progress.items.filter((i) => i.status === "done" || i.status === "error").length;
+  progress.completed++;
 }
 
 function emitProgress(onUpdate: ((patch: any) => void) | undefined, progress: WebProgress) {
@@ -355,8 +422,14 @@ function renderProgress(progress: WebProgress, theme: any, spinner: string, expa
   const verb = isSearch ? "Searching web" : progress.total === 1 ? "Fetching page" : "Fetching pages";
   let text = `${isSearch ? "🔎" : "🌐"} ${verb}${progress.total > 1 ? `  ${progressBar(progress.completed, progress.total)} ${progress.completed}/${progress.total}` : "…"}`;
   const visible = expanded ? progress.items : progress.items.slice(0, 6);
+  const iconMap: Record<string, string> = {
+    done: theme.fg("success", "✓"),
+    error: theme.fg("error", "✕"),
+    current: theme.fg("warning", spinner),
+    pending: theme.fg("muted", "·"),
+  };
   for (const item of visible) {
-    const icon = item.status === "done" ? theme.fg("success", "✓") : item.status === "error" ? theme.fg("error", "✕") : item.status === "current" ? theme.fg("warning", spinner) : theme.fg("muted", "·");
+    const icon = iconMap[item.status] ?? theme.fg("muted", "·");
     const note = item.error ? theme.fg("error", ` ${item.error}`) : item.note ? theme.fg("muted", ` ${item.note}`) : "";
     text += `\n   ${icon} ${theme.fg(item.status === "pending" ? "muted" : "accent", quote(truncateMiddle(item.label, 100)))}${note}`;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-kit",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Context-efficient web search and fetch tools for Pi.",
   "type": "module",
   "license": "MIT",

package/src/config.ts

@@ -1,4 +1,4 @@
-import { existsSync, readFileSync } from "node:fs";
+import { readFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import type { FetchProviderName, SearchProviderName, WebKitConfig } from "./types.js";
@@ -30,7 +30,7 @@ export function resolveConfig(flags: { providerSearch?: unknown; providerFetch?:
   });
   const home = env.HOME ?? homedir();
   for (const path of [join(home, ".pi/agent/pi-web-kit.json"), join(cwd, ".pi-web-kit.json")]) {
-    if (existsSync(path)) cfg = merge(cfg, JSON.parse(readFileSync(path, "utf8")) as PartialConfig);
+    try { cfg = merge(cfg, JSON.parse(readFileSync(path, "utf8")) as PartialConfig); } catch (e: any) { if (e?.code !== "ENOENT") throw e; }
   }
   cfg = merge(cfg, {
     provider_search: flags.providerSearch as SearchProviderName | undefined,
@@ -63,9 +63,11 @@ export function validateFetchProvider(name: string): asserts name is FetchProvid
   if (!FETCH.includes(name as FetchProviderName)) throw new Error(`Unknown fetch provider '${name}'. Expected one of: ${FETCH.join(", ")}.`);
 }
 
+const PROVIDER_ENV_NAMES = { exa: "EXA_API_KEY", tinyfish: "TINYFISH_API_KEY", brave: "BRAVE_SEARCH_API_KEY", firecrawl: "FIRECRAWL_API_KEY" } as const;
+
 export function requireKey(config: WebKitConfig, provider: "exa" | "tinyfish" | "brave" | "firecrawl"): string {
   const key = config.apiKeys[provider];
-  const envName = provider === "exa" ? "EXA_API_KEY" : provider === "tinyfish" ? "TINYFISH_API_KEY" : provider === "brave" ? "BRAVE_SEARCH_API_KEY" : "FIRECRAWL_API_KEY";
+  const envName = PROVIDER_ENV_NAMES[provider];
   if (!key) throw new Error(`${provider} provider requires ${envName} or apiKeys.${provider} in .pi-web-kit.json / ~/.pi/agent/pi-web-kit.json.`);
   return key;
 }

package/src/providers/exa-mcp.ts

@@ -1,17 +1,17 @@
 import { asSnippet, fetchWithTimeout, normalizeUrls } from "../http.js";
+import { DEFAULT_NUM_RESULTS } from "../limits.js";
 import { urlsMatch } from "../urls.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { applyExaFetchFallbacks } from "./fallback.js";
 
-let nextId = 1;
-
 export class ExaMcpProvider implements SearchProvider, FetchProvider {
   private sessionId?: string;
+  private nextId = 1;
   constructor(private config: WebKitConfig) {}
 
   async search(input: SearchInput, signal?: AbortSignal) {
-    const result = await this.callTool("web_search_exa", { query: input.query, numResults: input.numResults ?? 10 }, signal);
-    return { provider: "exa_mcp" as const, query: input.query, results: normalizeSearch(result, input.query) };
+    const result = await this.callTool("web_search_exa", { query: input.query, numResults: input.numResults ?? DEFAULT_NUM_RESULTS }, signal);
+    return { provider: "exa_mcp" as const, query: input.query, results: normalizeSearch(result) };
   }
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
@@ -44,7 +44,7 @@ export class ExaMcpProvider implements SearchProvider, FetchProvider {
     const headers: Record<string, string> = { "content-type": "application/json", accept: "application/json, text/event-stream" };
     if (this.sessionId) headers["mcp-session-id"] = this.sessionId;
     if (this.config.apiKeys.exa) headers["x-api-key"] = this.config.apiKeys.exa;
-    const body = notification ? { jsonrpc: "2.0", method, params } : { jsonrpc: "2.0", id: nextId++, method, params };
+    const body = notification ? { jsonrpc: "2.0", method, params } : { jsonrpc: "2.0", id: this.nextId++, method, params };
     const res = await fetchWithTimeout("https://mcp.exa.ai/mcp", { method: "POST", headers, body: JSON.stringify(body), signal, timeoutMs: 45_000 });
     const text = await res.text();
     if (!res.ok) throw new Error(`Exa MCP ${method} failed: ${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
@@ -70,7 +70,7 @@ function textFromContent(result: any): string {
   return typeof content === "string" ? content : JSON.stringify(content);
 }
 
-function normalizeSearch(result: any, _query: string) {
+function normalizeSearch(result: any) {
   const structured = result?.structuredContent ?? result?.result ?? result;
   const list = structured.results ?? structured.data ?? structured.items;
   if (Array.isArray(list)) return list.map((r: any, i: number) => ({ title: r.title, url: r.url, snippet: asSnippet(r.snippet ?? r.text ?? r.summary ?? r.highlights), siteName: r.siteName, position: i + 1 })).filter((r: any) => r.url);

package/src/providers/exa.ts

@@ -1,4 +1,5 @@
 import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { DEFAULT_NUM_RESULTS } from "../limits.js";
 import { urlsMatch } from "../urls.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
@@ -11,7 +12,7 @@ export class ExaProvider implements SearchProvider, FetchProvider {
   async search(input: SearchInput, signal?: AbortSignal) {
     const body = {
       query: input.query,
-      numResults: input.numResults ?? 10,
+      numResults: input.numResults ?? DEFAULT_NUM_RESULTS,
       contents: input.contents ?? { highlights: true },
       includeDomains: input.includeDomains,
       excludeDomains: input.excludeDomains,

package/src/providers/fallback.ts

@@ -62,13 +62,14 @@ function fallbackProviders(config: WebKitConfig): Array<{ name: FetchProviderNam
   return providers;
 }
 
-function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, FetchPage> {
+export function mapFetchResults(requested: string[], fetched: WebFetchResult): Map<string, FetchPage> {
   const out = new Map<string, FetchPage>();
   const remaining = [...(fetched.results ?? [])];
   for (const url of requested) {
     const index = remaining.findIndex((item) => urlsMatch(item.url, url));
     if (index >= 0) out.set(url, remaining.splice(index, 1)[0]);
   }
+  // providers may not echo back the canonical URL; fall back to positional match
   requested.forEach((url, index) => { if (!out.has(url) && fetched.results?.[index]) out.set(url, fetched.results[index]); });
   return out;
 }

package/src/providers/firecrawl.ts

@@ -1,5 +1,5 @@
 import { asSnippet, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
-import { FETCH_CONCURRENCY } from "../limits.js";
+import { DEFAULT_NUM_RESULTS, FETCH_CONCURRENCY } from "../limits.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
 
@@ -13,7 +13,7 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
       method: "POST", headers: this.headers(), signal, timeoutMs: 45000,
       body: JSON.stringify({
         query: input.query,
-        limit: input.numResults ?? 10,
+        limit: input.numResults ?? DEFAULT_NUM_RESULTS,
         location: input.location,
         country: input.country,
         includeDomains: input.includeDomains,

package/src/providers/tinyfish.ts

@@ -1,4 +1,5 @@
 import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { MAX_URL_COUNT } from "../limits.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
 
@@ -20,7 +21,7 @@ export class TinyFishProvider implements SearchProvider, FetchProvider {
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
     const urls = normalizeUrls(input);
-    if (urls.length > 10) throw new Error("TinyFish fetch supports a maximum of 10 URLs per request.");
+    if (urls.length > MAX_URL_COUNT) throw new Error(`TinyFish fetch supports a maximum of ${MAX_URL_COUNT} URLs per request.`);
     const data = await requestJson<any>("https://api.fetch.tinyfish.ai", {
       method: "POST",
       headers: { "content-type": "application/json", "X-API-Key": this.key },

package/src/urls.ts

@@ -31,9 +31,11 @@ export function normalizeUrlInput(input: { url?: string; urls?: string[] }, maxC
 export function urlsMatch(a: string | undefined, b: string | undefined): boolean {
   if (!a || !b) return false;
   try {
-    if (canonicalWebUrl(a) === canonicalWebUrl(b)) return true;
+    const ca = canonicalWebUrl(a);
+    const cb = canonicalWebUrl(b);
+    if (ca === cb) return true;
     const trimSlash = (s: string) => s.endsWith("/") ? s.slice(0, -1) : s;
-    return trimSlash(canonicalWebUrl(a)) === trimSlash(canonicalWebUrl(b));
+    return trimSlash(ca) === trimSlash(cb);
   } catch {
     return a === b;
   }