pi-web-kit 0.2.4 → 0.4.0

package/extensions/index.ts

@@ -1,40 +1,120 @@
 import { createHash } from "node:crypto";
-import { type ExtensionAPI, getAgentDir } from "@earendil-works/pi-coding-agent";
+import { type ExtensionAPI, type ExtensionContext, type ToolDefinition } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
-import { Type } from "typebox";
+import { Type, type TSchema } from "typebox";
 import { fetchCache, type CachedPage } from "../src/cache.js";
 import { resolveConfig } from "../src/config.js";
-import { DEFAULT_FETCH_LIMIT, DEFAULT_NUM_RESULTS, MAX_LIMIT, MAX_NUM_RESULTS, MAX_OFFSET, MAX_QUERY_COUNT, MAX_URL_COUNT, MULTI_FETCH_LIMIT } from "../src/limits.js";
+import { applySearchContextBudget, capSearchResultLimit, DEFAULT_FETCH_LIMIT, DEFAULT_NUM_RESULTS, DEFAULT_SEARCH_CONTEXT_TOKENS, MAX_LIMIT, MAX_NUM_RESULTS, MAX_OFFSET, MAX_QUERY_COUNT, MAX_SEARCH_CONTEXT_TOKENS, MAX_URL_COUNT, MULTI_FETCH_LIMIT, safePrefix, TINYFISH_MAX_PAGE } from "../src/limits.js";
 import { createCodeSearchProvider, createContext7Provider, createFetchProvider, createSearchProvider } from "../src/providers/index.js";
 import { mapFetchResults } from "../src/providers/fallback.js";
 import type { FetchProviderName, SearchProviderName, WebFetchResult } from "../src/types.js";
 import { canonicalWebUrl, normalizeUrlInput } from "../src/urls.js";
 import { reportInstallTelemetry } from "../src/install-telemetry.js";
+import { projectOutput, publicPageError, redactOutput, redactText, WEB_OUTPUT_SCHEMAS, WEB_TOOL_METADATA } from "../src/output.js";
+
+const REGISTERED_TOOLS_ENTRY = "pi-web-kit:registered-tools";
 
 export default function (pi: ExtensionAPI) {
   void reportInstallTelemetry();
   pi.registerFlag("web-provider-search", {
-    description: "Temporary pi-web-kit search provider override (exa_mcp, exa, tinyfish, brave, firecrawl)",
+    description: "Temporary pi-web-kit search provider override (exa, tinyfish, brave, firecrawl)",
     type: "string",
   });
   pi.registerFlag("web-provider-fetch", {
-    description: "Temporary pi-web-kit fetch provider override (exa_mcp, exa, tinyfish, markdown_new, firecrawl)",
+    description: "Temporary pi-web-kit fetch provider override (exa, tinyfish, markdown_new, firecrawl)",
     type: "string",
   });
 
   let registeredConfig = "";
-  pi.on("session_start", (_event, ctx) => {
+  let rememberedTools: string[] | undefined;
+  pi.on("session_start", (event, ctx) => {
     const startupConfig = runtimeConfig(pi, ctx.cwd, projectIsTrusted(ctx));
     const signature = toolConfigSignature(startupConfig);
     if (signature === registeredConfig) return;
-    registerTools(pi, startupConfig);
-    syncOptionalTools(pi, startupConfig);
+    const available = [
+      "web_search", "web_fetch",
+      ...(startupConfig.apiKeys.context7 ? ["library_search", "library_docs"] : []),
+      ...(startupConfig.apiKeys.exa ? ["code_search"] : []),
+    ];
+    const previous = registrationHistory(ctx);
+    // Remember availability, not activation. Pi owns active/pending selection.
+    // Without history (an older session), conservatively preserve inactivity on
+    // the first reload instead of guessing which tools were manually disabled.
+    registerTools(pi, startupConfig, event.reason === "reload" ? new Set(previous ?? available) : undefined);
+    rememberedTools = recordRegisteredTools(pi, ctx, available);
+    // A session change can drop provider credentials without replacing this
+    // extension instance. Retained definitions must not remain active then.
+    if (typeof pi.getActiveTools === "function" && typeof pi.setActiveTools === "function") {
+      const unavailable = new Set([
+        ...(!startupConfig.apiKeys.context7 ? ["library_search", "library_docs"] : []),
+        ...(!startupConfig.apiKeys.exa ? ["code_search"] : []),
+      ]);
+      const active = pi.getActiveTools();
+      if (active.some(name => unavailable.has(name))) pi.setActiveTools(active.filter(name => !unavailable.has(name)));
+    }
     registeredConfig = signature;
   });
+  pi.on("session_shutdown", (event, ctx) => {
+    // Tree navigation can restore an older branch record while the live
+    // registry still knows newer tools. Save that fact before runtime teardown.
+    if (event.reason === "reload" && rememberedTools) recordRegisteredTools(pi, ctx, rememberedTools);
+  });
+}
+
+function registrationHistory(ctx: ExtensionContext): string[] | undefined {
+  const entry = ctx.sessionManager?.getBranch().filter(item => item.type === "custom" && item.customType === REGISTERED_TOOLS_ENTRY).at(-1);
+  return entry?.type === "custom" ? registeredToolNames(entry.data) : undefined;
 }
 
-function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolveConfig>) {
-  pi.registerTool({
+function recordRegisteredTools(pi: ExtensionAPI, ctx: ExtensionContext, tools: readonly string[]): string[] {
+  const previous = registrationHistory(ctx);
+  const remembered = [...new Set([...(previous ?? []), ...tools])].sort();
+  if (!previous || JSON.stringify(previous) !== JSON.stringify(remembered)) {
+    pi.appendEntry?.(REGISTERED_TOOLS_ENTRY, { version: 1, tools: remembered });
+  }
+  return remembered;
+}
+
+function registeredToolNames(data: unknown): string[] | undefined {
+  if (!data || typeof data !== "object" || !("version" in data) || data.version !== 1
+    || !("tools" in data) || !Array.isArray(data.tools)
+    || data.tools.length > Object.keys(WEB_OUTPUT_SCHEMAS).length
+    || !data.tools.every(name => typeof name === "string" && Object.hasOwn(WEB_OUTPUT_SCHEMAS, name))) return undefined;
+  return [...new Set(data.tools)].sort();
+}
+
+function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolveConfig>, previouslyRegistered?: ReadonlySet<string>) {
+  // All five tools stay direct. Metadata must not make excluded/inactive tools callable.
+  type DataTool = Omit<ToolDefinition, "execute"> & {
+    execute(...args: Parameters<ToolDefinition["execute"]>): Promise<unknown>;
+  };
+  const settings = pi.getSettings?.().defaultTools;
+  const disabledByDefault = (name: string) => Array.isArray(settings)
+    && settings.filter(entry => entry === `+${name}` || entry === `-${name}`).at(-1) === `-${name}`;
+  const register = (tool: DataTool) => pi.registerTool({
+    ...WEB_TOOL_METADATA,
+    ...tool,
+    // Late registration must not undo explicit settings or session deactivation.
+    // Pi restores previously active/pending names on reload even when this is false.
+    // Only first availability uses the default; re-registration cannot undo a
+    // manual deactivation, even if settings positively select this tool.
+    defaultActive: !previouslyRegistered?.has(tool.name) && !disabledByDefault(tool.name),
+    outputSchema: WEB_OUTPUT_SCHEMAS[tool.name],
+    async execute(id, args, signal, onUpdate, ctx) {
+      const secrets = Object.values(runtimeConfig(pi, ctx.cwd, projectIsTrusted(ctx)).apiKeys);
+      try {
+        const result = await tool.execute(id, args, signal,
+          onUpdate ? update => onUpdate(redactOutput(update, secrets)) : undefined, ctx);
+        // Some providers represent an aborted fetch as per-page failures.
+        // Cancellation is still a failed call, never a successful data snapshot.
+        if (signal?.aborted) throw new Error("Web tool cancelled.");
+        return jsonToolResult(result, WEB_OUTPUT_SCHEMAS[tool.name], secrets);
+      } catch (error) {
+        throw new Error(redactText(error instanceof Error ? error.message : "Web tool failed.", secrets).slice(0, 1000));
+      }
+    },
+  });
+  register({
     name: "web_search",
     label: "Web Search",
     description: buildSearchDescription(startupConfig.provider_search),
@@ -53,11 +133,19 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
     async execute(_toolCallId, rawParams, signal, onUpdate, ctx) {
       const params = rawParams as Record<string, any>;
       const queries = normalizeQueries(params);
-      const numResults = parseInteger(params.numResults, DEFAULT_NUM_RESULTS, "numResults", 1, MAX_NUM_RESULTS);
       if (queries.length === 0) throw new Error("web_search requires query or queries.");
 
       const config = runtimeConfig(pi, ctx.cwd, projectIsTrusted(ctx));
       assertProviderUnchanged("web_search", startupConfig.provider_search, config.provider_search);
+      const requestedResultLimit = parseInteger(
+        params.numResults ?? (config.provider_search === "brave" ? params.maxUrls : undefined),
+        DEFAULT_NUM_RESULTS,
+        "numResults",
+        1,
+      );
+      const effectiveResultLimit = capSearchResultLimit(config.provider_search, requestedResultLimit);
+      const requestedContextTokens = params.contextTokens == null ? undefined : parseInteger(params.contextTokens, 0, "contextTokens", 1);
+      const effectiveContextTokens = Math.min(requestedContextTokens ?? DEFAULT_SEARCH_CONTEXT_TOKENS, MAX_SEARCH_CONTEXT_TOKENS);
       const provider = createSearchProvider(config);
       const grouped = [];
       const progress = createProgress("search", config.provider_search, queries);
@@ -65,13 +153,24 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
       for (const query of queries) {
         markProgressCurrent(progress, query);
         emitProgress(onUpdate, progress);
-        const result = await provider.search({ ...params, query, numResults }, signal);
-        grouped.push({ query, results: result.results });
-        markProgressDone(progress, query, `${result.results.length} results`);
+        const result = await provider.search({ ...params, query, numResults: effectiveResultLimit, contextTokens: requestedContextTokens == null ? undefined : effectiveContextTokens }, signal);
+        const bounded = applySearchContextBudget(result.results, effectiveContextTokens * 4);
+        grouped.push({
+          query,
+          requestedResultLimit,
+          effectiveResultLimit: result.effectiveResultLimit ?? effectiveResultLimit,
+          requestedContextTokens,
+          effectiveContextTokens,
+          contextCharacters: bounded.contextCharacters,
+          ...(bounded.omittedContextCharacters ? { omittedContextCharacters: bounded.omittedContextCharacters } : {}),
+          resultCount: bounded.results.length,
+          results: bounded.results,
+        });
+        markProgressDone(progress, query, `${bounded.results.length} results`);
         emitProgress(onUpdate, progress);
       }
       const result = { provider: config.provider_search, queries: grouped };
-      return jsonToolResult(result);
+      return result;
     },
     renderCall(args, theme) {
       return new Text(renderWebCall("search", args as Record<string, any>, theme), 0, 0);
@@ -81,7 +180,7 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
     },
   });
 
-  pi.registerTool({
+  register({
     name: "web_fetch",
     label: "Web Fetch",
     description: buildFetchDescription(startupConfig.provider_fetch),
@@ -103,7 +202,7 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
         updateFetchProgress(progress, event);
         emitProgress(onUpdate, progress);
       });
-      return jsonToolResult(result);
+      return result;
     },
     renderCall(args, theme) {
       return new Text(renderWebCall("fetch", args as Record<string, any>, theme), 0, 0);
@@ -114,7 +213,7 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
   });
 
   if (startupConfig.apiKeys.context7) {
-    pi.registerTool({
+    register({
       name: "library_search",
       label: "Library Search",
       description: "Resolve library, package, framework, SDK, API, or CLI names to canonical library IDs with version, trust, and snippet metadata. Use when you need to inspect candidate matches (official sources, versions, forks); library_docs resolves names automatically.",
@@ -128,11 +227,11 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
         const limit = parseInteger(params.limit, 10, "limit", 1, MAX_NUM_RESULTS);
         const provider = createContext7Provider(runtimeConfig(pi, ctx.cwd, projectIsTrusted(ctx)));
         const result = await provider.searchLibraries({ libraryName, query, fast: params.fast === true, limit }, signal);
-        return jsonToolResult(result);
+        return result;
       },
     });
 
-    pi.registerTool({
+    register({
       name: "library_docs",
       label: "Library Docs",
       description: "Fetch current, version-aware documentation and code examples for a library. Pass libraryName to resolve it automatically, or a known libraryId.",
@@ -152,13 +251,13 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
           if (!libraryId) throw new Error(`No library found for '${libraryName}'. Try library_search with a more specific name.`);
         }
         const result = await provider.getDocs({ libraryId, query, version: optionalString(params.version, "version"), type: "json", fast: params.fast === true, limit }, signal);
-        return jsonToolResult(result);
+        return result;
       },
     });
   }
 
   if (startupConfig.apiKeys.exa) {
-    pi.registerTool({
+    register({
       name: "code_search",
       label: "Code Search",
       description: "Find practical code examples, usage patterns, setup snippets, migrations, and error context.",
@@ -174,14 +273,12 @@ function registerTools(pi: ExtensionAPI, startupConfig: ReturnType<typeof resolv
         const tokensNum = parseTokensNum(params.tokensNum);
         const provider = createCodeSearchProvider(runtimeConfig(pi, ctx.cwd, projectIsTrusted(ctx)));
         const result = await provider.searchCode({ query, tokensNum }, signal);
-        return jsonToolResult(result);
+        return result;
       },
     });
   }
 }
 
-const OPTIONAL_TOOLS = ["library_search", "library_docs", "code_search"];
-
 function toolConfigSignature(config: ReturnType<typeof resolveConfig>): string {
   return JSON.stringify({
     search: config.provider_search,
@@ -191,16 +288,6 @@ function toolConfigSignature(config: ReturnType<typeof resolveConfig>): string {
   });
 }
 
-function syncOptionalTools(pi: ExtensionAPI, config: ReturnType<typeof resolveConfig>) {
-  if (typeof pi.getActiveTools !== "function" || typeof pi.setActiveTools !== "function") return;
-  const enabled = [
-    ...(config.apiKeys.context7 ? ["library_search", "library_docs"] : []),
-    ...(config.apiKeys.exa ? ["code_search"] : []),
-  ];
-  const active = pi.getActiveTools().filter((name) => !OPTIONAL_TOOLS.includes(name));
-  pi.setActiveTools([...new Set([...active, ...enabled])]);
-}
-
 type ProgressKind = "search" | "fetch";
 type ProgressItem = { label: string; status: "pending" | "current" | "done" | "error"; note?: string; error?: string };
 type WebProgress = { kind: ProgressKind; provider: string; total: number; completed: number; items: ProgressItem[] };
@@ -212,7 +299,9 @@ export function buildSearchSchema(provider: SearchProviderName) {
   const props: Record<string, any> = {
     query: Type.Optional(Type.String({ description: "Single search query" })),
     queries: Type.Optional(Type.Array(Type.String(), { description: `Multiple related search queries (max ${MAX_QUERY_COUNT})`, maxItems: MAX_QUERY_COUNT })),
-    numResults: Type.Optional(int("Results per query", 1, MAX_NUM_RESULTS)),
+    numResults: Type.Optional(int("Desired results per query; capped by the active provider", 1)),
+    contextTokens: Type.Optional(int("Desired extracted context tokens; capped to the tool output budget and may enable provider extraction", 1)),
+    purpose: Type.Optional(Type.String({ description: "Optional task/use-case hint when supported by the provider", minLength: 1, maxLength: 2_000 })),
   };
   if (provider === "exa") Object.assign(props, {
     includeDomains: Type.Optional(Type.Array(Type.String())),
@@ -223,13 +312,32 @@ export function buildSearchSchema(provider: SearchProviderName) {
     endCrawlDate: Type.Optional(Type.String()),
     type: Type.Optional(Type.String()),
     category: Type.Optional(Type.String()),
+    maxAgeHours: Type.Optional(int("Maximum Exa cached content age in hours; 0 forces live crawl, -1 disables live crawl", -1)),
+  });
+  if (provider === "tinyfish") Object.assign(props, {
+    page: Type.Optional(int("Deprecated starting result page", 0, TINYFISH_MAX_PAGE)),
+    location: Type.Optional(Type.String()), language: Type.Optional(Type.String()),
+    includeDomains: Type.Optional(Type.Array(Type.String())), excludeDomains: Type.Optional(Type.Array(Type.String())),
+    domainType: Type.Optional(Type.Union([Type.Literal("web"), Type.Literal("news"), Type.Literal("research_paper")])),
+    afterDate: Type.Optional(Type.String({ pattern: "^\\d{4}-\\d{2}-\\d{2}$" })), beforeDate: Type.Optional(Type.String({ pattern: "^\\d{4}-\\d{2}-\\d{2}$" })),
+    recencyMinutes: Type.Optional(int("Freshness window in minutes", 1, 5_256_000)),
+    pubYearMin: Type.Optional(int("Minimum publication year", 0, 9_999)), pubYearMax: Type.Optional(int("Maximum publication year", 0, 9_999)),
   });
-  if (provider === "tinyfish") Object.assign(props, { page: Type.Optional(int("Result page", 1, 10)) });
   if (provider === "brave") Object.assign(props, {
-    country: Type.Optional(Type.String()), searchLang: Type.Optional(Type.String()), uiLang: Type.Optional(Type.String()), safesearch: Type.Optional(Type.String()), freshness: Type.Optional(Type.String()), maxUrls: Type.Optional(int("Maximum URLs", 1, MAX_NUM_RESULTS)),
+    country: Type.Optional(Type.String({ minLength: 2, maxLength: 2 })), searchLang: Type.Optional(Type.String({ minLength: 2, maxLength: 10 })),
+    safesearch: Type.Optional(Type.Union([Type.Literal("off"), Type.Literal("moderate"), Type.Literal("strict")])),
+    freshness: Type.Optional(Type.String()), spellcheck: Type.Optional(Type.Boolean()),
+    contextThresholdMode: Type.Optional(Type.Union([Type.Literal("disabled"), Type.Literal("strict"), Type.Literal("balanced"), Type.Literal("lenient")])),
+    maxSnippets: Type.Optional(int("Maximum context snippets", 1, 256)),
+    maxTokensPerUrl: Type.Optional(int("Maximum context tokens per URL", 512, 8_192)),
+    maxSnippetsPerUrl: Type.Optional(int("Maximum context snippets per URL", 1, 100)),
+    goggles: Type.Optional(Type.Union([Type.String(), Type.Array(Type.String(), { maxItems: 3 })])),
+    maxUrls: Type.Optional(int("Deprecated alias for numResults", 1)),
   });
   if (provider === "firecrawl") Object.assign(props, {
-    location: Type.Optional(Type.String()), country: Type.Optional(Type.String()), includeDomains: Type.Optional(Type.Array(Type.String())), excludeDomains: Type.Optional(Type.Array(Type.String())), categories: Type.Optional(Type.Array(Type.String())), tbs: Type.Optional(Type.String()), scrape: Type.Optional(Type.Boolean({ description: "Enable default markdown scrape-on-search" })), scrapeOptions: Type.Optional(Type.Object({}, { additionalProperties: true, description: "Firecrawl scrapeOptions for search." })),
+    location: Type.Optional(Type.String()), country: Type.Optional(Type.String()), includeDomains: Type.Optional(Type.Array(Type.String())), excludeDomains: Type.Optional(Type.Array(Type.String())),
+    categories: Type.Optional(Type.Array(Type.Union([Type.Literal("research"), Type.Literal("pdf"), Type.Literal("developer")]))),
+    tbs: Type.Optional(Type.String()), scrape: Type.Optional(Type.Boolean({ description: "Enable default markdown scrape-on-search" })), scrapeOptions: Type.Optional(Type.Object({}, { additionalProperties: true, description: "Firecrawl scrapeOptions for search." })),
   });
   return Type.Object(props, { additionalProperties: false });
 }
@@ -241,12 +349,23 @@ export function buildFetchSchema(provider: FetchProviderName) {
     offset: Type.Optional(int("Character offset for cached/ranged reads", 0, MAX_OFFSET)),
     limit: Type.Optional(int("Maximum characters to return", 1, MAX_LIMIT)),
     refresh: Type.Optional(Type.Boolean({ description: "Refetch even if cached" })),
+    maxAgeMs: Type.Optional(int("Maximum local/provider-cached page age in milliseconds; 0 requests live content where supported", 0)),
   };
+  if (provider === "exa") Object.assign(props, { maxAgeHours: Type.Optional(int("Maximum Exa cached content age in hours; 0 forces live crawl, -1 disables live crawl", -1)) });
   if (provider === "tinyfish") Object.assign(props, { format: Type.Optional(Type.Union([Type.Literal("markdown"), Type.Literal("html"), Type.Literal("json")])), links: Type.Optional(Type.Boolean()), imageLinks: Type.Optional(Type.Boolean()) });
+  if (provider === "tinyfish") Object.assign(props, {
+    purpose: Type.Optional(Type.String({ minLength: 1, maxLength: 2_000 })),
+    ttl: Type.Optional(int("Provider cache freshness tolerance in seconds; 0 prefers live fetch", 0)),
+    perUrlTimeoutMs: Type.Optional(int("Per-URL timeout in milliseconds", 1, 110_000)),
+    includeSelectors: Type.Optional(Type.Array(Type.String({ minLength: 1, maxLength: 1_000 }), { minItems: 1, maxItems: 20 })),
+    excludeSelectors: Type.Optional(Type.Array(Type.String({ minLength: 1, maxLength: 1_000 }), { minItems: 1, maxItems: 20 })),
+  });
   if (provider === "markdown_new") Object.assign(props, { method: Type.Optional(Type.Union([Type.Literal("auto"), Type.Literal("ai"), Type.Literal("browser")])), retainImages: Type.Optional(Type.Boolean()) });
   if (provider === "firecrawl") Object.assign(props, {
     format: Type.Optional(Type.Union([Type.Literal("markdown"), Type.Literal("html"), Type.Literal("json")])),
-    onlyMainContent: Type.Optional(Type.Boolean()), waitFor: Type.Optional(int("Milliseconds to wait", 0, 60_000)), mobile: Type.Optional(Type.Boolean()), location: Type.Optional(Type.String()), maxAge: Type.Optional(int("Maximum cached page age", 0)),
+    onlyMainContent: Type.Optional(Type.Boolean()), waitFor: Type.Optional(int("Milliseconds to wait", 0, 60_000)), mobile: Type.Optional(Type.Boolean()),
+    location: Type.Optional(Type.Object({ country: Type.Optional(Type.String()), languages: Type.Optional(Type.Array(Type.String())) }, { additionalProperties: false })),
+    maxAge: Type.Optional(int("Maximum cached page age in milliseconds", 0)),
   });
   return Type.Object(props, { additionalProperties: false });
 }
@@ -304,6 +423,7 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
   const defaultLimit = urls.length > 1 ? MULTI_FETCH_LIMIT : DEFAULT_FETCH_LIMIT;
   const limit = parseInteger(params.limit, defaultLimit, "limit", 1, MAX_LIMIT);
   const refresh = params.refresh === true;
+  const maxAgeMs = localCacheMaxAge(providerName, params);
 
   const pages = new Map<string, { page?: CachedPage; cached: boolean; refreshed: boolean; error?: string }>();
   const cacheKeys = new Map<string, string>();
@@ -312,7 +432,7 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
     const cacheKey = buildCacheKey(providerName, url, params, config);
     cacheKeys.set(url, cacheKey);
     const cached = fetchCache.get(cacheKey);
-    if (cached && !refresh) {
+    if (cached && !refresh && (maxAgeMs == null || (maxAgeMs > 0 && Date.now() - cached.fetchedAt <= maxAgeMs))) {
       pages.set(url, { page: cached, cached: true, refreshed: false });
       onProgress?.({ status: "done", url, note: "cached" });
     } else {
@@ -327,7 +447,7 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
     for (const requestedUrl of missing) {
       const item = mapped.get(requestedUrl);
       if (!item || item.error) {
-        const error = item?.error ?? "No content returned.";
+        const error = publicPageError(item?.error ?? "No content returned.");
         pages.set(requestedUrl, { error, cached: false, refreshed: refresh });
         onProgress?.({ status: "error", url: requestedUrl, error });
         continue;
@@ -356,6 +476,14 @@ export async function fetchWithCache(providerName: FetchProviderName, params: Re
   }) };
 }
 
+function localCacheMaxAge(provider: FetchProviderName, params: Record<string, any>): number | undefined {
+  if (params.maxAgeMs != null) return parseInteger(params.maxAgeMs, 0, "maxAgeMs", 0);
+  if (provider === "exa" && typeof params.maxAgeHours === "number" && params.maxAgeHours >= 0) return params.maxAgeHours * 3_600_000;
+  if (provider === "tinyfish" && typeof params.ttl === "number") return params.ttl * 1_000;
+  if (provider === "firecrawl" && typeof params.maxAge === "number") return params.maxAge;
+  return undefined;
+}
+
 
 export function pageSlice(page: CachedPage, offset: number, limit: number, cached: boolean, refreshed: boolean) {
   const total = page.content.length;
@@ -380,19 +508,28 @@ function projectIsTrusted(ctx: { isProjectTrusted?: () => boolean }): boolean {
 }
 
 function runtimeConfig(pi: ExtensionAPI, cwd: string, projectTrusted: boolean) {
-  return resolveConfig(
-    { providerSearch: pi.getFlag("web-provider-search"), providerFetch: pi.getFlag("web-provider-fetch") },
-    cwd,
-    process.env,
-    { includeProject: projectTrusted },
-  );
+  try {
+    return resolveConfig(
+      { providerSearch: pi.getFlag("web-provider-search"), providerFetch: pi.getFlag("web-provider-fetch") },
+      cwd,
+      process.env,
+      { includeProject: projectTrusted },
+    );
+  } catch {
+    // JSON parse and invalid-provider errors can echo credential-bearing config.
+    throw new Error("Web configuration is invalid. Check provider names, flags and config JSON.");
+  }
 }
 
 const MAX_OUTPUT_BYTES = 50_000;
 
-export function jsonToolResult(result: unknown) {
-  const bounded = boundStructuredResult(result);
-  return { content: [{ type: "text" as const, text: JSON.stringify(bounded) }], details: boundedDetails(bounded) };
+export function jsonToolResult(result: unknown, schema?: TSchema, secrets: readonly string[] = []) {
+  const publicResult = schema ? projectOutput(schema, result) : result;
+  const bounded = boundStructuredResult(redactOutput(publicResult, secrets));
+  const text = JSON.stringify(bounded);
+  // The model and scripts receive exactly the same bounded, redacted JSON value.
+  const structuredContent = JSON.parse(text);
+  return { content: [{ type: "text" as const, text }], structuredContent, details: boundedDetails(structuredContent) };
 }
 
 function boundStructuredResult(result: unknown): unknown {
@@ -436,8 +573,20 @@ function isSearchResult(value: any): value is { provider?: string; queries: Arra
 }
 
 function boundSearchResult(value: { provider?: string; queries: Array<{ query?: string; results?: any[] }> }) {
-  const compact = limitStrings(value, 1_000) as typeof value;
-  const maxSnippet = Math.max(0, ...compact.queries.flatMap((group) => (group.results ?? []).map((item) => typeof item?.snippet === "string" ? item.snippet.length : 0)));
+  const compact = {
+    ...(limitStrings(value, 1_000) as Record<string, unknown>),
+    queries: value.queries.map((group) => ({
+      ...(limitStrings(group, 1_000) as Record<string, unknown>),
+      results: (group.results ?? []).map((item) => ({
+        ...(limitStrings(item, 1_000) as Record<string, unknown>),
+        ...(typeof item?.content === "string" ? { content: item.content } : {}),
+      })),
+    })),
+  } as typeof value;
+  const maxSnippet = Math.max(0, ...compact.queries.flatMap((group) => (group.results ?? []).flatMap((item) => [
+    typeof item?.snippet === "string" ? item.snippet.length : 0,
+    typeof item?.content === "string" ? item.content.length : 0,
+  ])));
   let low = 0;
   let high = maxSnippet;
   let best = searchResultWithLimits(compact, 0);
@@ -473,17 +622,29 @@ function boundSearchResult(value: { provider?: string; queries: Array<{ query?:
 function searchResultWithLimits(value: { provider?: string; queries: Array<{ query?: string; results?: any[] }> }, snippetChars: number, resultsPerQuery?: number) {
   return {
     ...value,
-    queries: value.queries.map((group) => ({
-      ...group,
-      results: (group.results ?? []).slice(0, resultsPerQuery).map((item) => {
-        if (typeof item?.snippet !== "string") return item;
-        if (snippetChars === 0) {
-          const { snippet: _snippet, ...metadata } = item;
-          return metadata;
+    queries: value.queries.map((group) => {
+      const originalContextCharacters = (group.results ?? []).reduce((total, item) => total + (typeof item?.content === "string" ? item.content.length : 0), 0);
+      const results = (group.results ?? []).slice(0, resultsPerQuery).map((item) => {
+        const bounded = { ...item };
+        for (const key of ["snippet", "content"] as const) {
+          if (typeof bounded[key] !== "string") continue;
+          if (snippetChars === 0) delete bounded[key];
+          else bounded[key] = safePrefix(bounded[key], snippetChars);
         }
-        return { ...item, snippet: safePrefix(item.snippet, snippetChars) };
-      }),
-    })),
+        return bounded;
+      });
+      const omittedResultCount = Math.max(0, (group.results?.length ?? 0) - results.length);
+      const contextCharacters = results.reduce((total, item) => total + (typeof item?.content === "string" ? item.content.length : 0), 0);
+      const omittedContextCharacters = (typeof (group as any).omittedContextCharacters === "number" ? (group as any).omittedContextCharacters : 0) + originalContextCharacters - contextCharacters;
+      return {
+        ...group,
+        contextCharacters,
+        ...(omittedContextCharacters ? { omittedContextCharacters } : {}),
+        resultCount: results.length,
+        ...(omittedResultCount ? { omittedResultCount } : {}),
+        results,
+      };
+    }),
   };
 }
 
@@ -521,12 +682,6 @@ function fetchResultWithContentLimit(value: { provider?: string; results: any[]
   };
 }
 
-function safePrefix(value: string, maxChars: number): string {
-  let end = Math.min(value.length, maxChars);
-  if (end > 0 && /[\uD800-\uDBFF]/.test(value[end - 1])) end--;
-  return value.slice(0, end);
-}
-
 function limitStrings(value: unknown, maxChars: number): unknown {
   if (typeof value === "string") return safePrefix(value, maxChars);
   if (Array.isArray(value)) return value.map((item) => limitStrings(item, maxChars));
@@ -534,9 +689,11 @@ function limitStrings(value: unknown, maxChars: number): unknown {
   return value;
 }
 
-function parseInteger(value: unknown, defaultValue: number, name: string, min: number, max: number): number {
+function parseInteger(value: unknown, defaultValue: number, name: string, min: number, max?: number): number {
   if (value == null) return defaultValue;
-  if (typeof value !== "number" || !Number.isInteger(value) || !Number.isFinite(value) || value < min || value > max) throw new Error(`${name} must be a finite integer between ${min} and ${max}.`);
+  if (typeof value !== "number" || !Number.isInteger(value) || !Number.isFinite(value) || value < min || (max != null && value > max)) {
+    throw new Error(`${name} must be a finite integer ${max == null ? `>= ${min}` : `between ${min} and ${max}`}.`);
+  }
   return value;
 }
 
@@ -565,7 +722,6 @@ function fetchConfigDefaults(provider: FetchProviderName, config?: any): Record<
 function providerScope(provider: FetchProviderName, config?: any): string {
   const keyMap: Partial<Record<FetchProviderName, string | undefined>> = {
     exa: config?.apiKeys?.exa,
-    exa_mcp: config?.apiKeys?.exa,
     tinyfish: config?.apiKeys?.tinyfish,
     firecrawl: config?.apiKeys?.firecrawl,
   };
@@ -591,7 +747,14 @@ function searchDetails(value: any) {
     provider: value.provider,
     queries: value.queries.map((q: any) => ({
       query: q.query,
+      requestedResultLimit: q.requestedResultLimit,
+      effectiveResultLimit: q.effectiveResultLimit,
+      requestedContextTokens: q.requestedContextTokens,
+      effectiveContextTokens: q.effectiveContextTokens,
+      contextCharacters: q.contextCharacters,
+      omittedContextCharacters: q.omittedContextCharacters,
       resultCount: (q.results ?? []).length,
+      omittedResultCount: q.omittedResultCount,
       results: (q.results ?? []).map((r: any) => ({ title: r.title, url: r.url, siteName: r.siteName, position: r.position })),
     })),
   };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-web-kit",
-  "version": "0.2.4",
+  "version": "0.4.0",
   "description": "Context-efficient web search and fetch tools for Pi.",
   "type": "module",
   "license": "MIT",
@@ -58,18 +58,21 @@
     "typebox": "*"
   },
   "devDependencies": {
-    "@earendil-works/pi-ai": "^0.80.10",
-    "@earendil-works/pi-coding-agent": "^0.80.10",
-    "@earendil-works/pi-tui": "^0.80.10",
-    "@types/node": "^26.1.1",
-    "tsx": "^4.23.1",
-    "typebox": "^1.3.6",
+    "@earendil-works/pi-ai": "1.1.0",
+    "@earendil-works/pi-coding-agent": "1.1.0",
+    "@earendil-works/pi-tui": "1.1.0",
+    "@types/node": "^26.6.3",
+    "tsx": "^4.23.15",
+    "typebox": "^1.3.34",
     "typescript": "^7.0.2"
   },
   "publishConfig": {
     "access": "public"
   },
   "engines": {
-    "node": ">=20.6.0"
+    "node": ">=22.19.0"
+  },
+  "dependencies": {
+    "@mocito/install-telemetry": "0.1.1"
   }
 }

package/src/config.ts

@@ -3,11 +3,11 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import type { FetchProviderName, SearchProviderName, WebKitConfig } from "./types.js";
 
-const SEARCH = ["exa_mcp", "exa", "tinyfish", "brave", "firecrawl"] as const;
-const FETCH = ["exa_mcp", "exa", "tinyfish", "markdown_new", "firecrawl"] as const;
+const SEARCH = ["exa", "tinyfish", "brave", "firecrawl"] as const;
+const FETCH = ["exa", "tinyfish", "markdown_new", "firecrawl"] as const;
 const DEFAULT_CONFIG: WebKitConfig = {
-  provider_search: "exa_mcp",
-  provider_fetch: "exa_mcp",
+  provider_search: "exa",
+  provider_fetch: "exa",
   apiKeys: {},
   markdownNew: { method: "auto", retainImages: false },
 };

package/src/http.ts

@@ -26,16 +26,26 @@ export async function fetchWithTimeout(url: string, init: RequestInit & { timeou
 export async function requestJson<T>(url: string, init: RequestInit & { timeoutMs?: number } = {}): Promise<T> {
   const res = await fetchWithTimeout(url, init);
   const text = await res.text();
-  if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
+  if (!res.ok) throw new Error(`Provider request failed (HTTP ${res.status}).`);
   if (!text) return undefined as T;
-  return JSON.parse(text) as T;
+  try { return JSON.parse(text) as T; }
+  catch { throw new Error("Provider returned invalid JSON."); }
 }
 
 export function asSnippet(value: unknown): string | undefined {
+  return asText(value)?.slice(0, 1000) || undefined;
+}
+
+export function asText(value: unknown): string | undefined {
   if (value == null) return undefined;
-  if (Array.isArray(value)) return value.filter(Boolean).join("\n").slice(0, 1000) || undefined;
-  if (typeof value === "string") return value.slice(0, 1000) || undefined;
-  return JSON.stringify(value).slice(0, 1000);
+  if (Array.isArray(value)) return value.filter(Boolean).map((item) => typeof item === "string" ? item : JSON.stringify(item)).join("\n") || undefined;
+  if (typeof value === "string") return value || undefined;
+  return JSON.stringify(value);
+}
+
+export function withoutContent(value: any): Record<string, unknown> {
+  const { text: _text, content: _content, markdown: _markdown, html: _html, summary: _summary, highlights: _highlights, ...metadata } = value ?? {};
+  return metadata;
 }
 
 export function normalizeUrls(input: { url?: string; urls?: string[] }, maxCount?: number): string[] {