pi-web-kit 0.2.4 → 0.4.0

package/src/install-telemetry.ts

@@ -1,17 +1,35 @@
 import { readFileSync } from "node:fs";
-import { mkdir, writeFile } from "node:fs/promises";
 import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { reportInstallTelemetry as report } from "@mocito/install-telemetry";
 import { getAgentDir } from "@earendil-works/pi-coding-agent";
 
 const PACKAGE_NAME = "pi-web-kit";
-const INSTALL_TELEMETRY_URL = "https://mocito.dev/api/report-install";
-const INSTALL_TELEMETRY_TIMEOUT_MS = 5000;
+const INSTALL_TELEMETRY_ENDPOINT = "https://mocito.dev/api/report-install";
+const CI_ENVIRONMENT_VARIABLES = [
+  "APPVEYOR",
+  "BITBUCKET_BUILD_NUMBER",
+  "BUILDKITE",
+  "CIRCLECI",
+  "CODESPACES",
+  "DRONE",
+  "GITHUB_ACTIONS",
+  "GITLAB_CI",
+  "JENKINS_URL",
+  "NETLIFY",
+  "TEAMCITY_VERSION",
+  "TF_BUILD",
+  "TRAVIS",
+  "VERCEL",
+];
 
-type InstallTelemetryState = { lastReportedVersion?: string };
+interface PiSettingsDocument {
+  enableInstallTelemetry?: unknown;
+}
 
 function readJsonFile(path: string): unknown {
   try {
-    return JSON.parse(readFileSync(path, "utf8"));
+    return JSON.parse(readFileSync(path, "utf8")) as unknown;
   } catch {
     return {};
   }
@@ -22,48 +40,38 @@ function isTruthyEnvFlag(value: string | undefined): boolean {
   return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes";
 }
 
-function isInstallTelemetryEnabled(): boolean {
-  if (isTruthyEnvFlag(process.env.PI_OFFLINE)) return false;
-  if (isTruthyEnvFlag(process.env.CI) || isTruthyEnvFlag(process.env.GITHUB_ACTIONS)) return false;
-  if (process.env.PI_TELEMETRY !== undefined) return isTruthyEnvFlag(process.env.PI_TELEMETRY);
-  const settings = readJsonFile(join(getAgentDir(), "settings.json")) as { enableInstallTelemetry?: unknown };
-  return settings.enableInstallTelemetry !== false;
+function isPresentEnvFlag(value: string | undefined): boolean {
+  if (!value) return false;
+  const normalized = value.toLowerCase();
+  return normalized !== "0" && normalized !== "false" && normalized !== "no";
 }
 
-function getPackageVersion(): string {
-  try {
-    const packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { version?: unknown };
-    return typeof packageJson.version === "string" && packageJson.version.length > 0 ? packageJson.version : "0.0.0";
-  } catch {
-    return "0.0.0";
-  }
+export function isInstallTelemetryEnabled(env: NodeJS.ProcessEnv = process.env, settingsPath = join(getAgentDir(), "settings.json")): boolean {
+  if (isTruthyEnvFlag(env.CI)) return false;
+  if (CI_ENVIRONMENT_VARIABLES.some((name) => isPresentEnvFlag(env[name]))) return false;
+  if (isTruthyEnvFlag(env.PI_OFFLINE)) return false;
+
+  const settings = readJsonFile(settingsPath) as PiSettingsDocument;
+  if (settings.enableInstallTelemetry === false) return false;
+  if (env.PI_TELEMETRY !== undefined) return isTruthyEnvFlag(env.PI_TELEMETRY);
+  return true;
 }
 
-function getInstallTelemetryUserAgent(version: string): string {
-  const runtimeVersions = process.versions as NodeJS.ProcessVersions & { bun?: string };
-  const runtime = runtimeVersions.bun ? `bun/${runtimeVersions.bun}` : `node/${process.version}`;
-  return `${PACKAGE_NAME}/${version} (${process.platform}; ${runtime}; ${process.arch})`;
+function getPackageVersion(): string {
+  const packageJson = readJsonFile(fileURLToPath(new URL("../package.json", import.meta.url))) as { version?: unknown };
+  return typeof packageJson.version === "string" && packageJson.version.length > 0 ? packageJson.version : "0.0.0";
 }
 
-export async function reportInstallTelemetry(): Promise<void> {
+export function reportInstallTelemetry(): void {
   try {
-    if (!isInstallTelemetryEnabled()) return;
-
-    const version = getPackageVersion();
-    const telemetryDir = join(getAgentDir(), "extensions");
-    const statePath = join(telemetryDir, "pi-web-kit-install.json");
-    const state = readJsonFile(statePath) as InstallTelemetryState;
-    if (state.lastReportedVersion === version) return;
-
-    await mkdir(telemetryDir, { recursive: true });
-    await writeFile(statePath, `${JSON.stringify({ lastReportedVersion: version }, null, 2)}\n`);
-
-    const params = new URLSearchParams({ tool: PACKAGE_NAME, version });
-    await fetch(`${INSTALL_TELEMETRY_URL}?${params.toString()}`, {
-      headers: { "User-Agent": getInstallTelemetryUserAgent(version) },
-      signal: AbortSignal.timeout(INSTALL_TELEMETRY_TIMEOUT_MS),
-    });
+    void report({
+      endpoint: INSTALL_TELEMETRY_ENDPOINT,
+      tool: PACKAGE_NAME,
+      version: getPackageVersion(),
+      statePath: join(getAgentDir(), "extensions", "pi-web-kit-install.json"),
+      enabled: isInstallTelemetryEnabled(),
+    }).catch(() => undefined);
   } catch {
-    // Best-effort install telemetry: ignore settings, filesystem, and network failures.
+    // Best-effort telemetry: ignore local policy and filesystem failures.
   }
 }

package/src/limits.ts

@@ -1,8 +1,19 @@
+import type { SearchProviderName } from "./types.js";
+
 export const MAX_QUERY_COUNT = 5;
 export const MAX_URL_COUNT = 10;
 export const MAX_URL_LENGTH = 2_048;
 export const MAX_NUM_RESULTS = 20;
 export const DEFAULT_NUM_RESULTS = 10;
+export const DEFAULT_SEARCH_CONTEXT_TOKENS = 8_192;
+export const MAX_SEARCH_CONTEXT_TOKENS = 10_000;
+export const SEARCH_RESULT_LIMITS: Readonly<Record<SearchProviderName, number | undefined>> = {
+  exa: 100,
+  brave: 50,
+  firecrawl: 100,
+  tinyfish: undefined,
+};
+export const TINYFISH_MAX_PAGE = 10;
 export const MAX_OFFSET = 10_000_000;
 export const MAX_LIMIT = 100_000;
 export const DEFAULT_FETCH_LIMIT = 30_000;
@@ -12,3 +23,31 @@ export const FETCH_CACHE_MAX_ENTRIES = 100;
 export const FETCH_CACHE_MAX_BYTES = 20 * 1024 * 1024;
 export const FETCH_CACHE_TTL_MS = 30 * 60 * 1000;
 export const FETCH_CONCURRENCY = 3;
+
+export function capSearchResultLimit(provider: SearchProviderName, requested: number): number {
+  return Math.min(requested, SEARCH_RESULT_LIMITS[provider] ?? requested);
+}
+
+export function applySearchContextBudget(results: Array<{ content?: string; [key: string]: unknown }>, maxCharacters: number) {
+  let remaining = maxCharacters;
+  let remainingContentResults = results.filter((result) => typeof result.content === "string").length;
+  let contextCharacters = 0;
+  let omittedContextCharacters = 0;
+  const bounded = results.map((result) => {
+    if (typeof result.content !== "string") return result;
+    const share = Math.max(0, Math.floor(remaining / remainingContentResults));
+    const content = safePrefix(result.content, share);
+    remaining -= content.length;
+    remainingContentResults--;
+    contextCharacters += content.length;
+    omittedContextCharacters += result.content.length - content.length;
+    return { ...result, content };
+  });
+  return { results: bounded, contextCharacters, omittedContextCharacters };
+}
+
+export function safePrefix(value: string, maxChars: number): string {
+  let end = Math.min(value.length, maxChars);
+  if (end > 0 && /[\uD800-\uDBFF]/.test(value[end - 1])) end--;
+  return value.slice(0, end);
+}

package/src/output.ts

@@ -0,0 +1,140 @@
+import { Type, type TSchema } from "typebox";
+import { Check } from "typebox/value";
+
+const object = (properties: Record<string, TSchema>) => Type.Object(properties, { additionalProperties: false });
+const string = () => Type.Optional(Type.String());
+const number = () => Type.Optional(Type.Number());
+const boolean = () => Type.Optional(Type.Boolean());
+const truncated = object({ truncated: Type.Literal(true), message: Type.String() });
+const output = (schema: TSchema) => Type.Union([schema, truncated]);
+
+// Only explicitly selected fields cross the tool boundary. Provider metadata,
+// cache keys, response headers and Context7's untyped `rules` are not a contract.
+export const WEB_OUTPUT_SCHEMAS: Record<string, TSchema> = {
+  web_search: output(object({
+    provider: Type.String(),
+    queries: Type.Array(object({
+      query: Type.String(), requestedResultLimit: Type.Integer(), effectiveResultLimit: Type.Integer(),
+      requestedContextTokens: Type.Optional(Type.Integer()), effectiveContextTokens: Type.Integer(),
+      contextCharacters: Type.Integer(), omittedContextCharacters: Type.Optional(Type.Integer()),
+      resultCount: Type.Integer(), omittedResultCount: Type.Optional(Type.Integer()),
+      results: Type.Array(object({
+        url: Type.String(), title: string(), snippet: string(), content: string(),
+        contentFormat: Type.Optional(Type.Union([Type.Literal("markdown"), Type.Literal("text")])),
+        siteName: string(), position: number(),
+      })),
+    })),
+  })),
+  web_fetch: output(object({
+    provider: Type.String(),
+    results: Type.Array(Type.Union([
+      object({ url: Type.String(), error: Type.String() }),
+      object({
+        url: Type.String(), fetchedUrl: Type.String(), title: string(), content: Type.String(),
+        format: Type.Union([Type.Literal("markdown"), Type.Literal("html"), Type.Literal("json")]),
+        cached: Type.Boolean(), refreshed: Type.Boolean(),
+        range: object({
+          offset: Type.Integer(), limit: Type.Integer(), returned: Type.Integer(), total: Type.Integer(),
+          truncated: Type.Boolean(), hasPrevious: Type.Boolean(), hasNext: Type.Boolean(),
+          nextOffset: Type.Optional(Type.Integer()),
+        }),
+      }),
+    ])),
+  })),
+  library_search: output(object({
+    provider: Type.Literal("context7"), libraryName: Type.String(), query: Type.String(),
+    searchFilterApplied: boolean(),
+    results: Type.Array(object({
+      id: Type.String(), title: string(), description: string(), branch: string(),
+      lastUpdateDate: string(), state: string(), totalTokens: number(), totalSnippets: number(),
+      stars: number(), trustScore: number(), benchmarkScore: number(),
+      versions: Type.Optional(Type.Array(Type.String())),
+    })),
+  })),
+  library_docs: output(object({
+    provider: Type.Literal("context7"), libraryId: Type.String(), query: Type.String(),
+    codeSnippets: Type.Array(object({
+      codeTitle: string(), codeDescription: string(), codeLanguage: string(), codeTokens: number(),
+      codeId: string(), pageTitle: string(), sourceFile: string(), isDynamic: boolean(),
+      codeList: Type.Optional(Type.Array(object({ language: Type.String(), code: Type.String() }))),
+    })),
+    infoSnippets: Type.Array(object({
+      pageId: string(), breadcrumb: string(), content: Type.String(), contentTokens: number(),
+    })),
+  })),
+  code_search: output(object({
+    provider: Type.Literal("exa"), query: Type.String(), response: Type.String(),
+    resultsCount: number(), searchTime: number(), outputTokens: number(), requestId: string(),
+  })),
+};
+
+export const WEB_TOOL_METADATA = {
+  namespace: {
+    name: "web",
+    description: "Bounded web research, library documentation and code examples.",
+    instructions: "Keep existing tool names: web_search, web_fetch, library_search, library_docs, code_search. Await describeTool(name) for provider-specific inputs and output types. Calls return objects, not JSON strings. Fetch failures are per-item error strings; check them before using content. A top-level truncated:true/message result asks you to refine the request. Validation, cancellation and request failures reject. Parallel independent reads are supported; provider requests can incur costs. Availability depends on configured providers and active tools. These hints do not grant approval.",
+  },
+  annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
+} as const;
+
+/** Project to an allowlist before bounding or publishing. Invalid required data fails closed. */
+export function projectOutput(schema: TSchema, value: unknown): any {
+  const shape = schema as TSchema & {
+    anyOf?: TSchema[]; type?: string; properties?: Record<string, TSchema>;
+    required?: string[]; items?: TSchema;
+  };
+  if (shape.anyOf) {
+    for (const variant of shape.anyOf) {
+      try { return projectOutput(variant, value); } catch { /* try the next declared shape */ }
+    }
+    throw new Error("Provider returned an invalid result shape.");
+  }
+  let projected = value;
+  if (shape.type === "object" && value && typeof value === "object" && !Array.isArray(value)) {
+    projected = Object.fromEntries(Object.entries(shape.properties ?? {}).flatMap(([key, field]) => {
+      if (!Object.hasOwn(value, key) || (value as any)[key] == null) return [];
+      try { return [[key, projectOutput(field as TSchema, (value as any)[key])]]; }
+      catch {
+        if (shape.required?.includes(key)) throw new Error("Provider returned an invalid result shape.");
+        return [];
+      }
+    }));
+  } else if (shape.type === "array" && shape.items && Array.isArray(value)) {
+    projected = value.map(item => projectOutput(shape.items!, item));
+  }
+  if (!Check(schema, projected)) throw new Error("Provider returned an invalid result shape.");
+  return projected;
+}
+
+export function redactText(text: string, secrets: readonly string[] = []): string {
+  let redacted = text;
+  for (const secret of secrets.filter(Boolean).sort((a, b) => b.length - a.length)) {
+    redacted = redacted.split(secret).join("*".repeat(secret.length));
+  }
+  return redacted
+    .replace(/(https?:\/\/)([^\s/@"<>]+)@/gi, (_match, prefix, secret) => `${prefix}${"*".repeat(secret.length)}@`)
+    // Header context distinguishes credentials from prose such as "Basic
+    // authentication" or "Bearer token", without exempting short credentials.
+    .replace(/(\b(?:proxy-)?authorization\b["']?\s*[:=]\s*["'`]?(?:Bearer|Basic)[ \t]+)([A-Za-z0-9._~+/=-]+)/gi, (_match, prefix, secret) => `${prefix}${"*".repeat(secret.length)}`)
+    .replace(/([?&](?:api[-_]?key|access[-_]?token|token|secret|password)=)([^&#\s"'<>]+)/gi, (_match, prefix, secret) => `${prefix}${"*".repeat(secret.length)}`);
+}
+
+export function redactOutput(value: any, secrets: readonly string[]): any {
+  if (typeof value === "string") return redactText(value, secrets);
+  if (Array.isArray(value)) return value.map(item => redactOutput(item, secrets));
+  if (value && typeof value === "object") {
+    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, redactOutput(item, secrets)]));
+  }
+  return value;
+}
+
+/** Provider error bodies are not page content and must not become tool output. */
+export function publicPageError(error: unknown): string {
+  if (typeof error === "string" && (
+    /^Provider request failed \(HTTP \d{3}\)\.$/.test(error)
+    || /^Request timed out after \d+ms$/.test(error)
+    || /^No content returned(?: by (?:TinyFish|Exa(?: contents endpoint)?))?\.$/.test(error)
+    || error === "Provider returned invalid JSON."
+  )) return error;
+  return "Provider could not fetch this page.";
+}

package/src/providers/brave.ts

@@ -1,4 +1,4 @@
-import { asSnippet, requestJson } from "../http.js";
+import { asSnippet, asText, requestJson } from "../http.js";
 import type { SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
 
@@ -7,24 +7,51 @@ export class BraveProvider implements SearchProvider {
   constructor(config: WebKitConfig) { this.key = requireKey(config, "brave"); }
 
   async search(input: SearchInput, signal?: AbortSignal) {
-    const url = new URL("https://api.search.brave.com/res/v1/llm/context");
-    url.searchParams.set("q", input.query);
+    const params: Record<string, string | number | boolean | string[]> = { q: boundedQuery(input.query) };
     if (input.numResults) {
-      const count = String(Math.max(1, Math.min(input.numResults, 20)));
-      url.searchParams.set("count", count);
-      url.searchParams.set("maximum_number_of_urls", String(input.maxUrls ?? count));
+      params.count = input.numResults;
+      params.maximum_number_of_urls = input.numResults;
+    }
+    if (input.contextTokens) params.maximum_number_of_tokens = Math.max(1_024, Math.min(input.contextTokens, 32_768));
+    if (typeof input.country === "string") params.country = input.country;
+    if (typeof input.searchLang === "string") params.search_lang = input.searchLang;
+    if (typeof input.safesearch === "string") params.safesearch = input.safesearch;
+    if (typeof input.freshness === "string") params.freshness = input.freshness;
+    if (typeof input.spellcheck === "boolean") params.spellcheck = input.spellcheck;
+    if (typeof input.contextThresholdMode === "string") params.context_threshold_mode = input.contextThresholdMode;
+    if (typeof input.maxSnippets === "number") params.maximum_number_of_snippets = input.maxSnippets;
+    if (typeof input.maxTokensPerUrl === "number") params.maximum_number_of_tokens_per_url = input.maxTokensPerUrl;
+    if (typeof input.maxSnippetsPerUrl === "number") params.maximum_number_of_snippets_per_url = input.maxSnippetsPerUrl;
+    if (typeof input.goggles === "string" || Array.isArray(input.goggles)) params.goggles = input.goggles;
+
+    const url = new URL("https://api.search.brave.com/res/v1/llm/context");
+    for (const [name, value] of Object.entries(params)) {
+      if (Array.isArray(value)) for (const item of value) url.searchParams.append(name, item);
+      else url.searchParams.set(name, String(value));
     }
-    if (typeof input.country === "string") url.searchParams.set("country", input.country);
-    if (typeof input.searchLang === "string") url.searchParams.set("search_lang", input.searchLang);
-    if (typeof input.uiLang === "string") url.searchParams.set("ui_lang", input.uiLang);
-    if (typeof input.safesearch === "string") url.searchParams.set("safesearch", input.safesearch);
-    if (typeof input.freshness === "string") url.searchParams.set("freshness", input.freshness);
-    const data = await requestJson<any>(url.toString(), { headers: { "X-Subscription-Token": this.key, accept: "application/json" }, signal, timeoutMs: 30000 });
+    const usePost = input.goggles != null || url.toString().length > 2_000;
+    const data = await requestJson<any>(usePost ? `${url.origin}${url.pathname}` : url.toString(), {
+      method: usePost ? "POST" : undefined,
+      headers: { "X-Subscription-Token": this.key, accept: "application/json", ...(usePost ? { "content-type": "application/json" } : {}) },
+      body: usePost ? JSON.stringify(params) : undefined,
+      signal,
+      timeoutMs: 30000,
+    });
     const sources = data.sources ?? data.web?.results ?? [];
     const snippets = data.grounding?.generic ?? [];
-    const rows = sources.length ? sources : snippets;
-    return { provider: "brave" as const, query: input.query, results: rows.map((r: any, i: number) => ({
-      title: r.title ?? r.name, url: r.url, snippet: asSnippet(r.snippets ?? r.description ?? snippets[i]?.snippets), siteName: r.site_name ?? r.source, position: i + 1,
-    })).filter((r: any) => r.url) };
+    const sourceRows = Array.isArray(sources)
+      ? sources
+      : Object.entries(sources).map(([url, source]: [string, any]) => ({ ...source, url }));
+    const rows = snippets.length ? snippets : sourceRows;
+    return { provider: "brave" as const, query: input.query, results: rows.map((r: any, i: number) => {
+      const content = asText(r.snippets ?? r.snippet ?? r.description ?? snippets[i]?.snippets);
+      return {
+        title: r.title ?? r.name, url: r.url, snippet: asSnippet(content), content, contentFormat: content ? "markdown" as const : undefined, siteName: r.site_name ?? r.source, position: i + 1,
+      };
+    }).filter((r: any) => r.url) };
   }
 }
+
+function boundedQuery(query: string): string {
+  return query.trim().split(/\s+/).slice(0, 50).join(" ").slice(0, 400);
+}

package/src/providers/exa.ts

@@ -1,4 +1,4 @@
-import { asSnippet, normalizeUrls, requestJson } from "../http.js";
+import { asSnippet, asText, normalizeUrls, requestJson, withoutContent } from "../http.js";
 import { DEFAULT_NUM_RESULTS } from "../limits.js";
 import { urlsMatch } from "../urls.js";
 import type { ExaCodeInput, ExaCodeResult, FetchInput, FetchProvider, SearchInput, SearchProvider, WebFetchResult, WebKitConfig } from "../types.js";
@@ -14,10 +14,15 @@ export class ExaProvider implements SearchProvider, FetchProvider {
   }
 
   async search(input: SearchInput, signal?: AbortSignal) {
+    const textCharacters = input.contextTokens ? Math.min(100_000, Math.max(1, Math.floor(input.contextTokens * 4 / (input.numResults ?? DEFAULT_NUM_RESULTS)))) : undefined;
     const body = {
       query: input.query,
       numResults: input.numResults ?? DEFAULT_NUM_RESULTS,
-      contents: input.contents ?? { highlights: true },
+      contents: input.contents ?? {
+        highlights: input.purpose ? { query: input.purpose } : true,
+        ...(textCharacters ? { text: { maxCharacters: textCharacters } } : {}),
+        ...(typeof input.maxAgeHours === "number" ? { maxAgeHours: input.maxAgeHours } : {}),
+      },
       includeDomains: input.includeDomains,
       excludeDomains: input.excludeDomains,
       startPublishedDate: input.startPublishedDate,
@@ -37,13 +42,18 @@ export class ExaProvider implements SearchProvider, FetchProvider {
     return {
       provider: "exa" as const,
       query: input.query,
-      results: (data.results ?? []).map((r: any, i: number) => ({
-        title: r.title,
-        url: r.url,
-        snippet: asSnippet(r.highlights ?? r.summary ?? r.text),
-        siteName: r.author ?? r.publishedDate,
-        position: i + 1,
-      })).filter((r: any) => r.url),
+      results: (data.results ?? []).map((r: any, i: number) => {
+        const content = asText(r.text ?? r.highlights ?? r.summary);
+        return {
+          title: r.title,
+          url: r.url,
+          snippet: asSnippet(content),
+          content,
+          contentFormat: content ? "markdown" as const : undefined,
+          siteName: r.author ?? r.publishedDate,
+          position: i + 1,
+        };
+      }).filter((r: any) => r.url),
     };
   }
 
@@ -53,7 +63,14 @@ export class ExaProvider implements SearchProvider, FetchProvider {
     const data = await requestJson<any>("https://api.exa.ai/contents", {
       method: "POST",
       headers: this.headers(),
-      body: JSON.stringify({ urls, text: true, highlights: false }),
+      body: JSON.stringify({
+        urls,
+        text: { maxCharacters: 100_000 },
+        highlights: false,
+        maxAgeHours: input.refresh === true
+          ? 0
+          : input.maxAgeHours ?? (typeof input.maxAgeMs === "number" ? Math.floor(input.maxAgeMs / 3_600_000) : undefined),
+      }),
       signal,
       timeoutMs: 45000,
     });
@@ -61,7 +78,7 @@ export class ExaProvider implements SearchProvider, FetchProvider {
     const primary: WebFetchResult = { provider: "exa", results: urls.map((url, i) => {
       const r: any = list.find((item: any) => urlsMatch(item.url, url)) ?? list[i];
       if (!r) return { url, error: "No content returned by Exa contents endpoint." };
-      return { url: r.url ?? url, title: r.title, content: r.text ?? r.summary ?? "", format: "markdown" as const, metadata: r };
+      return { url: r.url ?? url, title: r.title, content: r.text ?? r.summary ?? "", format: "markdown" as const, metadata: withoutContent(r) };
     }) };
     return applyExaFetchFallbacks(this.config, input, urls, primary, signal);
   }

package/src/providers/firecrawl.ts

@@ -1,4 +1,4 @@
-import { asSnippet, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
+import { asSnippet, asText, mapConcurrent, normalizeUrls, requestJson } from "../http.js";
 import { DEFAULT_NUM_RESULTS, FETCH_CONCURRENCY } from "../limits.js";
 import type { FetchInput, FetchProvider, SearchInput, SearchProvider, WebKitConfig } from "../types.js";
 import { requireKey } from "../config.js";
@@ -20,13 +20,22 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
         excludeDomains: input.excludeDomains,
         categories: input.categories,
         tbs: input.tbs,
-        scrapeOptions: input.scrapeOptions ?? (input.scrape === true ? { formats: ["markdown"] } : undefined),
+        scrapeOptions: input.scrapeOptions ?? (input.scrape === true || input.contextTokens ? {
+          formats: ["markdown"],
+          onlyMainContent: true,
+          maxAge: input.maxAge,
+        } : undefined),
       }),
     });
     const list = data.data?.web ?? data.web ?? data.data ?? [];
-    return { provider: "firecrawl" as const, query: input.query, results: list.map((r: any, i: number) => ({
-      title: r.title, url: r.url, snippet: asSnippet(r.description ?? r.markdown ?? r.content), siteName: r.siteName, position: i + 1,
-    })).filter((r: any) => r.url) };
+    return { provider: "firecrawl" as const, query: input.query, results: list.map((r: any, i: number) => {
+      const content = asText(r.markdown ?? r.content ?? r.highlights);
+      return {
+        title: r.title, url: r.url, snippet: asSnippet(r.highlights ?? r.description ?? content), content,
+        contentFormat: content ? (r.markdown ? "markdown" as const : "text" as const) : undefined,
+        siteName: r.siteName, position: i + 1,
+      };
+    }).filter((r: any) => r.url) };
   }
 
   async fetch(input: FetchInput, signal?: AbortSignal) {
@@ -44,7 +53,7 @@ export class FirecrawlProvider implements SearchProvider, FetchProvider {
             waitFor: input.waitFor,
             mobile: input.mobile,
             location: input.location,
-            maxAge: input.maxAge,
+            maxAge: input.refresh === true ? 0 : input.maxAge ?? input.maxAgeMs,
           }),
         });
         const d = data.data ?? data;

package/src/providers/index.ts

@@ -2,7 +2,6 @@ import { validateFetchProvider, validateSearchProvider } from "../config.js";
 import type { FetchProvider, SearchProvider, WebKitConfig } from "../types.js";
 import { BraveProvider } from "./brave.js";
 import { Context7Provider } from "./context7.js";
-import { ExaMcpProvider } from "./exa-mcp.js";
 import { ExaProvider } from "./exa.js";
 import { FirecrawlProvider } from "./firecrawl.js";
 import { MarkdownNewProvider } from "./markdown-new.js";
@@ -11,7 +10,6 @@ import { TinyFishProvider } from "./tinyfish.js";
 export function createSearchProvider(config: WebKitConfig): SearchProvider {
   validateSearchProvider(config.provider_search);
   switch (config.provider_search) {
-    case "exa_mcp": return new ExaMcpProvider(config);
     case "exa": return new ExaProvider(config);
     case "tinyfish": return new TinyFishProvider(config);
     case "brave": return new BraveProvider(config);
@@ -22,7 +20,6 @@ export function createSearchProvider(config: WebKitConfig): SearchProvider {
 export function createFetchProvider(config: WebKitConfig): FetchProvider {
   validateFetchProvider(config.provider_fetch);
   switch (config.provider_fetch) {
-    case "exa_mcp": return new ExaMcpProvider(config);
     case "exa": return new ExaProvider(config);
     case "tinyfish": return new TinyFishProvider(config);
     case "markdown_new": return new MarkdownNewProvider(config);

package/src/providers/markdown-new.ts

@@ -15,14 +15,22 @@ export class MarkdownNewProvider implements FetchProvider {
           body: JSON.stringify({
             url,
             method: input.method ?? this.config.markdownNew.method,
-            retainImages: input.retainImages ?? this.config.markdownNew.retainImages,
+            retain_images: input.retainImages ?? this.config.markdownNew.retainImages,
           }),
           signal,
           timeoutMs: 45_000,
         });
         const text = await res.text();
-        if (!res.ok) throw new Error(`${res.status} ${res.statusText}: ${text.slice(0, 1000)}`);
-        return { url, content: text, format: "markdown" as const };
+        if (!res.ok) throw new Error(`Provider request failed (HTTP ${res.status}).`);
+        return {
+          url,
+          content: text,
+          format: "markdown" as const,
+          metadata: {
+            estimatedTokens: numberHeader(res.headers.get("x-markdown-tokens")),
+            conversionMethod: res.headers.get("x-markdown-method") ?? undefined,
+          },
+        };
       } catch (e) {
         return { url, error: e instanceof Error ? e.message : String(e) };
       }
@@ -30,3 +38,9 @@ export class MarkdownNewProvider implements FetchProvider {
     return { provider: "markdown_new" as const, results };
   }
 }
+
+function numberHeader(value: string | null): number | undefined {
+  if (value == null) return undefined;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}