webfox 4.2.2 → 4.4.0

package/README.md

@@ -25,11 +25,25 @@ configuration in your terminal, TypeScript applications, and pi.
 
 ## 🚀 Installation
 
-- **Terminal:** Install the `web` command:
+- **Terminal:** Install the `web` command with [Homebrew](https://github.com/mavam/homebrew-tap):
+
+  ```sh
+  brew install mavam/tap/webfox
+  ```
+
+  Or use npm:
 
   ```sh
   npm install -g webfox
   ```
+- **Nix:** Run the CLI directly:
+
+  ```sh
+  nix run github:mavam/webfox -- --help
+  ```
+
+  The flake exports `packages.<system>.default` and `overlays.default` for Linux
+  (`aarch64`, `x86_64`) and macOS (Apple Silicon).
 - **TypeScript:** Add `webfox` as an application dependency.
 - **Pi:** Install the extension:
 
@@ -37,6 +51,7 @@ configuration in your terminal, TypeScript applications, and pi.
   pi install npm:webfox
   ```
 
+  This installs the extension, not the `web` command on your shell's `PATH`.
   Then follow [Use with Pi](#-use-with-pi) to select a web provider and supply its
   API key.
 
@@ -134,6 +149,7 @@ Use different providers for different tasks:
 | [OpenAI]     |   ✔︎    |          |   ✔︎    |    ✔︎     |
 | [Parallel]   |   ✔︎    |    ✔︎     |        |          |
 | [Perplexity] |   ✔︎    |          |   ✔︎    |    ✔︎     |
+| [SerpBase]   |   ✔︎    |          |        |          |
 | [Serper]     |   ✔︎    |          |        |          |
 | [Tavily]     |   ✔︎    |    ✔︎     |        |          |
 | [Valyu]      |   ✔︎    |    ✔︎     |   ✔︎    |    ✔︎     |
@@ -150,6 +166,7 @@ Use different providers for different tasks:
 [OpenAI]: ./docs/provider.md#openai
 [Parallel]: ./docs/provider.md#parallel
 [Perplexity]: ./docs/provider.md#perplexity
+[SerpBase]: ./docs/provider.md#serpbase
 [Serper]: ./docs/provider.md#serper
 [Tavily]: ./docs/provider.md#tavily
 [Valyu]: ./docs/provider.md#valyu
@@ -204,6 +221,9 @@ Install the [Pi](https://pi.dev) extension:
 pi install npm:webfox
 ```
 
+To also use the `web` command in your terminal, follow [Installation](#-installation).
+Both installations share the same Webfox configuration.
+
 Add to `~/.config/webfox/config.yaml`:
 
 ```yaml

package/dist/adapter-RL7JNB22.js

@@ -0,0 +1,183 @@
+import {
+  SERPBASE_MODES,
+  serpbaseProvider,
+  validateOptions
+} from "./chunk-VYMIR35R.js";
+import "./chunk-GPACLYNT.js";
+import {
+  trimSnippet
+} from "./chunk-X6ZBIHZC.js";
+import {
+  WebfoxError,
+  httpError
+} from "./chunk-U4BLULLV.js";
+
+// src/providers/serpbase/adapter.ts
+var DEFAULT_BASE_URL = "https://api.serpbase.dev";
+var RETRYABLE_STATUSES = /* @__PURE__ */ new Set([1029, 1500, 1502, 1503, 1504]);
+var FEATURE_ID = /^0x[0-9a-f]+:0x[0-9a-f]+$/i;
+var adapter = {
+  async search(request, config, context) {
+    const options = {
+      ...request.options,
+      mode: request.options?.mode ?? "search"
+    };
+    validateOptions(serpbaseProvider, "search", options);
+    const mode = options.mode;
+    const endpoint = SERPBASE_MODES[mode];
+    if (mode === "maps-detail" && !FEATURE_ID.test(request.query)) {
+      throw new WebfoxError(
+        "INVALID_INPUT",
+        "SerpBase maps-detail requires a feature_id (0x...:0x...) in each queries entry. First use mode maps and copy the returned feature_id; names, place_id, CID, and Maps URLs are not accepted."
+      );
+    }
+    const apiKey = config.credentials?.api;
+    if (!apiKey) throw new Error("SerpBase search is missing an API key");
+    const response = await fetch(
+      `${(config.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "")}/google/${endpoint.path}`,
+      {
+        method: "POST",
+        headers: { "content-type": "application/json", "X-API-Key": apiKey },
+        body: JSON.stringify({
+          ...mode === "maps-detail" ? { feature_id: request.query } : { q: request.query },
+          ...Object.fromEntries(
+            endpoint.options.flatMap(
+              (key) => options[key] === void 0 ? [] : [[key, options[key]]]
+            )
+          )
+        }),
+        signal: context.signal
+      }
+    );
+    const body = await response.text();
+    let payload;
+    try {
+      payload = asRecord(JSON.parse(body));
+    } catch {
+    }
+    const status = payload?.status;
+    const apiFailure = typeof status === "number" && status !== 0;
+    if (!response.ok || apiFailure) {
+      const detail = (text(payload?.error) ?? body.trim()).slice(0, 500);
+      const message = `SerpBase ${mode} request failed (HTTP ${response.status}${apiFailure ? `, status ${status}` : ""})${detail ? `: ${detail}` : "."}`;
+      if (apiFailure) {
+        throw new WebfoxError("PROVIDER_FAILURE", message, {
+          retryable: RETRYABLE_STATUSES.has(status)
+        });
+      }
+      throw httpError(response, message);
+    }
+    const collection = payload?.[endpoint.result];
+    if (!payload || status !== 0 || collection !== void 0 && (mode === "maps-detail" ? !asRecord(collection) : !Array.isArray(collection))) {
+      throw new WebfoxError(
+        "PROVIDER_FAILURE",
+        `SerpBase returned an invalid ${mode} response.`,
+        { retryable: false }
+      );
+    }
+    const searchContext = { ...payload };
+    for (const { result } of Object.values(SERPBASE_MODES))
+      delete searchContext[result];
+    const entries = mode === "maps-detail" ? collection === void 0 ? [] : [collection] : collection ?? [];
+    const results = entries.map((entry) => toSearchResult(entry, searchContext, mode)).filter((result) => result !== void 0);
+    return {
+      provider: "serpbase",
+      // No upstream count parameter or automatic pagination: one page or place.
+      results: results.slice(
+        0,
+        Math.max(1, Math.trunc(request.maxResults || 0))
+      )
+    };
+  }
+};
+function toSearchResult(entry, searchContext, mode) {
+  const record = asRecord(entry);
+  if (!record || mode === "images" && isNavigationImage(record))
+    return void 0;
+  const isPlace = mode === "maps" || mode === "maps-detail";
+  const featureId = text(record.feature_id);
+  const url = isPlace ? text(record.google_maps_url) ?? text(record.url) ?? text(record.website) ?? (featureId && FEATURE_ID.test(featureId) ? `https://www.google.com/maps?ftid=${encodeURIComponent(featureId)}` : void 0) : text(record.link) ?? text(record.url) ?? (mode === "images" ? text(record.image_url) : void 0);
+  if (!url) return void 0;
+  const { title, link: _link, url: _url, snippet, ...metadata } = record;
+  return {
+    title: text(title) ?? text(record.name) ?? url,
+    url,
+    snippet: resultSnippet(record, mode),
+    metadata: { ...metadata, mode, searchContext }
+  };
+}
+function resultSnippet(record, mode) {
+  const lines = [];
+  const excerpt = text(record.snippet) ?? text(record.short_description) ?? text(record.description);
+  if (excerpt && !(mode === "videos" && /^\s*Duration:\s*Posted:\s*$/i.test(excerpt)))
+    lines.push(trimSnippet(excerpt));
+  const add = (label, value, limit = 600) => {
+    if (typeof value === "number" && Number.isFinite(value))
+      value = String(value);
+    if (text(value))
+      lines.push(`${label}: ${trimSnippet(value, limit)}`);
+  };
+  const json = (label, value) => {
+    if (value && typeof value === "object")
+      add(label, JSON.stringify(value), 1200);
+  };
+  if (mode === "images") {
+    add("Image URL", record.image_url, Infinity);
+    add("Source page", text(record.link) ?? text(record.url), Infinity);
+    add(
+      "Thumbnail",
+      text(record.thumbnail_url) ?? text(record.thumbnail),
+      Infinity
+    );
+    add("Source", text(record.source) ?? text(record.domain));
+  } else if (mode === "news" || mode === "videos") {
+    add("Source", record.source);
+    const published = text(record.published_at) ?? text(record.time);
+    add(
+      published && /^\d{1,3}:\d{2}(?::\d{2})?$/.test(published.trim()) ? "Time (upstream)" : "Published",
+      published
+    );
+    if (mode === "videos") add("Duration", record.duration);
+    add(
+      "Thumbnail",
+      text(record.thumbnail_url) ?? text(record.thumbnail),
+      Infinity
+    );
+  } else if (mode === "maps" || mode === "maps-detail") {
+    add("feature_id", record.feature_id, Infinity);
+    add("Address", record.address);
+    add("Rating", record.rating);
+    add("Reviews", record.review_count);
+    add("Phone", text(record.phone_international) ?? text(record.phone));
+    add("Website", record.website, Infinity);
+    if (typeof record.latitude === "number" && typeof record.longitude === "number")
+      add("Coordinates", `${record.latitude}, ${record.longitude}`);
+    if (Array.isArray(record.types))
+      add(
+        "Categories",
+        record.types.filter((item) => typeof item === "string").join(", ")
+      );
+    json("Hours", record.hours);
+    json("Open status", record.open_status);
+    if (mode === "maps-detail") json("Attributes", record.attributes);
+  }
+  return lines.join("\n");
+}
+function isNavigationImage(record) {
+  try {
+    const image = new URL(String(record.image_url));
+    const source = new URL(String(record.link ?? record.url));
+    return image.hostname === "www.gstatic.com" && image.pathname === "/m/images/icons/googleg.gif" && (source.hostname === "www.google.com" || source.hostname === "google.com") && (source.pathname === "/" || source.pathname === "/search");
+  } catch {
+    return false;
+  }
+}
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : void 0;
+}
+function text(value) {
+  return typeof value === "string" && value.trim() ? value : void 0;
+}
+export {
+  adapter
+};

package/dist/{adapter-A7LV6L55.js → adapter-VW4FMMJC.js}

@@ -4,7 +4,7 @@ import {
 } from "./chunk-ZVTVERW5.js";
 import {
   WEBFOX_ERROR_CODES
-} from "./chunk-WVHMWLKU.js";
+} from "./chunk-GPACLYNT.js";
 import {
   WebfoxError
 } from "./chunk-U4BLULLV.js";

package/dist/application-types.d.ts

@@ -18,6 +18,7 @@ export interface CapabilityInspection {
     provider?: ProviderId;
     configured: boolean;
     optionSchema?: Record<string, unknown>;
+    promptGuidelines?: readonly string[];
     defaults: {
         maxResults?: number;
         options: Record<string, unknown>;

package/dist/{chunk-ZBD4DRCY.js → chunk-7KCSWOBG.js}

@@ -5,20 +5,22 @@ import {
   withSignal
 } from "./chunk-WCTOYZVB.js";
 import {
-  configurationSchema,
+  configurationSchema
+} from "./chunk-VGVUGOAV.js";
+import {
   effectiveOptions,
   optionSchema,
   providers,
   selectProvider,
   validateConfiguredOptions
-} from "./chunk-FMYKJ7JC.js";
+} from "./chunk-VYMIR35R.js";
 import {
   cancelProcessGroup,
   killProcessGroup
 } from "./chunk-ZVTVERW5.js";
 import {
   CAPABILITIES
-} from "./chunk-WVHMWLKU.js";
+} from "./chunk-GPACLYNT.js";
 import {
   WebfoxError,
   asWebfoxError
@@ -361,7 +363,7 @@ var ExecutionRuntime = class {
   credentials;
   async execute(plan, request) {
     const { capability, definition, inputs } = plan;
-    const timeoutMs = request.timeoutMs ?? (capability === "research" ? plan.policy.researchTimeoutMs ?? 18e5 : plan.policy.timeoutMs ?? 3e4);
+    const timeoutMs = request.timeoutMs ?? (capability === "research" ? plan.policy.researchTimeoutMs : plan.policy.timeoutMs) ?? definition.capabilities[capability]?.defaultTimeoutMs ?? (capability === "research" ? 18e5 : 3e4);
     if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2147483647)
       throw new WebfoxError(
         "INVALID_INPUT",
@@ -689,6 +691,11 @@ function createWebfox(options = {}) {
         capability,
         provider: selected,
         configured: configured(definition, capability),
+        ...definition.capabilities[capability]?.promptGuidelines ? {
+          promptGuidelines: [
+            ...definition.capabilities[capability].promptGuidelines
+          ]
+        } : {},
         // Static provider schemas contain no credentials. Redacting property
         // names such as maximum_number_of_tokens would invalidate the schema.
         optionSchema: optionSchema(

package/dist/{chunk-QRHDXM2L.js → chunk-BTWECET5.js}

@@ -9,7 +9,7 @@ function renderTextDocument(document, { includeErrors = true } = {}) {
       body = entry.value.results.length ? entry.value.results.map(
         (result, i) => `${i + 1}. ${result.title}
    ${result.url}
-   ${result.snippet}`
+   ${result.snippet.replaceAll("\n", "\n   ")}`
       ).join("\n\n") : "No results found.";
     else if ("text" in entry.value) body = entry.value.text;
     else body = renderContents(entry.value);

package/dist/{chunk-WVHMWLKU.js → chunk-GPACLYNT.js}

@@ -17,6 +17,7 @@ var PROVIDER_IDS = [
   "openai",
   "parallel",
   "perplexity",
+  "serpbase",
   "serper",
   "tavily",
   "valyu",

package/dist/chunk-VGVUGOAV.js

@@ -0,0 +1,244 @@
+import {
+  optionSchema,
+  providers
+} from "./chunk-VYMIR35R.js";
+import {
+  CAPABILITIES
+} from "./chunk-GPACLYNT.js";
+
+// package.json
+var package_default = {
+  name: "webfox",
+  version: "4.4.0",
+  description: "Search the web, extract pages, get grounded answers, and run deep research with the web CLI, a TypeScript library, and a pi extension. Bring your own providers and API keys.",
+  type: "module",
+  files: [
+    "dist",
+    "README.md",
+    "assets/webfox-lockup.png",
+    "assets/webfox-lockup-dark.png",
+    "docs/cli-experience.md",
+    "docs/provider.md",
+    "docs/reference.md",
+    "LICENSE",
+    "example-config.yaml",
+    "examples"
+  ],
+  keywords: [
+    "webfox",
+    "web-cli",
+    "typescript-library",
+    "pi-package",
+    "pi-extension",
+    "coding-agent",
+    "web-search",
+    "web-contents",
+    "web-answer",
+    "web-research",
+    "provider-routing",
+    "brave",
+    "cloudflare",
+    "custom",
+    "exa",
+    "firecrawl",
+    "gemini",
+    "linkup",
+    "ollama",
+    "openai",
+    "parallel",
+    "perplexity",
+    "serper",
+    "tavily",
+    "valyu"
+  ],
+  author: "mavam",
+  license: "MIT",
+  repository: {
+    type: "git",
+    url: "git+https://github.com/mavam/webfox.git"
+  },
+  engines: {
+    node: ">=22.0.0"
+  },
+  publishConfig: {
+    access: "public"
+  },
+  pi: {
+    extensions: [
+      "./dist/pi.js"
+    ]
+  },
+  bin: {
+    web: "./dist/cli.js"
+  },
+  main: "./dist/index.js",
+  types: "./dist/index.d.ts",
+  exports: {
+    ".": {
+      types: "./dist/index.d.ts",
+      import: "./dist/index.js"
+    },
+    "./pi": {
+      types: "./dist/pi.d.ts",
+      import: "./dist/pi.js"
+    },
+    "./config.schema.json": "./dist/config.schema.json"
+  },
+  scripts: {
+    build: "rm -rf dist && esbuild index=src/index.ts cli=src/cli.ts pi=src/pi.ts generate-schema=scripts/generate-schema.ts --bundle --splitting --format=esm --platform=node --target=node22 --packages=external --outdir=dist && node dist/generate-schema.js && biome format --write src/config.schema.json >&2 && rm dist/generate-schema.js && tsc --project tsconfig.build.json && cp src/config.schema.json dist/config.schema.json && chmod +x dist/cli.js",
+    prepare: "npm run build",
+    prepack: "npm run build",
+    check: "tsc --noEmit",
+    format: "biome format --write .",
+    "format:check": "biome format .",
+    "smoke:live": "npm run build && node scripts/live-smoke.mjs",
+    "smoke:package": "node scripts/package-smoke.mjs",
+    test: "vitest run",
+    "test:watch": "vitest"
+  },
+  dependencies: {
+    "@mendable/firecrawl-js": "^4.38.0",
+    "@perplexity-ai/perplexity_ai": "^0.38.5",
+    "@tavily/core": "^0.7.9",
+    cloudflare: "^7.1.0",
+    commander: "^14.0.3",
+    "exa-js": "^2.19.0",
+    "linkup-sdk": "^3.6.0",
+    openai: "^7.10.0",
+    "parallel-web": "^1.3.3",
+    picocolors: "^1.1.1",
+    typebox: "^1.3.6",
+    "valyu-js": "^2.10.1",
+    yaml: "^2.9.0"
+  },
+  devDependencies: {
+    "@biomejs/biome": "^2.5.4",
+    "@earendil-works/pi-ai": "^0.85.1",
+    "@earendil-works/pi-coding-agent": "^0.85.1",
+    "@earendil-works/pi-tui": "^0.85.1",
+    "@types/node": "^26.1.1",
+    esbuild: "^0.28.1",
+    typescript: "^7.0.2",
+    vitest: "^4.1.10"
+  },
+  peerDependencies: {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*"
+  },
+  peerDependenciesMeta: {
+    "@earendil-works/pi-coding-agent": {
+      optional: true
+    },
+    "@earendil-works/pi-tui": {
+      optional: true
+    }
+  }
+};
+
+// src/package-metadata.ts
+var PACKAGE_NAME = package_default.name;
+var PACKAGE_VERSION = package_default.version;
+var CONFIG_SCHEMA_URL = `https://unpkg.com/${PACKAGE_NAME}@latest/dist/config.schema.json`;
+
+// src/configuration/schema.ts
+var text = { type: "string", minLength: 1 };
+var object = (properties) => ({
+  type: "object",
+  additionalProperties: false,
+  properties
+});
+var source = {
+  oneOf: ["env", "value", "command"].map((key) => ({
+    ...object({
+      [key]: key === "command" ? { type: "array", minItems: 1, items: text } : text
+    }),
+    required: [key]
+  }))
+};
+var sourceMap = { type: "object", additionalProperties: source };
+var command = {
+  ...object({
+    argv: { type: "array", minItems: 1, items: text },
+    cwd: text,
+    env: sourceMap
+  }),
+  required: ["argv"]
+};
+var positive = { type: "integer", minimum: 1 };
+var timeout = { ...positive, maximum: 2147483647 };
+var fields = {
+  credentials: sourceMap,
+  baseUrl: text,
+  accountId: source,
+  commands: object(
+    Object.fromEntries(CAPABILITIES.map((capability) => [capability, command]))
+  )
+};
+var configurationSchema = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: CONFIG_SCHEMA_URL,
+  title: "webfox configuration",
+  ...object({
+    $schema: text,
+    defaults: object(
+      Object.fromEntries(
+        CAPABILITIES.map((capability) => [
+          capability,
+          object({
+            provider: {
+              enum: Object.values(providers).filter((provider) => provider.capabilities[capability]).map((provider) => provider.id)
+            },
+            ...capability === "search" ? { maxResults: positive } : {}
+          })
+        ])
+      )
+    ),
+    execution: object({
+      timeoutMs: timeout,
+      researchTimeoutMs: timeout,
+      retries: { type: "integer", minimum: 0 },
+      retryDelayMs: { type: "integer", minimum: 0 },
+      concurrency: positive
+    }),
+    providers: object(
+      Object.fromEntries(
+        Object.values(providers).map((provider) => [
+          provider.id,
+          object(
+            Object.fromEntries(
+              provider.fields.map((field) => [
+                field,
+                field === "options" ? object(
+                  Object.fromEntries(
+                    CAPABILITIES.filter(
+                      (capability) => provider.capabilities[capability]
+                    ).map((capability) => [
+                      capability,
+                      optionSchema(provider, capability, "defaults") ?? {
+                        type: "object",
+                        additionalProperties: provider.id === "custom"
+                      }
+                    ])
+                  )
+                ) : field === "credentials" ? object(
+                  Object.fromEntries(
+                    provider.credentials.map((credential) => [
+                      credential.name,
+                      source
+                    ])
+                  )
+                ) : fields[field]
+              ])
+            )
+          )
+        ])
+      )
+    )
+  })
+};
+
+export {
+  PACKAGE_VERSION,
+  CONFIG_SCHEMA_URL,
+  configurationSchema
+};