webveil 0.0.0 → 0.1.1

package/dist/core/security.js

@@ -0,0 +1,141 @@
+// SSRF guard: the security seam wrapped AROUND the egress-bound `fetch`, so it
+// covers BOTH webveil's own GETs AND distilly's rule-rewritten requests (see
+// docs/adr/0001: the guard lives inside the egress fetch). Adapts the range
+// classification + DNS-resolve approach of leing2021/pi-search's `security.ts`.
+//
+// THE RELAXATION RULE (load-bearing, recorded in the task's Decisions): the
+// guard BLOCKS private/loopback/link-local/etc. addresses on DIRECT egress, and
+// RELAXES ENTIRELY under a proxy egress (`http` | `socks5`). Tor/Mullvad
+// legitimately reach private-looking addresses (e.g. `10.64.0.1`), AND a local
+// DNS lookup for a proxied request would itself be a deanonymizing leak, so
+// under a proxy we neither block nor resolve locally; the proxy owns egress.
+import { lookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+/** Thrown when the SSRF guard refuses a request to a private/blocked address. */
+export class SsrfError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SsrfError';
+    }
+}
+/** A proxy egress owns egress + DNS, so the local SSRF guard relaxes for it. */
+function egressIsProxy(config) {
+    return config.egress.mode === 'http' || config.egress.mode === 'socks5';
+}
+/**
+ * Is this LITERAL IP private / non-public? Covers the ranges that must never be
+ * reachable from a direct-egress web fetch:
+ *   IPv4: 0.0.0.0/8, 10/8 (RFC1918), 127/8 (loopback), 169.254/16 (link-local,
+ *     incl. the 169.254.169.254 cloud metadata endpoint), 172.16/12 (RFC1918),
+ *     192.168/16 (RFC1918), 100.64/10 (CGNAT), 192.0.0/24, 192.0.2/24,
+ *     198.18/15, 198.51.100/24, 203.0.113/24, 224/4 (multicast), 240/4
+ *     (reserved).
+ *   IPv6: ::1 (loopback), :: (unspecified), fc00::/7 (ULA), fe80::/10
+ *     (link-local), ff00::/8 (multicast), plus IPv4-mapped (::ffff:a.b.c.d,
+ *     re-checked as IPv4). Default-deny: anything outside global unicast
+ *     (2000::/3) is treated as non-public.
+ */
+export function isPrivateIp(ip) {
+    const kind = isIP(ip);
+    if (kind === 4)
+        return isPrivateIpv4(ip);
+    if (kind === 6)
+        return isPrivateIpv6(ip);
+    return false; // not a literal IP; hostname handling resolves it first
+}
+function isPrivateIpv4(ip) {
+    const parts = ip.split('.').map((p) => Number(p));
+    if (parts.length !== 4 ||
+        parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255))
+        return true; // malformed → treat as non-public (fail closed)
+    const [a, b] = parts;
+    if (a === 0 || a === 10 || a === 127)
+        return true;
+    if (a === 169 && b === 254)
+        return true; // link-local incl. cloud metadata
+    if (a === 172 && b >= 16 && b <= 31)
+        return true; // 172.16/12
+    if (a === 192 && b === 168)
+        return true; // 192.168/16
+    if (a === 100 && b >= 64 && b <= 127)
+        return true; // 100.64/10 CGNAT
+    if (a === 192 && b === 0)
+        return true; // 192.0.0/24 + 192.0.2/24
+    if (a === 198 && (b === 18 || b === 19))
+        return true; // 198.18/15 benchmark
+    if (a === 198 && b === 51)
+        return true; // 198.51.100/24 TEST-NET-2
+    if (a === 203 && b === 0)
+        return true; // 203.0.113/24 TEST-NET-3
+    if (a >= 224)
+        return true; // 224/4 multicast + 240/4 reserved
+    return false;
+}
+function isPrivateIpv6(ip) {
+    const lower = ip.toLowerCase();
+    if (lower === '::1' || lower === '::')
+        return true; // loopback / unspecified
+    // IPv4-mapped (::ffff:a.b.c.d): re-check the embedded IPv4.
+    const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+    if (mapped)
+        return isPrivateIpv4(mapped[1]);
+    const head = lower.split(':')[0] ?? '';
+    const first = parseInt(head || '0', 16);
+    if (Number.isNaN(first))
+        return true; // fail closed on anything unparseable
+    if ((first & 0xfe00) === 0xfc00)
+        return true; // fc00::/7 ULA
+    if ((first & 0xffc0) === 0xfe80)
+        return true; // fe80::/10 link-local
+    if ((first & 0xff00) === 0xff00)
+        return true; // ff00::/8 multicast
+    // Default-deny: only global unicast 2000::/3 is public.
+    return (first & 0xe000) !== 0x2000;
+}
+/**
+ * Assert a URL is safe to fetch under THIS config's egress. Under a proxy egress
+ * it always passes (the proxy owns egress + DNS). Under direct egress it rejects
+ * a literal private IP and, for a hostname, resolves it locally and rejects if it
+ * maps to a private IP (so a name pointing at 127.0.0.1 / metadata is caught).
+ */
+export async function assertPublicUrl(url, config) {
+    if (egressIsProxy(config))
+        return; // proxy owns egress + DNS; relax entirely
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new SsrfError(`webveil SSRF: malformed url ${url}`);
+    }
+    const host = parsed.hostname.replace(/^\[|\]$/g, ''); // strip IPv6 brackets
+    if (isIP(host)) {
+        if (isPrivateIp(host))
+            throw new SsrfError(`webveil SSRF: blocked private address ${host}`);
+        return;
+    }
+    // A hostname: resolve it locally (safe on direct egress) and check every
+    // address it maps to, so a name pointing at a private IP is also blocked.
+    const addrs = await lookup(host, { all: true });
+    for (const { address } of addrs)
+        if (isPrivateIp(address))
+            throw new SsrfError(`webveil SSRF: ${host} resolves to private address ${address}`);
+}
+/**
+ * Wrap an egress-bound `fetch` with the SSRF guard. The returned fetch checks
+ * EVERY request URL (so it covers distilly's rule-rewritten requests too, not
+ * only webveil's own GET) before delegating to the underlying egress fetch.
+ * This is what `core.fetch()` injects into distilly. See docs/adr/0001.
+ */
+export function guardEgressFetch(fetch, config) {
+    return (async (input, init) => {
+        const url = typeof input === 'string'
+            ? input
+            : input instanceof URL
+                ? input.href
+                : input.url;
+        await assertPublicUrl(url, config);
+        return fetch(input, init);
+    });
+}
+//# sourceMappingURL=security.js.map

package/dist/core/security.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.js","sourceRoot":"","sources":["../../src/core/security.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,6EAA6E;AAC7E,4EAA4E;AAC5E,gFAAgF;AAChF,EAAE;AACF,4EAA4E;AAC5E,gFAAgF;AAChF,yEAAyE;AACzE,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAE7E,OAAO,EAAC,MAAM,EAAC,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAC,IAAI,EAAC,MAAM,UAAU,CAAC;AAI9B,iFAAiF;AACjF,MAAM,OAAO,SAAU,SAAQ,KAAK;IACnC,YAAY,OAAe;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IACzB,CAAC;CACD;AAED,gFAAgF;AAChF,SAAS,aAAa,CAAC,MAAc;IACpC,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,CAAC,wDAAwD;AACvE,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAChC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,IACC,KAAK,CAAC,MAAM,KAAK,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QAE3D,OAAO,IAAI,CAAC,CAAC,gDAAgD;IAC9D,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAyC,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAClD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,kCAAkC;IAC3E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,YAAY;IAC9D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,aAAa;IACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,kBAAkB;IACrE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IACjE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sBAAsB;IAC5E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,2BAA2B;IACnE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IACjE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,mCAAmC;IAC9D,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAChC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/B,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;IAC7E,4DAA4D;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC5D,IAAI,MAAM;QAAE,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sCAAsC;IAC5E,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,eAAe;IAC7D,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,uBAAuB;IACrE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IACnE,wDAAwD;IACxD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,GAAW,EACX,MAAc;IAEd,IAAI,aAAa,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,0CAA0C;IAC7E,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,SAAS,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB;IAC5E,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAChB,IAAI,WAAW,CAAC,IAAI,CAAC;YACpB,MAAM,IAAI,SAAS,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;QACtE,OAAO;IACR,CAAC;IACD,yEAAyE;IACzE,0EAA0E;IAC1E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAC,GAAG,EAAE,IAAI,EAAC,CAAC,CAAC;IAC9C,KAAK,MAAM,EAAC,OAAO,EAAC,IAAI,KAAK;QAC5B,IAAI,WAAW,CAAC,OAAO,CAAC;YACvB,MAAM,IAAI,SAAS,CAClB,iBAAiB,IAAI,gCAAgC,OAAO,EAAE,CAC9D,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC/B,KAAkB,EAClB,MAAc;IAEd,OAAO,CAAC,KAAK,EAAE,KAAwB,EAAE,IAAkB,EAAE,EAAE;QAC9D,MAAM,GAAG,GACR,OAAO,KAAK,KAAK,QAAQ;YACxB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,KAAK,YAAY,GAAG;gBACrB,CAAC,CAAC,KAAK,CAAC,IAAI;gBACZ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QACf,MAAM,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,KAAc,EAAE,IAAa,CAAC,CAAC;IAC7C,CAAC,CAAgB,CAAC;AACnB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,22 @@
+export { resolveConfig } from './core/config.js';
+export type { Config, Egress, FetchSize, PartialConfig, ResolveOptions, } from './core/config.js';
+export { buildDispatcher, createEgressFetch, EgressError, } from './core/egress.js';
+export type { Dispatcher, EgressFetch } from './core/egress.js';
+export { createHttp } from './core/http.js';
+export { extract } from './core/extract.js';
+export type { ExtractOptions, ExtractDeps } from './core/extract.js';
+export { assertPublicUrl, guardEgressFetch, isPrivateIp, SsrfError, } from './core/security.js';
+export type { Backend, Http, HttpRequestOptions, SearchResult, FetchResult, SearchOptions, FetchOptions, } from './core/backends/types.js';
+export { backendNames, getBackend } from './core/backends/registry.js';
+export type { BackendFactory } from './core/backends/registry.js';
+export { createSearxngBackend } from './core/backends/searxng.js';
+export { createTavilyCompatBackend } from './core/backends/tavily-compat.js';
+export { createCustomBackend } from './core/backends/custom.js';
+export type { SpawnFn } from './core/backends/custom.js';
+export { search } from './core/search.js';
+export type { SearchCoreOptions, SearchDeps } from './core/search.js';
+export { fetch, fetchAll } from './core/fetch.js';
+export type { FetchCoreOptions, FetchDeps } from './core/fetch.js';
+export { createCli } from './cli.js';
+export type { CliDeps } from './cli.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,YAAY,EACX,MAAM,EACN,MAAM,EACN,SAAS,EACT,aAAa,EACb,cAAc,GACd,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACN,eAAe,EACf,iBAAiB,EACjB,WAAW,GACX,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAC,UAAU,EAAE,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAG9D,OAAO,EAAC,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAG1C,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAC1C,YAAY,EAAC,cAAc,EAAE,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAGnE,OAAO,EACN,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,SAAS,GACT,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACX,OAAO,EACP,IAAI,EACJ,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,GACZ,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,6BAA6B,CAAC;AACrE,YAAY,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAC,yBAAyB,EAAC,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAC,mBAAmB,EAAC,MAAM,2BAA2B,CAAC;AAC9D,YAAY,EAAC,OAAO,EAAC,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AACxC,YAAY,EAAC,iBAAiB,EAAE,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGpE,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAChD,YAAY,EAAC,gBAAgB,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAGjE,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AACnC,YAAY,EAAC,OAAO,EAAC,MAAM,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,40 @@
+// webveil — anonymous-capable, self-hosted, account-free web search + fetch for agents.
+//
+// This is the public surface. The framework-agnostic core lives under src/core:
+//   - core/config.ts            : config seam (per-folder .pi/webveil.json + global + env)
+//   - core/egress.ts            : egress seam (direct | http | socks5/Tor) — dispatcher + egress fetch
+//   - core/http.ts              : the proxied `http` helper handed to backends
+//   - core/extract.ts           : Extractor seam (distilly/fetch + injected egress fetch)
+//   - core/backends/types.ts    : backend seam (the Backend interface + result shapes)
+//   - core/backends/registry.ts : name -> Backend dispatcher
+//   - core/backends/searxng.ts  : the keyless self-hosted SearXNG backend
+//   - core/backends/tavily-compat.ts : the generic Tavily-shaped backend (/search + /extract)
+//   - core/search.ts            : the framework-agnostic search() both frontends call
+//   - core/security.ts          : SSRF guard wrapped around the egress fetch
+//   - core/fetch.ts             : the framework-agnostic fetch() both frontends call
+//   - core/backends/custom.ts  : the local-command escape hatch (JSON stdin/stdout)
+//   - cli.ts                    : the incur CLI + MCP frontend (the `webveil` bin)
+// pi-webveil (sibling package) wraps the SAME core functions as registerTool
+// web_search / web_fetch, in-process, as an Ollama drop-in.
+// config seam
+export { resolveConfig } from './core/config.js';
+// egress seam
+export { buildDispatcher, createEgressFetch, EgressError, } from './core/egress.js';
+// http helper
+export { createHttp } from './core/http.js';
+// Extractor seam (distilly/fetch over webveil's egress)
+export { extract } from './core/extract.js';
+// SSRF guard (wrapped around the egress fetch; covers distilly's requests too)
+export { assertPublicUrl, guardEgressFetch, isPrivateIp, SsrfError, } from './core/security.js';
+// backend registry + implementations
+export { backendNames, getBackend } from './core/backends/registry.js';
+export { createSearxngBackend } from './core/backends/searxng.js';
+export { createTavilyCompatBackend } from './core/backends/tavily-compat.js';
+export { createCustomBackend } from './core/backends/custom.js';
+// core search (the framework-agnostic search() both frontends call)
+export { search } from './core/search.js';
+// core fetch (the framework-agnostic fetch() + list-ready fetchAll internal)
+export { fetch, fetchAll } from './core/fetch.js';
+// incur CLI + MCP frontend (the `webveil` bin builds and serves this)
+export { createCli } from './cli.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,wFAAwF;AACxF,EAAE;AACF,gFAAgF;AAChF,2FAA2F;AAC3F,uGAAuG;AACvG,+EAA+E;AAC/E,0FAA0F;AAC1F,uFAAuF;AACvF,6DAA6D;AAC7D,0EAA0E;AAC1E,8FAA8F;AAC9F,sFAAsF;AACtF,6EAA6E;AAC7E,qFAAqF;AACrF,oFAAoF;AACpF,mFAAmF;AACnF,6EAA6E;AAC7E,4DAA4D;AAE5D,cAAc;AACd,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAS/C,cAAc;AACd,OAAO,EACN,eAAe,EACf,iBAAiB,EACjB,WAAW,GACX,MAAM,kBAAkB,CAAC;AAG1B,cAAc;AACd,OAAO,EAAC,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAE1C,wDAAwD;AACxD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAG1C,+EAA+E;AAC/E,OAAO,EACN,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,SAAS,GACT,MAAM,oBAAoB,CAAC;AAa5B,qCAAqC;AACrC,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAErE,OAAO,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAC,yBAAyB,EAAC,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAC,mBAAmB,EAAC,MAAM,2BAA2B,CAAC;AAG9D,oEAAoE;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAGxC,6EAA6E;AAC7E,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAGhD,sEAAsE;AACtE,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC"}

package/package.json

@@ -1,4 +1,64 @@
 {
   "name": "webveil",
-  "version": "0.0.0"
-}
+  "version": "0.1.1",
+  "description": "Anonymous-capable, self-hosted, account-free web search and fetch for agents. CLI + MCP (built on incur), pi-agnostic. Swappable backend and egress (direct, http proxy, socks5/Tor).",
+  "license": "AGPL-3.0-or-later",
+  "keywords": [
+    "web-search",
+    "web-fetch",
+    "mcp",
+    "cli",
+    "agents",
+    "searxng",
+    "tor",
+    "anonymous",
+    "self-hosted"
+  ],
+  "author": "wighawag",
+  "homepage": "https://github.com/wighawag/webveil/tree/main/packages/webveil#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/wighawag/webveil.git",
+    "directory": "packages/webveil"
+  },
+  "bugs": {
+    "url": "https://github.com/wighawag/webveil/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "bin": {
+    "webveil": "./dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@modelcontextprotocol/server": "2.0.0-alpha.2",
+    "distilly": "^0.1.0",
+    "fetch-socks": "^1.3.3",
+    "incur": "0.4.10",
+    "undici": "^7.28.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.2.0",
+    "as-soon": "^0.1.5",
+    "tsx": "^4.21.0",
+    "typescript": "^5.3.3",
+    "vitest": "^4.0.18"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "as-soon -w src pnpm build",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+// webveil — the incur-based CLI + MCP frontend. ONE `Cli.create()` definition
+// yields the CLI, an MCP server (`--mcp`), skills (`skills add`), a `--llms`
+// manifest, TOON output, and token pagination for free (incur). Pi-agnostic:
+// any agent (pi via pi-mcp-adapter, Claude Code, Cursor, Codex, bash) consumes
+// it the same way. The `webveil` bin points at the built `dist/cli.js`.
+//
+// This is the THIN frontend: each command only parses argv/options and calls
+// the SAME framework-agnostic core (`search()` / `fetch()`) the pi extension
+// calls. The core owns config/egress/backend/extraction; this file owns no
+// network logic of its own.
+//
+// Testability: `createCli(deps)` takes the core functions as injectable deps so
+// a test wires fakes and asserts the commands call the core (via `cli.serve`
+// with custom argv/stdout) WITHOUT touching the network. The bottom of the file
+// builds the real CLI and serves it when run as the bin.
+
+import {argv} from 'node:process';
+import {fileURLToPath} from 'node:url';
+import {Cli, z} from 'incur';
+import {search as coreSearch} from './core/search.js';
+import {fetch as coreFetch} from './core/fetch.js';
+
+/**
+ * The two core functions the frontend wraps, seamed so tests can inject fakes.
+ * Defaults are the real core; a test passes spies to assert the wiring.
+ */
+export interface CliDeps {
+	search?: typeof coreSearch;
+	fetch?: typeof coreFetch;
+}
+
+/** The size presets `fetch` accepts, mirroring the core's `FetchSize`. */
+const SIZES = ['s', 'm', 'l', 'f'] as const;
+
+/**
+ * Build the webveil CLI. Returns the incur `Cli` so a caller (the bin below, or
+ * a test) decides how to serve it. The `search`/`fetch` commands forward to the
+ * injected core, normalizing nothing themselves — the core already deduped,
+ * clamped, and size-bounded.
+ */
+export function createCli(deps: CliDeps = {}) {
+	const search = deps.search ?? coreSearch;
+	const fetch = deps.fetch ?? coreFetch;
+
+	return Cli.create('webveil', {
+		description:
+			'Anonymous-capable, self-hosted, account-free web search + fetch for agents.',
+	})
+		.command('search', {
+			description: 'Search the web via the configured backend and egress.',
+			args: z.object({
+				query: z.string().describe('The search query'),
+			}),
+			options: z.object({
+				maxResults: z.coerce
+					.number()
+					.optional()
+					.describe('Maximum number of results to return'),
+			}),
+			alias: {maxResults: 'n'},
+			async run(c) {
+				const results = await search(c.args.query, {
+					maxResults: c.options.maxResults,
+				});
+				return {results};
+			},
+		})
+		.command('fetch', {
+			description:
+				'Fetch a URL as clean, size-bounded markdown via the configured egress.',
+			args: z.object({
+				url: z.string().describe('The URL to fetch'),
+			}),
+			options: z.object({
+				size: z
+					.enum(SIZES)
+					.optional()
+					.describe('Page-size budget preset: s | m | l | f'),
+			}),
+			alias: {size: 's'},
+			async run(c) {
+				return fetch(c.args.url, {size: c.options.size});
+			},
+		});
+}
+
+// The real CLI (also `export default` so `incur gen` can import it for typed
+// CTAs). Serving is GUARDED to the bin entry below, so importing this module in
+// a test never consumes `process.argv` or exits the process.
+const cli = createCli();
+
+/** True when this module is the process entry (the `webveil` bin), not imported. */
+function isMain(): boolean {
+	const entry = argv[1];
+	if (!entry) return false;
+	try {
+		return fileURLToPath(import.meta.url) === entry;
+	} catch {
+		return false;
+	}
+}
+
+if (isMain()) cli.serve();
+
+export default cli;

package/src/core/backends/custom.ts

@@ -0,0 +1,159 @@
+// custom backend — the local-command escape hatch (contract lifted from
+// pi-web-providers' custom-wrapper). Instead of an HTTP source, it spawns a
+// configured local command, writes the request as JSON to its stdin, and parses
+// `SearchResult[]` from its stdout. This lets any local script be a backend.
+//
+// Egress note: this backend owns its own I/O (the spawned command does whatever
+// it wants), so the handed `http` helper is unused here — there is no outbound
+// HTTP for webveil to proxy. It still returns the normalized SearchResult shape.
+//
+// Command source: the configured `baseUrl` carries the command line, parsed as a
+// whitespace-separated argv (first token = executable, rest = args), matching how
+// the other backends read `baseUrl` as "where results come from". (Recorded
+// decision; see the task's Decisions block.)
+//
+// Contract:
+//   stdin  <- JSON: {"query": string, "maxResults"?: number}
+//   stdout -> JSON: SearchResult[]  (each {title, url, snippet?})
+// Malformed stdout (non-JSON, not an array, or entries missing url/title) FAILS
+// CLEARLY — it never silently returns an empty list.
+
+import {spawn as defaultSpawn} from 'node:child_process';
+import type {Config} from '../config.js';
+import type {Backend, Http, SearchOptions, SearchResult} from './types.js';
+
+/** The JSON request written to the command's stdin. */
+interface CustomRequest {
+	query: string;
+	maxResults?: number;
+}
+
+/** The result of running the command: its stdout text (and exit status). */
+interface CommandRun {
+	stdout: string;
+	stderr: string;
+	code: number | null;
+}
+
+/**
+ * Minimal `spawn` shape this backend needs, seamed so a test can inject a fake
+ * without a real subprocess. Defaults to `node:child_process` `spawn`.
+ */
+export type SpawnFn = typeof defaultSpawn;
+
+function str(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Parse the configured command line into [executable, ...args]. */
+function parseCommand(baseUrl: string): [string, string[]] {
+	const parts = baseUrl.trim().split(/\s+/).filter(Boolean);
+	if (parts.length === 0)
+		throw new Error(
+			'custom: no command configured (set baseUrl to the command to run)',
+		);
+	return [parts[0]!, parts.slice(1)];
+}
+
+/**
+ * Normalize one stdout entry into a SearchResult, FAILING CLEARLY on a malformed
+ * entry rather than dropping it — the custom contract is explicit, so a missing
+ * url/title is a contract violation the user should see, not a silent skip.
+ */
+function toResult(entry: unknown, index: number): SearchResult {
+	if (typeof entry !== 'object' || entry === null)
+		throw new Error(
+			`custom: malformed output — result[${index}] is not an object`,
+		);
+	const hit = entry as Record<string, unknown>;
+	const url = str(hit.url);
+	const title = str(hit.title);
+	if (!url || !title)
+		throw new Error(
+			`custom: malformed output — result[${index}] is missing a url or title`,
+		);
+	const snippet = str(hit.snippet);
+	return snippet ? {title, url, snippet} : {title, url};
+}
+
+/** Parse the command's stdout into SearchResult[], failing clearly on garbage. */
+function parseOutput(stdout: string): SearchResult[] {
+	const trimmed = stdout.trim();
+	if (trimmed.length === 0)
+		throw new Error('custom: command produced no output');
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(trimmed);
+	} catch (cause) {
+		throw new Error(
+			`custom: malformed output — stdout is not valid JSON: ${(cause as Error).message}`,
+		);
+	}
+	if (!Array.isArray(parsed))
+		throw new Error(
+			'custom: malformed output — expected a JSON array of results',
+		);
+	return parsed.map(toResult);
+}
+
+/** Spawn the command, write the request to stdin, and collect stdout/stderr. */
+function runCommand(
+	spawn: SpawnFn,
+	exe: string,
+	args: string[],
+	request: CustomRequest,
+	signal?: AbortSignal,
+): Promise<CommandRun> {
+	return new Promise<CommandRun>((resolve, reject) => {
+		const child = spawn(exe, args, {
+			stdio: ['pipe', 'pipe', 'pipe'],
+			signal,
+		});
+		let stdout = '';
+		let stderr = '';
+		child.stdout?.on('data', (chunk) => (stdout += String(chunk)));
+		child.stderr?.on('data', (chunk) => (stderr += String(chunk)));
+		child.on('error', (err) =>
+			reject(new Error(`custom: failed to spawn '${exe}': ${err.message}`)),
+		);
+		child.on('close', (code) => resolve({stdout, stderr, code}));
+		child.stdin?.on('error', () => {
+			// A command that exits before reading stdin closes the pipe; ignore the
+			// EPIPE here and let the close handler report via exit code/stderr.
+		});
+		child.stdin?.end(JSON.stringify(request));
+	});
+}
+
+/**
+ * Build a custom backend bound to the configured command. The command owns its
+ * own I/O; webveil hands it the request as JSON on stdin and parses
+ * SearchResult[] from stdout, failing clearly on malformed output.
+ */
+export function createCustomBackend(
+	config: Config,
+	spawn: SpawnFn = defaultSpawn,
+): Backend {
+	const [exe, args] = parseCommand(config.baseUrl);
+	return {
+		async search(
+			query: string,
+			_http: Http,
+			options: SearchOptions = {},
+		): Promise<SearchResult[]> {
+			const request: CustomRequest = {query};
+			if (options.maxResults !== undefined)
+				request.maxResults = options.maxResults;
+			const run = await runCommand(spawn, exe, args, request, options.signal);
+			if (run.code !== 0)
+				throw new Error(
+					`custom: command '${exe}' exited with code ${run.code}` +
+						(run.stderr.trim() ? `: ${run.stderr.trim()}` : ''),
+				);
+			const results = parseOutput(run.stdout);
+			return options.maxResults !== undefined
+				? results.slice(0, options.maxResults)
+				: results;
+		},
+	};
+}

package/src/core/backends/registry.ts

@@ -0,0 +1,41 @@
+// backend registry — a tiny `name -> Backend` dispatcher (concept trimmed from
+// pi-search-hub's registry). Each backend registers a factory keyed by its config
+// `backend` name; `getBackend` resolves the name to a constructed Backend (handed
+// the resolved config so it knows its instance baseUrl / apiKey) and fails clearly
+// on an unknown name. Later backend tasks (tavily-compat, custom) append their own
+// registrations to FACTORIES below.
+
+import type {Config} from '../config.js';
+import type {Backend} from './types.js';
+import {createSearxngBackend} from './searxng.js';
+import {createTavilyCompatBackend} from './tavily-compat.js';
+import {createCustomBackend} from './custom.js';
+
+/** Builds a Backend from the resolved config (knows its baseUrl / apiKey). */
+export type BackendFactory = (config: Config) => Backend;
+
+/** name -> factory. New backends add an entry here. */
+const FACTORIES: Record<string, BackendFactory> = {
+	searxng: createSearxngBackend,
+	'tavily-compat': createTavilyCompatBackend,
+	custom: createCustomBackend,
+};
+
+/** The backend names the registry can resolve. */
+export function backendNames(): string[] {
+	return Object.keys(FACTORIES);
+}
+
+/**
+ * Resolve a backend name to a constructed Backend. Throws clearly on an unknown
+ * name (listing the known ones) so a misconfigured `backend` fails loud, never
+ * silently no-ops.
+ */
+export function getBackend(name: string, config: Config): Backend {
+	const factory = FACTORIES[name];
+	if (!factory)
+		throw new Error(
+			`webveil: unknown backend '${name}' (known: ${backendNames().join(', ')})`,
+		);
+	return factory(config);
+}

package/src/core/backends/searxng.ts

@@ -0,0 +1,70 @@
+// searxng backend — the keyless, self-hosted metasearch default. Queries a
+// SearXNG instance's JSON API (`/search?format=json`) THROUGH the handed `http`
+// helper (never a direct fetch, so egress is not bypassable) and normalizes the
+// response into SearchResult[].
+
+import type {Config} from '../config.js';
+import type {Backend, Http, SearchOptions, SearchResult} from './types.js';
+
+/** The shape of one entry in a SearXNG JSON `results` array (subset we use). */
+interface SearxngResult {
+	url?: unknown;
+	title?: unknown;
+	content?: unknown;
+}
+
+/** The SearXNG JSON API response (subset we use). */
+interface SearxngResponse {
+	results?: SearxngResult[];
+}
+
+function str(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Normalize one SearXNG hit; drop entries without a usable url + title. */
+function toResult(hit: SearxngResult): SearchResult | undefined {
+	const url = str(hit.url);
+	const title = str(hit.title);
+	if (!url || !title) return undefined;
+	const snippet = str(hit.content);
+	return snippet ? {title, url, snippet} : {title, url};
+}
+
+/** Build the SearXNG JSON search URL for a query against the instance baseUrl. */
+function buildUrl(baseUrl: string, query: string): string {
+	const url = new URL(
+		'search',
+		baseUrl.endsWith('/') ? baseUrl : baseUrl + '/',
+	);
+	url.searchParams.set('q', query);
+	url.searchParams.set('format', 'json');
+	return url.toString();
+}
+
+/**
+ * Build a SearXNG backend bound to the configured instance. The returned backend
+ * only ever touches the network via the injected `http` helper.
+ */
+export function createSearxngBackend(config: Config): Backend {
+	const baseUrl = config.baseUrl;
+	return {
+		async search(
+			query: string,
+			http: Http,
+			options: SearchOptions = {},
+		): Promise<SearchResult[]> {
+			const body = await http.fetchJson<SearxngResponse>(
+				buildUrl(baseUrl, query),
+				{headers: {accept: 'application/json'}, signal: options.signal},
+			);
+			const results = Array.isArray(body.results) ? body.results : [];
+			const normalized = results
+				.map(toResult)
+				.filter((r): r is SearchResult => r !== undefined);
+			return options.maxResults !== undefined
+				? normalized.slice(0, options.maxResults)
+				: normalized;
+		},
+	};
+}