webveil 0.0.0 → 0.1.0

package/dist/core/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/core/search.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAC,MAAM,EAAE,cAAc,EAAC,MAAM,aAAa,CAAC;AAExD,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,aAAa,CAAC;AAG5C,OAAO,KAAK,EAAC,IAAI,EAAE,aAAa,EAAE,YAAY,EAAC,MAAM,qBAAqB,CAAC;AAU3E;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IAC1B,aAAa,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,cAAc,KAAK,MAAM,CAAC;IACrD,eAAe,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,UAAU,GAAG,SAAS,CAAC;IAC7D,UAAU,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,GAAG,SAAS,KAAK,IAAI,CAAC;IAC1D,UAAU,CAAC,EAAE,CACZ,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,KACV;QACJ,MAAM,EAAE,CACP,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,IAAI,EACV,OAAO,CAAC,EAAE,aAAa,KACnB,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;KAC7B,CAAC;CACF;AAED,iFAAiF;AACjF,MAAM,WAAW,iBAAkB,SAAQ,aAAa,EAAE,cAAc;CAAG;AAc3E;;;;;;;GAOG;AACH,wBAAsB,MAAM,CAC3B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,iBAAsB,EAC/B,IAAI,GAAE,UAAe,GACnB,OAAO,CAAC,YAAY,EAAE,CAAC,CAwBzB"}

package/dist/core/search.js

@@ -0,0 +1,65 @@
+// core search — the plain, framework-agnostic `search()` BOTH frontends (the
+// incur CLI/MCP and the pi extension) call. It owns the wiring and the
+// caller-facing post-processing; the per-source parsing lives in the backend.
+//
+// Flow: resolve config → build the egress dispatcher → bind the proxied `http`
+// helper to it → select the backend from the registry → call the backend with
+// ONLY that proxied helper → normalize (dedup + clamp) the SearchResult[].
+//
+// The egress invariant (docs/adr/0001): the backend is handed only the
+// dispatcher-bound `http` helper, so it physically cannot reach a global fetch
+// and bypass the configured egress. A configured-but-unbuildable proxy throws at
+// buildDispatcher (fail-loud), never silently un-proxied.
+import { resolveConfig as defaultResolveConfig } from './config.js';
+import { buildDispatcher as defaultBuildDispatcher } from './egress.js';
+import { createHttp as defaultCreateHttp } from './http.js';
+import { getBackend as defaultGetBackend } from './backends/registry.js';
+/**
+ * Default cap on returned results when the caller does not pass `maxResults`.
+ * Keeps an agent's context small by default; a caller can raise/lower it per
+ * call. (Recorded decision: there is no configured default, so the core sets
+ * one; see the task's Decisions block.)
+ */
+const DEFAULT_MAX_RESULTS = 10;
+/** Dedup by url (the hit's identity), preserving first-seen order. */
+function dedup(results) {
+    const seen = new Set();
+    const out = [];
+    for (const r of results) {
+        if (seen.has(r.url))
+            continue;
+        seen.add(r.url);
+        out.push(r);
+    }
+    return out;
+}
+/**
+ * Search the configured backend over the configured egress and return
+ * normalized `SearchResult[]` (deduped by url, then clamped to `maxResults`).
+ *
+ * Dedup runs BEFORE the clamp so the caller gets up to `maxResults` UNIQUE hits,
+ * not a window that duplicates eat into; for the same reason the backend is NOT
+ * asked to pre-clamp (only the abort signal is forwarded).
+ */
+export async function search(query, options = {}, deps = {}) {
+    const resolveConfig = deps.resolveConfig ?? defaultResolveConfig;
+    const buildDispatcher = deps.buildDispatcher ?? defaultBuildDispatcher;
+    const createHttp = deps.createHttp ?? defaultCreateHttp;
+    const getBackend = deps.getBackend ?? defaultGetBackend;
+    const config = resolveConfig({
+        cwd: options.cwd,
+        env: options.env,
+        globalPath: options.globalPath,
+    });
+    // Build the dispatcher FIRST: a configured-but-unbuildable proxy throws here,
+    // before any network access (never an un-proxied request).
+    const dispatcher = buildDispatcher(config);
+    const http = createHttp(dispatcher);
+    const backend = getBackend(config.backend, config);
+    // Hand the backend ONLY the proxied helper (no maxResults: dedup happens
+    // here, over the full set, so the clamp below is over UNIQUE results).
+    const raw = await backend.search(query, http, { signal: options.signal });
+    const maxResults = options.maxResults ?? DEFAULT_MAX_RESULTS;
+    return dedup(raw).slice(0, maxResults);
+}
+//# sourceMappingURL=search.js.map

package/dist/core/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/core/search.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,uEAAuE;AACvE,8EAA8E;AAC9E,EAAE;AACF,+EAA+E;AAC/E,8EAA8E;AAC9E,2EAA2E;AAC3E,EAAE;AACF,uEAAuE;AACvE,+EAA+E;AAC/E,iFAAiF;AACjF,0DAA0D;AAE1D,OAAO,EAAC,aAAa,IAAI,oBAAoB,EAAC,MAAM,aAAa,CAAC;AAElE,OAAO,EAAC,eAAe,IAAI,sBAAsB,EAAC,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAC,UAAU,IAAI,iBAAiB,EAAC,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAC,UAAU,IAAI,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AAGvE;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAC;AA4B/B,sEAAsE;AACtE,SAAS,KAAK,CAAC,OAAuB;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YAAE,SAAS;QAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACb,CAAC;IACD,OAAO,GAAG,CAAC;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC3B,KAAa,EACb,UAA6B,EAAE,EAC/B,OAAmB,EAAE;IAErB,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,oBAAoB,CAAC;IACjE,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,sBAAsB,CAAC;IACvE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAC;IACxD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,iBAAiB,CAAC;IAExD,MAAM,MAAM,GAAG,aAAa,CAAC;QAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,UAAU,EAAE,OAAO,CAAC,UAAU;KAC9B,CAAC,CAAC;IAEH,8EAA8E;IAC9E,2DAA2D;IAC3D,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC;IAEpC,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnD,yEAAyE;IACzE,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAC,CAAC,CAAC;IAExE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,mBAAmB,CAAC;IAC7D,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACxC,CAAC"}

package/dist/core/security.d.ts

@@ -0,0 +1,35 @@
+import type { Config } from './config.js';
+import type { EgressFetch } from './egress.js';
+/** Thrown when the SSRF guard refuses a request to a private/blocked address. */
+export declare class SsrfError extends Error {
+    constructor(message: string);
+}
+/**
+ * Is this LITERAL IP private / non-public? Covers the ranges that must never be
+ * reachable from a direct-egress web fetch:
+ *   IPv4: 0.0.0.0/8, 10/8 (RFC1918), 127/8 (loopback), 169.254/16 (link-local,
+ *     incl. the 169.254.169.254 cloud metadata endpoint), 172.16/12 (RFC1918),
+ *     192.168/16 (RFC1918), 100.64/10 (CGNAT), 192.0.0/24, 192.0.2/24,
+ *     198.18/15, 198.51.100/24, 203.0.113/24, 224/4 (multicast), 240/4
+ *     (reserved).
+ *   IPv6: ::1 (loopback), :: (unspecified), fc00::/7 (ULA), fe80::/10
+ *     (link-local), ff00::/8 (multicast), plus IPv4-mapped (::ffff:a.b.c.d,
+ *     re-checked as IPv4). Default-deny: anything outside global unicast
+ *     (2000::/3) is treated as non-public.
+ */
+export declare function isPrivateIp(ip: string): boolean;
+/**
+ * Assert a URL is safe to fetch under THIS config's egress. Under a proxy egress
+ * it always passes (the proxy owns egress + DNS). Under direct egress it rejects
+ * a literal private IP and, for a hostname, resolves it locally and rejects if it
+ * maps to a private IP (so a name pointing at 127.0.0.1 / metadata is caught).
+ */
+export declare function assertPublicUrl(url: string, config: Config): Promise<void>;
+/**
+ * Wrap an egress-bound `fetch` with the SSRF guard. The returned fetch checks
+ * EVERY request URL (so it covers distilly's rule-rewritten requests too, not
+ * only webveil's own GET) before delegating to the underlying egress fetch.
+ * This is what `core.fetch()` injects into distilly. See docs/adr/0001.
+ */
+export declare function guardEgressFetch(fetch: EgressFetch, config: Config): EgressFetch;
+//# sourceMappingURL=security.d.ts.map

package/dist/core/security.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.d.ts","sourceRoot":"","sources":["../../src/core/security.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AACxC,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,aAAa,CAAC;AAE7C,iFAAiF;AACjF,qBAAa,SAAU,SAAQ,KAAK;gBACvB,OAAO,EAAE,MAAM;CAI3B;AAOD;;;;;;;;;;;;GAYG;AACH,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAK/C;AAuCD;;;;;GAKG;AACH,wBAAsB,eAAe,CACpC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC,CAsBf;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC/B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,GACZ,WAAW,CAWb"}

package/dist/core/security.js

@@ -0,0 +1,141 @@
+// SSRF guard: the security seam wrapped AROUND the egress-bound `fetch`, so it
+// covers BOTH webveil's own GETs AND distilly's rule-rewritten requests (see
+// docs/adr/0001: the guard lives inside the egress fetch). Adapts the range
+// classification + DNS-resolve approach of leing2021/pi-search's `security.ts`.
+//
+// THE RELAXATION RULE (load-bearing, recorded in the task's Decisions): the
+// guard BLOCKS private/loopback/link-local/etc. addresses on DIRECT egress, and
+// RELAXES ENTIRELY under a proxy egress (`http` | `socks5`). Tor/Mullvad
+// legitimately reach private-looking addresses (e.g. `10.64.0.1`), AND a local
+// DNS lookup for a proxied request would itself be a deanonymizing leak, so
+// under a proxy we neither block nor resolve locally; the proxy owns egress.
+import { lookup } from 'node:dns/promises';
+import { isIP } from 'node:net';
+/** Thrown when the SSRF guard refuses a request to a private/blocked address. */
+export class SsrfError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'SsrfError';
+    }
+}
+/** A proxy egress owns egress + DNS, so the local SSRF guard relaxes for it. */
+function egressIsProxy(config) {
+    return config.egress.mode === 'http' || config.egress.mode === 'socks5';
+}
+/**
+ * Is this LITERAL IP private / non-public? Covers the ranges that must never be
+ * reachable from a direct-egress web fetch:
+ *   IPv4: 0.0.0.0/8, 10/8 (RFC1918), 127/8 (loopback), 169.254/16 (link-local,
+ *     incl. the 169.254.169.254 cloud metadata endpoint), 172.16/12 (RFC1918),
+ *     192.168/16 (RFC1918), 100.64/10 (CGNAT), 192.0.0/24, 192.0.2/24,
+ *     198.18/15, 198.51.100/24, 203.0.113/24, 224/4 (multicast), 240/4
+ *     (reserved).
+ *   IPv6: ::1 (loopback), :: (unspecified), fc00::/7 (ULA), fe80::/10
+ *     (link-local), ff00::/8 (multicast), plus IPv4-mapped (::ffff:a.b.c.d,
+ *     re-checked as IPv4). Default-deny: anything outside global unicast
+ *     (2000::/3) is treated as non-public.
+ */
+export function isPrivateIp(ip) {
+    const kind = isIP(ip);
+    if (kind === 4)
+        return isPrivateIpv4(ip);
+    if (kind === 6)
+        return isPrivateIpv6(ip);
+    return false; // not a literal IP; hostname handling resolves it first
+}
+function isPrivateIpv4(ip) {
+    const parts = ip.split('.').map((p) => Number(p));
+    if (parts.length !== 4 ||
+        parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255))
+        return true; // malformed → treat as non-public (fail closed)
+    const [a, b] = parts;
+    if (a === 0 || a === 10 || a === 127)
+        return true;
+    if (a === 169 && b === 254)
+        return true; // link-local incl. cloud metadata
+    if (a === 172 && b >= 16 && b <= 31)
+        return true; // 172.16/12
+    if (a === 192 && b === 168)
+        return true; // 192.168/16
+    if (a === 100 && b >= 64 && b <= 127)
+        return true; // 100.64/10 CGNAT
+    if (a === 192 && b === 0)
+        return true; // 192.0.0/24 + 192.0.2/24
+    if (a === 198 && (b === 18 || b === 19))
+        return true; // 198.18/15 benchmark
+    if (a === 198 && b === 51)
+        return true; // 198.51.100/24 TEST-NET-2
+    if (a === 203 && b === 0)
+        return true; // 203.0.113/24 TEST-NET-3
+    if (a >= 224)
+        return true; // 224/4 multicast + 240/4 reserved
+    return false;
+}
+function isPrivateIpv6(ip) {
+    const lower = ip.toLowerCase();
+    if (lower === '::1' || lower === '::')
+        return true; // loopback / unspecified
+    // IPv4-mapped (::ffff:a.b.c.d): re-check the embedded IPv4.
+    const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+    if (mapped)
+        return isPrivateIpv4(mapped[1]);
+    const head = lower.split(':')[0] ?? '';
+    const first = parseInt(head || '0', 16);
+    if (Number.isNaN(first))
+        return true; // fail closed on anything unparseable
+    if ((first & 0xfe00) === 0xfc00)
+        return true; // fc00::/7 ULA
+    if ((first & 0xffc0) === 0xfe80)
+        return true; // fe80::/10 link-local
+    if ((first & 0xff00) === 0xff00)
+        return true; // ff00::/8 multicast
+    // Default-deny: only global unicast 2000::/3 is public.
+    return (first & 0xe000) !== 0x2000;
+}
+/**
+ * Assert a URL is safe to fetch under THIS config's egress. Under a proxy egress
+ * it always passes (the proxy owns egress + DNS). Under direct egress it rejects
+ * a literal private IP and, for a hostname, resolves it locally and rejects if it
+ * maps to a private IP (so a name pointing at 127.0.0.1 / metadata is caught).
+ */
+export async function assertPublicUrl(url, config) {
+    if (egressIsProxy(config))
+        return; // proxy owns egress + DNS; relax entirely
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        throw new SsrfError(`webveil SSRF: malformed url ${url}`);
+    }
+    const host = parsed.hostname.replace(/^\[|\]$/g, ''); // strip IPv6 brackets
+    if (isIP(host)) {
+        if (isPrivateIp(host))
+            throw new SsrfError(`webveil SSRF: blocked private address ${host}`);
+        return;
+    }
+    // A hostname: resolve it locally (safe on direct egress) and check every
+    // address it maps to, so a name pointing at a private IP is also blocked.
+    const addrs = await lookup(host, { all: true });
+    for (const { address } of addrs)
+        if (isPrivateIp(address))
+            throw new SsrfError(`webveil SSRF: ${host} resolves to private address ${address}`);
+}
+/**
+ * Wrap an egress-bound `fetch` with the SSRF guard. The returned fetch checks
+ * EVERY request URL (so it covers distilly's rule-rewritten requests too, not
+ * only webveil's own GET) before delegating to the underlying egress fetch.
+ * This is what `core.fetch()` injects into distilly. See docs/adr/0001.
+ */
+export function guardEgressFetch(fetch, config) {
+    return (async (input, init) => {
+        const url = typeof input === 'string'
+            ? input
+            : input instanceof URL
+                ? input.href
+                : input.url;
+        await assertPublicUrl(url, config);
+        return fetch(input, init);
+    });
+}
+//# sourceMappingURL=security.js.map

package/dist/core/security.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"security.js","sourceRoot":"","sources":["../../src/core/security.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,6EAA6E;AAC7E,4EAA4E;AAC5E,gFAAgF;AAChF,EAAE;AACF,4EAA4E;AAC5E,gFAAgF;AAChF,yEAAyE;AACzE,+EAA+E;AAC/E,4EAA4E;AAC5E,6EAA6E;AAE7E,OAAO,EAAC,MAAM,EAAC,MAAM,mBAAmB,CAAC;AACzC,OAAO,EAAC,IAAI,EAAC,MAAM,UAAU,CAAC;AAI9B,iFAAiF;AACjF,MAAM,OAAO,SAAU,SAAQ,KAAK;IACnC,YAAY,OAAe;QAC1B,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IACzB,CAAC;CACD;AAED,gFAAgF;AAChF,SAAS,aAAa,CAAC,MAAc;IACpC,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,WAAW,CAAC,EAAU;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC,CAAC,wDAAwD;AACvE,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAChC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,IACC,KAAK,CAAC,MAAM,KAAK,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;QAE3D,OAAO,IAAI,CAAC,CAAC,gDAAgD;IAC9D,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,KAAyC,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAClD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,kCAAkC;IAC3E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,YAAY;IAC9D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,aAAa;IACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,kBAAkB;IACrE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IACjE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sBAAsB;IAC5E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC,CAAC,2BAA2B;IACnE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IACjE,IAAI,CAAC,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,CAAC,mCAAmC;IAC9D,OAAO,KAAK,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,EAAU;IAChC,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/B,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC,CAAC,yBAAyB;IAC7E,4DAA4D;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC5D,IAAI,MAAM;QAAE,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,sCAAsC;IAC5E,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,eAAe;IAC7D,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,uBAAuB;IACrE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IACnE,wDAAwD;IACxD,OAAO,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,MAAM,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,GAAW,EACX,MAAc;IAEd,IAAI,aAAa,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,0CAA0C;IAC7E,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,SAAS,CAAC,+BAA+B,GAAG,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB;IAC5E,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAChB,IAAI,WAAW,CAAC,IAAI,CAAC;YACpB,MAAM,IAAI,SAAS,CAAC,yCAAyC,IAAI,EAAE,CAAC,CAAC;QACtE,OAAO;IACR,CAAC;IACD,yEAAyE;IACzE,0EAA0E;IAC1E,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,EAAC,GAAG,EAAE,IAAI,EAAC,CAAC,CAAC;IAC9C,KAAK,MAAM,EAAC,OAAO,EAAC,IAAI,KAAK;QAC5B,IAAI,WAAW,CAAC,OAAO,CAAC;YACvB,MAAM,IAAI,SAAS,CAClB,iBAAiB,IAAI,gCAAgC,OAAO,EAAE,CAC9D,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC/B,KAAkB,EAClB,MAAc;IAEd,OAAO,CAAC,KAAK,EAAE,KAAwB,EAAE,IAAkB,EAAE,EAAE;QAC9D,MAAM,GAAG,GACR,OAAO,KAAK,KAAK,QAAQ;YACxB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,KAAK,YAAY,GAAG;gBACrB,CAAC,CAAC,KAAK,CAAC,IAAI;gBACZ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;QACf,MAAM,eAAe,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,KAAc,EAAE,IAAa,CAAC,CAAC;IAC7C,CAAC,CAAgB,CAAC;AACnB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,22 @@
+export { resolveConfig } from './core/config.js';
+export type { Config, Egress, FetchSize, PartialConfig, ResolveOptions, } from './core/config.js';
+export { buildDispatcher, createEgressFetch, EgressError, } from './core/egress.js';
+export type { Dispatcher, EgressFetch } from './core/egress.js';
+export { createHttp } from './core/http.js';
+export { extract } from './core/extract.js';
+export type { ExtractOptions, ExtractDeps } from './core/extract.js';
+export { assertPublicUrl, guardEgressFetch, isPrivateIp, SsrfError, } from './core/security.js';
+export type { Backend, Http, HttpRequestOptions, SearchResult, FetchResult, SearchOptions, FetchOptions, } from './core/backends/types.js';
+export { backendNames, getBackend } from './core/backends/registry.js';
+export type { BackendFactory } from './core/backends/registry.js';
+export { createSearxngBackend } from './core/backends/searxng.js';
+export { createTavilyCompatBackend } from './core/backends/tavily-compat.js';
+export { createCustomBackend } from './core/backends/custom.js';
+export type { SpawnFn } from './core/backends/custom.js';
+export { search } from './core/search.js';
+export type { SearchCoreOptions, SearchDeps } from './core/search.js';
+export { fetch, fetchAll } from './core/fetch.js';
+export type { FetchCoreOptions, FetchDeps } from './core/fetch.js';
+export { createCli } from './cli.js';
+export type { CliDeps } from './cli.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAoBA,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,YAAY,EACX,MAAM,EACN,MAAM,EACN,SAAS,EACT,aAAa,EACb,cAAc,GACd,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EACN,eAAe,EACf,iBAAiB,EACjB,WAAW,GACX,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAC,UAAU,EAAE,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAG9D,OAAO,EAAC,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAG1C,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAC1C,YAAY,EAAC,cAAc,EAAE,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAGnE,OAAO,EACN,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,SAAS,GACT,MAAM,oBAAoB,CAAC;AAG5B,YAAY,EACX,OAAO,EACP,IAAI,EACJ,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,GACZ,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,6BAA6B,CAAC;AACrE,YAAY,EAAC,cAAc,EAAC,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAC,yBAAyB,EAAC,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAC,mBAAmB,EAAC,MAAM,2BAA2B,CAAC;AAC9D,YAAY,EAAC,OAAO,EAAC,MAAM,2BAA2B,CAAC;AAGvD,OAAO,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AACxC,YAAY,EAAC,iBAAiB,EAAE,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAGpE,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAChD,YAAY,EAAC,gBAAgB,EAAE,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAGjE,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AACnC,YAAY,EAAC,OAAO,EAAC,MAAM,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,40 @@
+// webveil — anonymous-capable, self-hosted, account-free web search + fetch for agents.
+//
+// This is the public surface. The framework-agnostic core lives under src/core:
+//   - core/config.ts            : config seam (per-folder .pi/webveil.json + global + env)
+//   - core/egress.ts            : egress seam (direct | http | socks5/Tor) — dispatcher + egress fetch
+//   - core/http.ts              : the proxied `http` helper handed to backends
+//   - core/extract.ts           : Extractor seam (distilly/fetch + injected egress fetch)
+//   - core/backends/types.ts    : backend seam (the Backend interface + result shapes)
+//   - core/backends/registry.ts : name -> Backend dispatcher
+//   - core/backends/searxng.ts  : the keyless self-hosted SearXNG backend
+//   - core/backends/tavily-compat.ts : the generic Tavily-shaped backend (/search + /extract)
+//   - core/search.ts            : the framework-agnostic search() both frontends call
+//   - core/security.ts          : SSRF guard wrapped around the egress fetch
+//   - core/fetch.ts             : the framework-agnostic fetch() both frontends call
+//   - core/backends/custom.ts  : the local-command escape hatch (JSON stdin/stdout)
+//   - cli.ts                    : the incur CLI + MCP frontend (the `webveil` bin)
+// pi-webveil (sibling package) wraps the SAME core functions as registerTool
+// web_search / web_fetch, in-process, as an Ollama drop-in.
+// config seam
+export { resolveConfig } from './core/config.js';
+// egress seam
+export { buildDispatcher, createEgressFetch, EgressError, } from './core/egress.js';
+// http helper
+export { createHttp } from './core/http.js';
+// Extractor seam (distilly/fetch over webveil's egress)
+export { extract } from './core/extract.js';
+// SSRF guard (wrapped around the egress fetch; covers distilly's requests too)
+export { assertPublicUrl, guardEgressFetch, isPrivateIp, SsrfError, } from './core/security.js';
+// backend registry + implementations
+export { backendNames, getBackend } from './core/backends/registry.js';
+export { createSearxngBackend } from './core/backends/searxng.js';
+export { createTavilyCompatBackend } from './core/backends/tavily-compat.js';
+export { createCustomBackend } from './core/backends/custom.js';
+// core search (the framework-agnostic search() both frontends call)
+export { search } from './core/search.js';
+// core fetch (the framework-agnostic fetch() + list-ready fetchAll internal)
+export { fetch, fetchAll } from './core/fetch.js';
+// incur CLI + MCP frontend (the `webveil` bin builds and serves this)
+export { createCli } from './cli.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,wFAAwF;AACxF,EAAE;AACF,gFAAgF;AAChF,2FAA2F;AAC3F,uGAAuG;AACvG,+EAA+E;AAC/E,0FAA0F;AAC1F,uFAAuF;AACvF,6DAA6D;AAC7D,0EAA0E;AAC1E,8FAA8F;AAC9F,sFAAsF;AACtF,6EAA6E;AAC7E,qFAAqF;AACrF,oFAAoF;AACpF,mFAAmF;AACnF,6EAA6E;AAC7E,4DAA4D;AAE5D,cAAc;AACd,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAS/C,cAAc;AACd,OAAO,EACN,eAAe,EACf,iBAAiB,EACjB,WAAW,GACX,MAAM,kBAAkB,CAAC;AAG1B,cAAc;AACd,OAAO,EAAC,UAAU,EAAC,MAAM,gBAAgB,CAAC;AAE1C,wDAAwD;AACxD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAC;AAG1C,+EAA+E;AAC/E,OAAO,EACN,eAAe,EACf,gBAAgB,EAChB,WAAW,EACX,SAAS,GACT,MAAM,oBAAoB,CAAC;AAa5B,qCAAqC;AACrC,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,6BAA6B,CAAC;AAErE,OAAO,EAAC,oBAAoB,EAAC,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAC,yBAAyB,EAAC,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAC,mBAAmB,EAAC,MAAM,2BAA2B,CAAC;AAG9D,oEAAoE;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,kBAAkB,CAAC;AAGxC,6EAA6E;AAC7E,OAAO,EAAC,KAAK,EAAE,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AAGhD,sEAAsE;AACtE,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC"}

package/package.json

@@ -1,4 +1,64 @@
 {
   "name": "webveil",
-  "version": "0.0.0"
-}
+  "version": "0.1.0",
+  "description": "Anonymous-capable, self-hosted, account-free web search and fetch for agents. CLI + MCP (built on incur), pi-agnostic. Swappable backend and egress (direct, http proxy, socks5/Tor).",
+  "license": "AGPL-3.0-or-later",
+  "keywords": [
+    "web-search",
+    "web-fetch",
+    "mcp",
+    "cli",
+    "agents",
+    "searxng",
+    "tor",
+    "anonymous",
+    "self-hosted"
+  ],
+  "author": "wighawag",
+  "homepage": "https://github.com/wighawag/webveil/tree/main/packages/webveil#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/wighawag/webveil.git",
+    "directory": "packages/webveil"
+  },
+  "bugs": {
+    "url": "https://github.com/wighawag/webveil/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "bin": {
+    "webveil": "./dist/cli.js"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@modelcontextprotocol/server": "2.0.0-alpha.2",
+    "distilly": "^0.1.0",
+    "fetch-socks": "^1.3.3",
+    "incur": "0.4.10",
+    "undici": "^7.28.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.2.0",
+    "as-soon": "^0.1.5",
+    "tsx": "^4.21.0",
+    "typescript": "^5.3.3",
+    "vitest": "^4.0.18"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "as-soon -w src pnpm build",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}

package/src/cli.ts

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+// webveil — the incur-based CLI + MCP frontend. ONE `Cli.create()` definition
+// yields the CLI, an MCP server (`--mcp`), skills (`skills add`), a `--llms`
+// manifest, TOON output, and token pagination for free (incur). Pi-agnostic:
+// any agent (pi via pi-mcp-adapter, Claude Code, Cursor, Codex, bash) consumes
+// it the same way. The `webveil` bin points at the built `dist/cli.js`.
+//
+// This is the THIN frontend: each command only parses argv/options and calls
+// the SAME framework-agnostic core (`search()` / `fetch()`) the pi extension
+// calls. The core owns config/egress/backend/extraction; this file owns no
+// network logic of its own.
+//
+// Testability: `createCli(deps)` takes the core functions as injectable deps so
+// a test wires fakes and asserts the commands call the core (via `cli.serve`
+// with custom argv/stdout) WITHOUT touching the network. The bottom of the file
+// builds the real CLI and serves it when run as the bin.
+
+import {argv} from 'node:process';
+import {fileURLToPath} from 'node:url';
+import {Cli, z} from 'incur';
+import {search as coreSearch} from './core/search.js';
+import {fetch as coreFetch} from './core/fetch.js';
+
+/**
+ * The two core functions the frontend wraps, seamed so tests can inject fakes.
+ * Defaults are the real core; a test passes spies to assert the wiring.
+ */
+export interface CliDeps {
+	search?: typeof coreSearch;
+	fetch?: typeof coreFetch;
+}
+
+/** The size presets `fetch` accepts, mirroring the core's `FetchSize`. */
+const SIZES = ['s', 'm', 'l', 'f'] as const;
+
+/**
+ * Build the webveil CLI. Returns the incur `Cli` so a caller (the bin below, or
+ * a test) decides how to serve it. The `search`/`fetch` commands forward to the
+ * injected core, normalizing nothing themselves — the core already deduped,
+ * clamped, and size-bounded.
+ */
+export function createCli(deps: CliDeps = {}) {
+	const search = deps.search ?? coreSearch;
+	const fetch = deps.fetch ?? coreFetch;
+
+	return Cli.create('webveil', {
+		description:
+			'Anonymous-capable, self-hosted, account-free web search + fetch for agents.',
+	})
+		.command('search', {
+			description: 'Search the web via the configured backend and egress.',
+			args: z.object({
+				query: z.string().describe('The search query'),
+			}),
+			options: z.object({
+				maxResults: z.coerce
+					.number()
+					.optional()
+					.describe('Maximum number of results to return'),
+			}),
+			alias: {maxResults: 'n'},
+			async run(c) {
+				const results = await search(c.args.query, {
+					maxResults: c.options.maxResults,
+				});
+				return {results};
+			},
+		})
+		.command('fetch', {
+			description:
+				'Fetch a URL as clean, size-bounded markdown via the configured egress.',
+			args: z.object({
+				url: z.string().describe('The URL to fetch'),
+			}),
+			options: z.object({
+				size: z
+					.enum(SIZES)
+					.optional()
+					.describe('Page-size budget preset: s | m | l | f'),
+			}),
+			alias: {size: 's'},
+			async run(c) {
+				return fetch(c.args.url, {size: c.options.size});
+			},
+		});
+}
+
+// The real CLI (also `export default` so `incur gen` can import it for typed
+// CTAs). Serving is GUARDED to the bin entry below, so importing this module in
+// a test never consumes `process.argv` or exits the process.
+const cli = createCli();
+
+/** True when this module is the process entry (the `webveil` bin), not imported. */
+function isMain(): boolean {
+	const entry = argv[1];
+	if (!entry) return false;
+	try {
+		return fileURLToPath(import.meta.url) === entry;
+	} catch {
+		return false;
+	}
+}
+
+if (isMain()) cli.serve();
+
+export default cli;