webveil 0.0.0 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +101 -0
- package/dist/cli.d.ts +58 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +91 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/backends/custom.d.ts +15 -0
- package/dist/core/backends/custom.d.ts.map +1 -0
- package/dist/core/backends/custom.js +106 -0
- package/dist/core/backends/custom.js.map +1 -0
- package/dist/core/backends/registry.d.ts +13 -0
- package/dist/core/backends/registry.d.ts.map +1 -0
- package/dist/core/backends/registry.js +31 -0
- package/dist/core/backends/registry.js.map +1 -0
- package/dist/core/backends/searxng.d.ts +8 -0
- package/dist/core/backends/searxng.d.ts.map +1 -0
- package/dist/core/backends/searxng.js +43 -0
- package/dist/core/backends/searxng.js.map +1 -0
- package/dist/core/backends/tavily-compat.d.ts +10 -0
- package/dist/core/backends/tavily-compat.d.ts.map +1 -0
- package/dist/core/backends/tavily-compat.js +85 -0
- package/dist/core/backends/tavily-compat.js.map +1 -0
- package/dist/core/backends/types.d.ts +48 -0
- package/dist/core/backends/types.d.ts.map +1 -0
- package/dist/core/backends/types.js +5 -0
- package/dist/core/backends/types.js.map +1 -0
- package/dist/core/config.d.ts +39 -0
- package/dist/core/config.d.ts.map +1 -0
- package/dist/core/config.js +72 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/egress.d.ts +30 -0
- package/dist/core/egress.d.ts.map +1 -0
- package/dist/core/egress.js +87 -0
- package/dist/core/egress.js.map +1 -0
- package/dist/core/extract.d.ts +45 -0
- package/dist/core/extract.d.ts.map +1 -0
- package/dist/core/extract.js +36 -0
- package/dist/core/extract.js.map +1 -0
- package/dist/core/fetch.d.ts +42 -0
- package/dist/core/fetch.d.ts.map +1 -0
- package/dist/core/fetch.js +76 -0
- package/dist/core/fetch.js.map +1 -0
- package/dist/core/http.d.ts +8 -0
- package/dist/core/http.d.ts.map +1 -0
- package/dist/core/http.js +49 -0
- package/dist/core/http.js.map +1 -0
- package/dist/core/search.d.ts +31 -0
- package/dist/core/search.d.ts.map +1 -0
- package/dist/core/search.js +65 -0
- package/dist/core/search.js.map +1 -0
- package/dist/core/security.d.ts +35 -0
- package/dist/core/security.d.ts.map +1 -0
- package/dist/core/security.js +141 -0
- package/dist/core/security.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +40 -0
- package/dist/index.js.map +1 -0
- package/package.json +62 -2
- package/src/cli.ts +106 -0
- package/src/core/backends/custom.ts +159 -0
- package/src/core/backends/registry.ts +41 -0
- package/src/core/backends/searxng.ts +70 -0
- package/src/core/backends/tavily-compat.ts +156 -0
- package/src/core/backends/types.ts +61 -0
- package/src/core/config.ts +106 -0
- package/src/core/egress.ts +106 -0
- package/src/core/extract.ts +82 -0
- package/src/core/fetch.ts +132 -0
- package/src/core/http.ts +62 -0
- package/src/core/search.ts +104 -0
- package/src/core/security.ts +141 -0
- package/src/index.ts +82 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// egress seam — how outbound HTTP leaves the machine. Yields TWO artifacts off
|
|
2
|
+
// the SAME undici dispatcher: the proxied `http` helper (see http.ts, handed to
|
|
3
|
+
// backends) and an egress-bound WHATWG `fetch` (injected into distilly/fetch).
|
|
4
|
+
//
|
|
5
|
+
// CRITICAL anonymity invariant (docs/adr/0001): egress is fail-loud. A
|
|
6
|
+
// configured proxy that cannot be built MUST throw — it must NEVER silently
|
|
7
|
+
// fall back to un-proxied (direct) transport.
|
|
8
|
+
|
|
9
|
+
import {Agent, type Dispatcher, ProxyAgent, fetch as undiciFetch} from 'undici';
|
|
10
|
+
import {socksDispatcher} from 'fetch-socks';
|
|
11
|
+
import type {Config, Egress} from './config.js';
|
|
12
|
+
|
|
13
|
+
/** Thrown when a configured egress proxy cannot be built. Never swallowed. */
|
|
14
|
+
export class EgressError extends Error {
|
|
15
|
+
constructor(message: string, options?: {cause?: unknown}) {
|
|
16
|
+
super(message, options);
|
|
17
|
+
this.name = 'EgressError';
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function socksFromUrl(raw: string): Dispatcher {
|
|
22
|
+
const url = new URL(raw); // throws on a malformed proxy URL → fail loud
|
|
23
|
+
const protocol = url.protocol.replace(':', '');
|
|
24
|
+
if (protocol !== 'socks5' && protocol !== 'socks' && protocol !== 'socks5h')
|
|
25
|
+
throw new EgressError(
|
|
26
|
+
`egress socks5: expected a socks5:// proxy url, got ${raw}`,
|
|
27
|
+
);
|
|
28
|
+
const port = Number(url.port);
|
|
29
|
+
if (!url.hostname || !Number.isInteger(port) || port <= 0)
|
|
30
|
+
throw new EgressError(`egress socks5: invalid host/port in ${raw}`);
|
|
31
|
+
return socksDispatcher({
|
|
32
|
+
type: 5,
|
|
33
|
+
host: url.hostname,
|
|
34
|
+
port,
|
|
35
|
+
userId: url.username || undefined,
|
|
36
|
+
password: url.password || undefined,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Build the undici Dispatcher for the config's egress mode:
|
|
42
|
+
* - direct → undefined (undici uses its default, un-proxied transport)
|
|
43
|
+
* - http → ProxyAgent
|
|
44
|
+
* - socks5 → socks dispatcher (undici Agent over a socks connector)
|
|
45
|
+
*
|
|
46
|
+
* Throws (fail loud) if a configured http/socks5 proxy cannot be built. It
|
|
47
|
+
* NEVER returns `undefined` (direct) as a fallback for a broken proxy.
|
|
48
|
+
*/
|
|
49
|
+
export function buildDispatcher(cfg: Config): Dispatcher | undefined {
|
|
50
|
+
const egress: Egress = cfg.egress;
|
|
51
|
+
switch (egress.mode) {
|
|
52
|
+
case 'direct':
|
|
53
|
+
return undefined;
|
|
54
|
+
case 'http':
|
|
55
|
+
try {
|
|
56
|
+
if (!egress.url) throw new Error('missing proxy url');
|
|
57
|
+
return new ProxyAgent(egress.url);
|
|
58
|
+
} catch (cause) {
|
|
59
|
+
throw new EgressError(
|
|
60
|
+
`egress http: could not build proxy for ${egress.url}`,
|
|
61
|
+
{cause},
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
case 'socks5':
|
|
65
|
+
try {
|
|
66
|
+
if (!egress.url) throw new Error('missing proxy url');
|
|
67
|
+
return socksFromUrl(egress.url);
|
|
68
|
+
} catch (cause) {
|
|
69
|
+
if (cause instanceof EgressError) throw cause;
|
|
70
|
+
throw new EgressError(
|
|
71
|
+
`egress socks5: could not build proxy for ${egress.url}`,
|
|
72
|
+
{cause},
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
default: {
|
|
76
|
+
const exhaustive: never = egress;
|
|
77
|
+
throw new EgressError(
|
|
78
|
+
`egress: unknown mode ${JSON.stringify(exhaustive)}`,
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** A WHATWG-compatible fetch bound to a specific egress dispatcher. */
|
|
85
|
+
export type EgressFetch = typeof globalThis.fetch;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Build an egress-bound WHATWG `fetch`: undici's `fetch` closed over the
|
|
89
|
+
* dispatcher from buildDispatcher(cfg). This is the `fetch` injected into
|
|
90
|
+
* distilly/fetch so distilly never has egress of its own. Same fail-loud
|
|
91
|
+
* guarantee: a broken proxy throws HERE (before any I/O), never goes un-proxied.
|
|
92
|
+
*/
|
|
93
|
+
export function createEgressFetch(cfg: Config): EgressFetch {
|
|
94
|
+
const dispatcher = buildDispatcher(cfg);
|
|
95
|
+
return ((input: RequestInfo | URL, init?: RequestInit) =>
|
|
96
|
+
undiciFetch(
|
|
97
|
+
input as never,
|
|
98
|
+
{
|
|
99
|
+
...((init ?? {}) as Record<string, unknown>),
|
|
100
|
+
dispatcher,
|
|
101
|
+
} as never,
|
|
102
|
+
)) as EgressFetch;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export type {Dispatcher};
|
|
106
|
+
export {Agent, ProxyAgent};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
// Extractor seam — turn a URL into clean, size-bounded markdown by calling
|
|
2
|
+
// distilly's NETWORKED `urlToMarkdown` (the `distilly/fetch` entrypoint),
|
|
3
|
+
// INJECTING webveil's egress-bound `fetch` as the only transport. distilly's
|
|
4
|
+
// network Rules (github/mdn/react.dev/vuejs.org) rewrite a matching URL to its
|
|
5
|
+
// raw `.md`/API source and fetch THAT over our egress — shorter, cleaner output;
|
|
6
|
+
// non-matching URLs run through distilly's pure core. See docs/adr/0001.
|
|
7
|
+
//
|
|
8
|
+
// THE HARD INVARIANT (load-bearing for anonymity): webveil ALWAYS injects its
|
|
9
|
+
// egress-bound `fetch` here and NEVER lets distilly use a global/default fetch.
|
|
10
|
+
// distilly throws if none is injected — the desired fail-loud. And the egress
|
|
11
|
+
// fetch itself throws (before any I/O) when a configured proxy is unbuildable
|
|
12
|
+
// (egress.ts), so a broken proxy can never become an un-proxied request.
|
|
13
|
+
//
|
|
14
|
+
// This is the DEFAULT extractor; a backend's own `/extract` (tavily-compat) may
|
|
15
|
+
// override it (wired in the core-fetch task).
|
|
16
|
+
|
|
17
|
+
import {urlToMarkdown as distillyUrlToMarkdown} from 'distilly/fetch';
|
|
18
|
+
import type {Config, FetchSize} from './config.js';
|
|
19
|
+
import {createEgressFetch, type EgressFetch} from './egress.js';
|
|
20
|
+
import type {FetchResult} from './backends/types.js';
|
|
21
|
+
|
|
22
|
+
/** The shape distilly's `urlToMarkdown` returns (the bits we surface). */
|
|
23
|
+
interface UrlToMarkdownResult {
|
|
24
|
+
markdown: string;
|
|
25
|
+
truncated: boolean;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** distilly's networked entrypoint, narrowed to what the seam injects/uses. */
|
|
29
|
+
type UrlToMarkdown = (
|
|
30
|
+
url: string | URL,
|
|
31
|
+
options: {fetch: EgressFetch; size?: FetchSize},
|
|
32
|
+
) => Promise<UrlToMarkdownResult>;
|
|
33
|
+
|
|
34
|
+
/** Per-call extractor options. */
|
|
35
|
+
export interface ExtractOptions {
|
|
36
|
+
/**
|
|
37
|
+
* Page-size budget for THIS call. Overrides the config's `fetchSize` when
|
|
38
|
+
* given. webveil's `s`/`m`/`l`/`f` preset maps STRAIGHT to distilly's `size`
|
|
39
|
+
* (the two enums are identical), so this is passed through verbatim.
|
|
40
|
+
*/
|
|
41
|
+
size?: FetchSize;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Seams the extractor's collaborators so it is testable WITHOUT real network or
|
|
46
|
+
* undici: tests inject a spy `urlToMarkdown` and/or a spy egress fetch to assert
|
|
47
|
+
* distilly is called with the egress fetch (never a global). Defaults wire the
|
|
48
|
+
* real `distilly/fetch` + `createEgressFetch`.
|
|
49
|
+
*/
|
|
50
|
+
export interface ExtractDeps {
|
|
51
|
+
/** distilly's networked `urlToMarkdown` (default: the real `distilly/fetch`). */
|
|
52
|
+
urlToMarkdown?: UrlToMarkdown;
|
|
53
|
+
/** Builds the egress-bound fetch from config (default: createEgressFetch). */
|
|
54
|
+
createEgressFetch?: (config: Config) => EgressFetch;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Extract a URL to clean, budget-bounded markdown via distilly over webveil's
|
|
59
|
+
* egress. Builds the egress-bound `fetch` (fail-loud on an unbuildable proxy),
|
|
60
|
+
* injects it into distilly's `urlToMarkdown`, maps the `s`/`m`/`l`/`f` preset to
|
|
61
|
+
* distilly's `size`, and surfaces distilly's `truncated`.
|
|
62
|
+
*
|
|
63
|
+
* @returns `{ url, markdown, truncated }` (a `FetchResult` without a `title`).
|
|
64
|
+
*/
|
|
65
|
+
export async function extract(
|
|
66
|
+
url: string,
|
|
67
|
+
config: Config,
|
|
68
|
+
options: ExtractOptions = {},
|
|
69
|
+
deps: ExtractDeps = {},
|
|
70
|
+
): Promise<FetchResult> {
|
|
71
|
+
const urlToMarkdown = deps.urlToMarkdown ?? distillyUrlToMarkdown;
|
|
72
|
+
const buildFetch = deps.createEgressFetch ?? createEgressFetch;
|
|
73
|
+
|
|
74
|
+
// Build the egress-bound fetch FIRST: a configured-but-unbuildable proxy
|
|
75
|
+
// throws here, before any network access (never an un-proxied request).
|
|
76
|
+
const fetch = buildFetch(config);
|
|
77
|
+
|
|
78
|
+
const size = options.size ?? config.fetchSize;
|
|
79
|
+
|
|
80
|
+
const {markdown, truncated} = await urlToMarkdown(url, {fetch, size});
|
|
81
|
+
return {url, markdown, truncated};
|
|
82
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// core fetch: the plain, framework-agnostic `fetch()` BOTH frontends (the incur
|
|
2
|
+
// CLI/MCP and the pi extension) call. Returns clean, size-bounded markdown with
|
|
3
|
+
// distilly's `truncated` flag.
|
|
4
|
+
//
|
|
5
|
+
// Flow (per URL): pick the content source (a backend's own `/extract`
|
|
6
|
+
// (tavily-compat) when the configured backend provides one, OTHERWISE the
|
|
7
|
+
// default distilly Extractor seam, urlToMarkdown over webveil's egress). The
|
|
8
|
+
// SSRF guard lives INSIDE the egress-bound fetch injected into distilly, so it
|
|
9
|
+
// covers distilly's rule-rewritten requests too (docs/adr/0001).
|
|
10
|
+
//
|
|
11
|
+
// LIST-READY INTERNALS (story 12): the work happens in `fetchAll(urls, …)`, a
|
|
12
|
+
// list-processing internal, so a future `web_batch_fetch` tool is a trivial add
|
|
13
|
+
// with no redesign. The public `fetch()` is a thin single-URL wrapper over it.
|
|
14
|
+
|
|
15
|
+
import {resolveConfig as defaultResolveConfig} from './config.js';
|
|
16
|
+
import type {Config, ResolveOptions} from './config.js';
|
|
17
|
+
import {createEgressFetch as defaultCreateEgressFetch} from './egress.js';
|
|
18
|
+
import type {EgressFetch} from './egress.js';
|
|
19
|
+
import {guardEgressFetch as defaultGuardEgressFetch} from './security.js';
|
|
20
|
+
import {createHttp as defaultCreateHttp} from './http.js';
|
|
21
|
+
import {buildDispatcher as defaultBuildDispatcher} from './egress.js';
|
|
22
|
+
import type {Dispatcher} from './egress.js';
|
|
23
|
+
import {extract as defaultExtract} from './extract.js';
|
|
24
|
+
import type {ExtractDeps} from './extract.js';
|
|
25
|
+
import {getBackend as defaultGetBackend} from './backends/registry.js';
|
|
26
|
+
import type {
|
|
27
|
+
Backend,
|
|
28
|
+
FetchOptions,
|
|
29
|
+
FetchResult,
|
|
30
|
+
Http,
|
|
31
|
+
} from './backends/types.js';
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Collaborators, seamed so the core is testable WITHOUT real config files,
|
|
35
|
+
* undici, network, or distilly: a test injects fakes to assert the
|
|
36
|
+
* backend-`/extract`-vs-distilly branch, the list path, and that the guarded
|
|
37
|
+
* egress fetch (never a global) is what reaches distilly. Defaults wire the real
|
|
38
|
+
* modules.
|
|
39
|
+
*/
|
|
40
|
+
export interface FetchDeps {
|
|
41
|
+
resolveConfig?: (options?: ResolveOptions) => Config;
|
|
42
|
+
getBackend?: (name: string, config: Config) => Backend;
|
|
43
|
+
buildDispatcher?: (config: Config) => Dispatcher | undefined;
|
|
44
|
+
createHttp?: (dispatcher: Dispatcher | undefined) => Http;
|
|
45
|
+
createEgressFetch?: (config: Config) => EgressFetch;
|
|
46
|
+
guardEgressFetch?: (fetch: EgressFetch, config: Config) => EgressFetch;
|
|
47
|
+
extract?: (
|
|
48
|
+
url: string,
|
|
49
|
+
config: Config,
|
|
50
|
+
options: {size?: Config['fetchSize']},
|
|
51
|
+
deps: ExtractDeps,
|
|
52
|
+
) => Promise<FetchResult>;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Per-call fetch options plus the config-resolution knobs (cwd/env/global). */
|
|
56
|
+
export interface FetchCoreOptions extends FetchOptions, ResolveOptions {}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Fetch a LIST of urls to clean, size-bounded markdown, in order. This is the
|
|
60
|
+
* list-ready internal (story 12): the single-URL `fetch()` below is a thin
|
|
61
|
+
* wrapper over it, so a future `web_batch_fetch` reuses this directly.
|
|
62
|
+
*
|
|
63
|
+
* Each url goes through the SAME content-source choice: a backend's own
|
|
64
|
+
* `/extract` (if the configured backend implements `fetch`) OR the default
|
|
65
|
+
* distilly Extractor with the GUARDED egress fetch injected.
|
|
66
|
+
*/
|
|
67
|
+
export async function fetchAll(
|
|
68
|
+
urls: string[],
|
|
69
|
+
options: FetchCoreOptions = {},
|
|
70
|
+
deps: FetchDeps = {},
|
|
71
|
+
): Promise<FetchResult[]> {
|
|
72
|
+
const resolveConfig = deps.resolveConfig ?? defaultResolveConfig;
|
|
73
|
+
const getBackend = deps.getBackend ?? defaultGetBackend;
|
|
74
|
+
const buildDispatcher = deps.buildDispatcher ?? defaultBuildDispatcher;
|
|
75
|
+
const createHttp = deps.createHttp ?? defaultCreateHttp;
|
|
76
|
+
const createEgressFetch = deps.createEgressFetch ?? defaultCreateEgressFetch;
|
|
77
|
+
const guardEgressFetch = deps.guardEgressFetch ?? defaultGuardEgressFetch;
|
|
78
|
+
const extract = deps.extract ?? defaultExtract;
|
|
79
|
+
|
|
80
|
+
const config = resolveConfig({
|
|
81
|
+
cwd: options.cwd,
|
|
82
|
+
env: options.env,
|
|
83
|
+
globalPath: options.globalPath,
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
const backend = getBackend(config.backend, config);
|
|
87
|
+
|
|
88
|
+
// A backend that provides its own `/extract` (tavily-compat) OVERRIDES the
|
|
89
|
+
// distilly Extractor; it is handed only the proxied http helper (built from
|
|
90
|
+
// the SAME dispatcher as the egress fetch), so it cannot bypass egress.
|
|
91
|
+
if (backend.fetch) {
|
|
92
|
+
const http = createHttp(buildDispatcher(config));
|
|
93
|
+
const backendFetch = backend.fetch.bind(backend);
|
|
94
|
+
return runAll(urls, (url) =>
|
|
95
|
+
backendFetch(url, http, {size: options.size, signal: options.signal}),
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Default path: distilly Extractor over webveil's egress. Build the
|
|
100
|
+
// egress-bound fetch ONCE, wrap it with the SSRF guard, and inject THAT into
|
|
101
|
+
// distilly (never a global fetch). The guard covers distilly's rule-rewritten
|
|
102
|
+
// requests too. A configured-but-unbuildable proxy throws at build time
|
|
103
|
+
// (fail-loud), before any I/O.
|
|
104
|
+
const guardedFetch = guardEgressFetch(createEgressFetch(config), config);
|
|
105
|
+
const extractDeps: ExtractDeps = {createEgressFetch: () => guardedFetch};
|
|
106
|
+
return runAll(urls, (url) =>
|
|
107
|
+
extract(url, config, {size: options.size}, extractDeps),
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Run a per-url worker over the list in order, collecting the results. */
|
|
112
|
+
async function runAll(
|
|
113
|
+
urls: string[],
|
|
114
|
+
work: (url: string) => Promise<FetchResult>,
|
|
115
|
+
): Promise<FetchResult[]> {
|
|
116
|
+
const out: FetchResult[] = [];
|
|
117
|
+
for (const url of urls) out.push(await work(url));
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Fetch ONE url to clean, size-bounded markdown (`{ markdown, truncated, … }`).
|
|
123
|
+
* A thin single-URL wrapper over the list-ready `fetchAll` (story 12).
|
|
124
|
+
*/
|
|
125
|
+
export async function fetch(
|
|
126
|
+
url: string,
|
|
127
|
+
options: FetchCoreOptions = {},
|
|
128
|
+
deps: FetchDeps = {},
|
|
129
|
+
): Promise<FetchResult> {
|
|
130
|
+
const [result] = await fetchAll([url], options, deps);
|
|
131
|
+
return result!;
|
|
132
|
+
}
|
package/src/core/http.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// http helper — the proxied `http` handed to backends. fetchJson / fetchText
|
|
2
|
+
// apply the egress dispatcher + a per-request timeout + abort. Distinct from the
|
|
3
|
+
// egress-bound WHATWG `fetch` (egress.ts), but bound to the SAME dispatcher, so
|
|
4
|
+
// a backend physically cannot bypass the configured egress.
|
|
5
|
+
|
|
6
|
+
import {type Dispatcher, fetch as undiciFetch} from 'undici';
|
|
7
|
+
import type {Http, HttpRequestOptions} from './backends/types.js';
|
|
8
|
+
|
|
9
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
10
|
+
|
|
11
|
+
async function request(
|
|
12
|
+
dispatcher: Dispatcher | undefined,
|
|
13
|
+
url: string,
|
|
14
|
+
options: HttpRequestOptions = {},
|
|
15
|
+
): Promise<Response> {
|
|
16
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
17
|
+
const controller = new AbortController();
|
|
18
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
19
|
+
if (options.signal)
|
|
20
|
+
options.signal.addEventListener('abort', () => controller.abort(), {
|
|
21
|
+
once: true,
|
|
22
|
+
});
|
|
23
|
+
try {
|
|
24
|
+
const res = await undiciFetch(url, {
|
|
25
|
+
method: options.method,
|
|
26
|
+
headers: options.headers,
|
|
27
|
+
body: options.body,
|
|
28
|
+
signal: controller.signal,
|
|
29
|
+
dispatcher,
|
|
30
|
+
} as never);
|
|
31
|
+
return res as unknown as Response;
|
|
32
|
+
} finally {
|
|
33
|
+
clearTimeout(timer);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Build the proxied http helper over a given dispatcher. Both methods throw on a
|
|
39
|
+
* non-2xx response so a backend never silently consumes an error body.
|
|
40
|
+
*/
|
|
41
|
+
export function createHttp(dispatcher: Dispatcher | undefined): Http {
|
|
42
|
+
return {
|
|
43
|
+
async fetchJson<T = unknown>(
|
|
44
|
+
url: string,
|
|
45
|
+
options?: HttpRequestOptions,
|
|
46
|
+
): Promise<T> {
|
|
47
|
+
const res = await request(dispatcher, url, options);
|
|
48
|
+
if (!res.ok)
|
|
49
|
+
throw new Error(`http ${res.status} ${res.statusText} for ${url}`);
|
|
50
|
+
return (await res.json()) as T;
|
|
51
|
+
},
|
|
52
|
+
async fetchText(
|
|
53
|
+
url: string,
|
|
54
|
+
options?: HttpRequestOptions,
|
|
55
|
+
): Promise<string> {
|
|
56
|
+
const res = await request(dispatcher, url, options);
|
|
57
|
+
if (!res.ok)
|
|
58
|
+
throw new Error(`http ${res.status} ${res.statusText} for ${url}`);
|
|
59
|
+
return await res.text();
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// core search — the plain, framework-agnostic `search()` BOTH frontends (the
|
|
2
|
+
// incur CLI/MCP and the pi extension) call. It owns the wiring and the
|
|
3
|
+
// caller-facing post-processing; the per-source parsing lives in the backend.
|
|
4
|
+
//
|
|
5
|
+
// Flow: resolve config → build the egress dispatcher → bind the proxied `http`
|
|
6
|
+
// helper to it → select the backend from the registry → call the backend with
|
|
7
|
+
// ONLY that proxied helper → normalize (dedup + clamp) the SearchResult[].
|
|
8
|
+
//
|
|
9
|
+
// The egress invariant (docs/adr/0001): the backend is handed only the
|
|
10
|
+
// dispatcher-bound `http` helper, so it physically cannot reach a global fetch
|
|
11
|
+
// and bypass the configured egress. A configured-but-unbuildable proxy throws at
|
|
12
|
+
// buildDispatcher (fail-loud), never silently un-proxied.
|
|
13
|
+
|
|
14
|
+
import {resolveConfig as defaultResolveConfig} from './config.js';
|
|
15
|
+
import type {Config, ResolveOptions} from './config.js';
|
|
16
|
+
import {buildDispatcher as defaultBuildDispatcher} from './egress.js';
|
|
17
|
+
import type {Dispatcher} from './egress.js';
|
|
18
|
+
import {createHttp as defaultCreateHttp} from './http.js';
|
|
19
|
+
import {getBackend as defaultGetBackend} from './backends/registry.js';
|
|
20
|
+
import type {Http, SearchOptions, SearchResult} from './backends/types.js';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Default cap on returned results when the caller does not pass `maxResults`.
|
|
24
|
+
* Keeps an agent's context small by default; a caller can raise/lower it per
|
|
25
|
+
* call. (Recorded decision: there is no configured default, so the core sets
|
|
26
|
+
* one; see the task's Decisions block.)
|
|
27
|
+
*/
|
|
28
|
+
const DEFAULT_MAX_RESULTS = 10;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Collaborators, seamed so the core is testable WITHOUT real config files,
|
|
32
|
+
* undici, or network: a test injects a fake `getBackend`/`createHttp` to assert
|
|
33
|
+
* the backend is handed only the proxied helper, and a fake backend returning
|
|
34
|
+
* duplicate/over-limit hits to assert dedup + clamp. Defaults wire the real
|
|
35
|
+
* config/egress/http/registry modules.
|
|
36
|
+
*/
|
|
37
|
+
export interface SearchDeps {
|
|
38
|
+
resolveConfig?: (options?: ResolveOptions) => Config;
|
|
39
|
+
buildDispatcher?: (config: Config) => Dispatcher | undefined;
|
|
40
|
+
createHttp?: (dispatcher: Dispatcher | undefined) => Http;
|
|
41
|
+
getBackend?: (
|
|
42
|
+
name: string,
|
|
43
|
+
config: Config,
|
|
44
|
+
) => {
|
|
45
|
+
search: (
|
|
46
|
+
query: string,
|
|
47
|
+
http: Http,
|
|
48
|
+
options?: SearchOptions,
|
|
49
|
+
) => Promise<SearchResult[]>;
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Per-call search options plus the config-resolution knobs (cwd/env/global). */
|
|
54
|
+
export interface SearchCoreOptions extends SearchOptions, ResolveOptions {}
|
|
55
|
+
|
|
56
|
+
/** Dedup by url (the hit's identity), preserving first-seen order. */
|
|
57
|
+
function dedup(results: SearchResult[]): SearchResult[] {
|
|
58
|
+
const seen = new Set<string>();
|
|
59
|
+
const out: SearchResult[] = [];
|
|
60
|
+
for (const r of results) {
|
|
61
|
+
if (seen.has(r.url)) continue;
|
|
62
|
+
seen.add(r.url);
|
|
63
|
+
out.push(r);
|
|
64
|
+
}
|
|
65
|
+
return out;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Search the configured backend over the configured egress and return
|
|
70
|
+
* normalized `SearchResult[]` (deduped by url, then clamped to `maxResults`).
|
|
71
|
+
*
|
|
72
|
+
* Dedup runs BEFORE the clamp so the caller gets up to `maxResults` UNIQUE hits,
|
|
73
|
+
* not a window that duplicates eat into; for the same reason the backend is NOT
|
|
74
|
+
* asked to pre-clamp (only the abort signal is forwarded).
|
|
75
|
+
*/
|
|
76
|
+
export async function search(
|
|
77
|
+
query: string,
|
|
78
|
+
options: SearchCoreOptions = {},
|
|
79
|
+
deps: SearchDeps = {},
|
|
80
|
+
): Promise<SearchResult[]> {
|
|
81
|
+
const resolveConfig = deps.resolveConfig ?? defaultResolveConfig;
|
|
82
|
+
const buildDispatcher = deps.buildDispatcher ?? defaultBuildDispatcher;
|
|
83
|
+
const createHttp = deps.createHttp ?? defaultCreateHttp;
|
|
84
|
+
const getBackend = deps.getBackend ?? defaultGetBackend;
|
|
85
|
+
|
|
86
|
+
const config = resolveConfig({
|
|
87
|
+
cwd: options.cwd,
|
|
88
|
+
env: options.env,
|
|
89
|
+
globalPath: options.globalPath,
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
// Build the dispatcher FIRST: a configured-but-unbuildable proxy throws here,
|
|
93
|
+
// before any network access (never an un-proxied request).
|
|
94
|
+
const dispatcher = buildDispatcher(config);
|
|
95
|
+
const http = createHttp(dispatcher);
|
|
96
|
+
|
|
97
|
+
const backend = getBackend(config.backend, config);
|
|
98
|
+
// Hand the backend ONLY the proxied helper (no maxResults: dedup happens
|
|
99
|
+
// here, over the full set, so the clamp below is over UNIQUE results).
|
|
100
|
+
const raw = await backend.search(query, http, {signal: options.signal});
|
|
101
|
+
|
|
102
|
+
const maxResults = options.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
103
|
+
return dedup(raw).slice(0, maxResults);
|
|
104
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// SSRF guard: the security seam wrapped AROUND the egress-bound `fetch`, so it
|
|
2
|
+
// covers BOTH webveil's own GETs AND distilly's rule-rewritten requests (see
|
|
3
|
+
// docs/adr/0001: the guard lives inside the egress fetch). Adapts the range
|
|
4
|
+
// classification + DNS-resolve approach of leing2021/pi-search's `security.ts`.
|
|
5
|
+
//
|
|
6
|
+
// THE RELAXATION RULE (load-bearing, recorded in the task's Decisions): the
|
|
7
|
+
// guard BLOCKS private/loopback/link-local/etc. addresses on DIRECT egress, and
|
|
8
|
+
// RELAXES ENTIRELY under a proxy egress (`http` | `socks5`). Tor/Mullvad
|
|
9
|
+
// legitimately reach private-looking addresses (e.g. `10.64.0.1`), AND a local
|
|
10
|
+
// DNS lookup for a proxied request would itself be a deanonymizing leak, so
|
|
11
|
+
// under a proxy we neither block nor resolve locally; the proxy owns egress.
|
|
12
|
+
|
|
13
|
+
import {lookup} from 'node:dns/promises';
|
|
14
|
+
import {isIP} from 'node:net';
|
|
15
|
+
import type {Config} from './config.js';
|
|
16
|
+
import type {EgressFetch} from './egress.js';
|
|
17
|
+
|
|
18
|
+
/** Thrown when the SSRF guard refuses a request to a private/blocked address. */
|
|
19
|
+
export class SsrfError extends Error {
|
|
20
|
+
constructor(message: string) {
|
|
21
|
+
super(message);
|
|
22
|
+
this.name = 'SsrfError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** A proxy egress owns egress + DNS, so the local SSRF guard relaxes for it. */
|
|
27
|
+
function egressIsProxy(config: Config): boolean {
|
|
28
|
+
return config.egress.mode === 'http' || config.egress.mode === 'socks5';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Is this LITERAL IP private / non-public? Covers the ranges that must never be
|
|
33
|
+
* reachable from a direct-egress web fetch:
|
|
34
|
+
* IPv4: 0.0.0.0/8, 10/8 (RFC1918), 127/8 (loopback), 169.254/16 (link-local,
|
|
35
|
+
* incl. the 169.254.169.254 cloud metadata endpoint), 172.16/12 (RFC1918),
|
|
36
|
+
* 192.168/16 (RFC1918), 100.64/10 (CGNAT), 192.0.0/24, 192.0.2/24,
|
|
37
|
+
* 198.18/15, 198.51.100/24, 203.0.113/24, 224/4 (multicast), 240/4
|
|
38
|
+
* (reserved).
|
|
39
|
+
* IPv6: ::1 (loopback), :: (unspecified), fc00::/7 (ULA), fe80::/10
|
|
40
|
+
* (link-local), ff00::/8 (multicast), plus IPv4-mapped (::ffff:a.b.c.d,
|
|
41
|
+
* re-checked as IPv4). Default-deny: anything outside global unicast
|
|
42
|
+
* (2000::/3) is treated as non-public.
|
|
43
|
+
*/
|
|
44
|
+
export function isPrivateIp(ip: string): boolean {
|
|
45
|
+
const kind = isIP(ip);
|
|
46
|
+
if (kind === 4) return isPrivateIpv4(ip);
|
|
47
|
+
if (kind === 6) return isPrivateIpv6(ip);
|
|
48
|
+
return false; // not a literal IP; hostname handling resolves it first
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function isPrivateIpv4(ip: string): boolean {
|
|
52
|
+
const parts = ip.split('.').map((p) => Number(p));
|
|
53
|
+
if (
|
|
54
|
+
parts.length !== 4 ||
|
|
55
|
+
parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)
|
|
56
|
+
)
|
|
57
|
+
return true; // malformed → treat as non-public (fail closed)
|
|
58
|
+
const [a, b] = parts as [number, number, number, number];
|
|
59
|
+
if (a === 0 || a === 10 || a === 127) return true;
|
|
60
|
+
if (a === 169 && b === 254) return true; // link-local incl. cloud metadata
|
|
61
|
+
if (a === 172 && b >= 16 && b <= 31) return true; // 172.16/12
|
|
62
|
+
if (a === 192 && b === 168) return true; // 192.168/16
|
|
63
|
+
if (a === 100 && b >= 64 && b <= 127) return true; // 100.64/10 CGNAT
|
|
64
|
+
if (a === 192 && b === 0) return true; // 192.0.0/24 + 192.0.2/24
|
|
65
|
+
if (a === 198 && (b === 18 || b === 19)) return true; // 198.18/15 benchmark
|
|
66
|
+
if (a === 198 && b === 51) return true; // 198.51.100/24 TEST-NET-2
|
|
67
|
+
if (a === 203 && b === 0) return true; // 203.0.113/24 TEST-NET-3
|
|
68
|
+
if (a >= 224) return true; // 224/4 multicast + 240/4 reserved
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function isPrivateIpv6(ip: string): boolean {
|
|
73
|
+
const lower = ip.toLowerCase();
|
|
74
|
+
if (lower === '::1' || lower === '::') return true; // loopback / unspecified
|
|
75
|
+
// IPv4-mapped (::ffff:a.b.c.d): re-check the embedded IPv4.
|
|
76
|
+
const mapped = lower.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
|
|
77
|
+
if (mapped) return isPrivateIpv4(mapped[1]!);
|
|
78
|
+
const head = lower.split(':')[0] ?? '';
|
|
79
|
+
const first = parseInt(head || '0', 16);
|
|
80
|
+
if (Number.isNaN(first)) return true; // fail closed on anything unparseable
|
|
81
|
+
if ((first & 0xfe00) === 0xfc00) return true; // fc00::/7 ULA
|
|
82
|
+
if ((first & 0xffc0) === 0xfe80) return true; // fe80::/10 link-local
|
|
83
|
+
if ((first & 0xff00) === 0xff00) return true; // ff00::/8 multicast
|
|
84
|
+
// Default-deny: only global unicast 2000::/3 is public.
|
|
85
|
+
return (first & 0xe000) !== 0x2000;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Assert a URL is safe to fetch under THIS config's egress. Under a proxy egress
|
|
90
|
+
* it always passes (the proxy owns egress + DNS). Under direct egress it rejects
|
|
91
|
+
* a literal private IP and, for a hostname, resolves it locally and rejects if it
|
|
92
|
+
* maps to a private IP (so a name pointing at 127.0.0.1 / metadata is caught).
|
|
93
|
+
*/
|
|
94
|
+
export async function assertPublicUrl(
|
|
95
|
+
url: string,
|
|
96
|
+
config: Config,
|
|
97
|
+
): Promise<void> {
|
|
98
|
+
if (egressIsProxy(config)) return; // proxy owns egress + DNS; relax entirely
|
|
99
|
+
let parsed: URL;
|
|
100
|
+
try {
|
|
101
|
+
parsed = new URL(url);
|
|
102
|
+
} catch {
|
|
103
|
+
throw new SsrfError(`webveil SSRF: malformed url ${url}`);
|
|
104
|
+
}
|
|
105
|
+
const host = parsed.hostname.replace(/^\[|\]$/g, ''); // strip IPv6 brackets
|
|
106
|
+
if (isIP(host)) {
|
|
107
|
+
if (isPrivateIp(host))
|
|
108
|
+
throw new SsrfError(`webveil SSRF: blocked private address ${host}`);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
// A hostname: resolve it locally (safe on direct egress) and check every
|
|
112
|
+
// address it maps to, so a name pointing at a private IP is also blocked.
|
|
113
|
+
const addrs = await lookup(host, {all: true});
|
|
114
|
+
for (const {address} of addrs)
|
|
115
|
+
if (isPrivateIp(address))
|
|
116
|
+
throw new SsrfError(
|
|
117
|
+
`webveil SSRF: ${host} resolves to private address ${address}`,
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Wrap an egress-bound `fetch` with the SSRF guard. The returned fetch checks
|
|
123
|
+
* EVERY request URL (so it covers distilly's rule-rewritten requests too, not
|
|
124
|
+
* only webveil's own GET) before delegating to the underlying egress fetch.
|
|
125
|
+
* This is what `core.fetch()` injects into distilly. See docs/adr/0001.
|
|
126
|
+
*/
|
|
127
|
+
export function guardEgressFetch(
|
|
128
|
+
fetch: EgressFetch,
|
|
129
|
+
config: Config,
|
|
130
|
+
): EgressFetch {
|
|
131
|
+
return (async (input: RequestInfo | URL, init?: RequestInit) => {
|
|
132
|
+
const url =
|
|
133
|
+
typeof input === 'string'
|
|
134
|
+
? input
|
|
135
|
+
: input instanceof URL
|
|
136
|
+
? input.href
|
|
137
|
+
: input.url;
|
|
138
|
+
await assertPublicUrl(url, config);
|
|
139
|
+
return fetch(input as never, init as never);
|
|
140
|
+
}) as EgressFetch;
|
|
141
|
+
}
|