webveil 0.0.0 → 0.1.1

package/src/core/backends/tavily-compat.ts

@@ -0,0 +1,156 @@
+// tavily-compat backend — a generic Tavily-shaped client (POST `/search` and an
+// optional POST `/extract`) selected purely by `baseUrl`, so it covers
+// orio-search / searcharvester / agent-search and any other Tavily-API-shaped
+// instance. Both endpoints go THROUGH the handed `http` helper (never a direct
+// fetch, so egress is not bypassable). `/search` normalizes to SearchResult[];
+// `/extract` is exposed as the optional `Backend.fetch` a later task uses to
+// override the distilly Extractor.
+//
+// Auth: a Bearer header is sent only when an apiKey is configured. The covered
+// self-hosted instances are typically keyless, so a missing key is normal, not
+// an error.
+
+import type {Config} from '../config.js';
+import type {
+	Backend,
+	FetchOptions,
+	FetchResult,
+	Http,
+	HttpRequestOptions,
+	SearchOptions,
+	SearchResult,
+} from './types.js';
+
+/** One entry in a Tavily `/search` `results` array (subset we use). */
+interface TavilySearchHit {
+	title?: unknown;
+	url?: unknown;
+	content?: unknown;
+}
+
+/** The Tavily `/search` response (subset we use). */
+interface TavilySearchResponse {
+	results?: TavilySearchHit[];
+}
+
+/** One entry in a Tavily `/extract` `results` array (subset we use). */
+interface TavilyExtractHit {
+	url?: unknown;
+	raw_content?: unknown;
+}
+
+/** One entry in a Tavily `/extract` `failed_results` array (subset we use). */
+interface TavilyExtractFailure {
+	url?: unknown;
+	error?: unknown;
+}
+
+/** The Tavily `/extract` response (subset we use). */
+interface TavilyExtractResponse {
+	results?: TavilyExtractHit[];
+	failed_results?: TavilyExtractFailure[];
+}
+
+function str(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Normalize one Tavily search hit; drop entries without a usable url + title. */
+function toResult(hit: TavilySearchHit): SearchResult | undefined {
+	const url = str(hit.url);
+	const title = str(hit.title);
+	if (!url || !title) return undefined;
+	const snippet = str(hit.content);
+	return snippet ? {title, url, snippet} : {title, url};
+}
+
+/** Resolve an endpoint path against the instance baseUrl. */
+function endpoint(baseUrl: string, path: string): string {
+	return new URL(
+		path,
+		baseUrl.endsWith('/') ? baseUrl : baseUrl + '/',
+	).toString();
+}
+
+/**
+ * Build a Tavily-compat backend bound to the configured instance. The returned
+ * backend only ever touches the network via the injected `http` helper. A Bearer
+ * header is added only when an apiKey is set (the covered instances are usually
+ * keyless).
+ */
+export function createTavilyCompatBackend(config: Config): Backend {
+	const baseUrl = config.baseUrl;
+	const apiKey = config.apiKey;
+
+	function headers(): Record<string, string> {
+		const h: Record<string, string> = {
+			'content-type': 'application/json',
+			accept: 'application/json',
+		};
+		if (apiKey) h.authorization = `Bearer ${apiKey}`;
+		return h;
+	}
+
+	function post(
+		path: string,
+		payload: unknown,
+		signal?: AbortSignal,
+	): HttpRequestOptions {
+		return {
+			method: 'POST',
+			headers: headers(),
+			body: JSON.stringify(payload),
+			signal,
+		};
+	}
+
+	return {
+		async search(
+			query: string,
+			http: Http,
+			options: SearchOptions = {},
+		): Promise<SearchResult[]> {
+			const payload: Record<string, unknown> = {query};
+			if (options.maxResults !== undefined)
+				payload.max_results = options.maxResults;
+			const body = await http.fetchJson<TavilySearchResponse>(
+				endpoint(baseUrl, 'search'),
+				post('search', payload, options.signal),
+			);
+			const results = Array.isArray(body.results) ? body.results : [];
+			const normalized = results
+				.map(toResult)
+				.filter((r): r is SearchResult => r !== undefined);
+			return options.maxResults !== undefined
+				? normalized.slice(0, options.maxResults)
+				: normalized;
+		},
+
+		async fetch(
+			url: string,
+			http: Http,
+			options: FetchOptions = {},
+		): Promise<FetchResult> {
+			// Tavily `/extract` has no `s/m/l/f` size knob (it has `format` /
+			// `extract_depth` instead); always request markdown. The default
+			// distilly Extractor owns webveil's size presets.
+			const body = await http.fetchJson<TavilyExtractResponse>(
+				endpoint(baseUrl, 'extract'),
+				post('extract', {urls: url, format: 'markdown'}, options.signal),
+			);
+			const failure = (body.failed_results ?? []).find(
+				(f) => str(f.url) === url,
+			);
+			if (failure)
+				throw new Error(
+					`tavily-compat: /extract failed for ${url}: ${str(failure.error) ?? 'unknown error'}`,
+				);
+			const hit = (body.results ?? [])[0];
+			const markdown = hit ? str(hit.raw_content) : undefined;
+			if (markdown === undefined)
+				throw new Error(`tavily-compat: no extract result for ${url}`);
+			// Tavily `/extract` returns no `truncated` flag and no page title.
+			return {url, markdown, truncated: false};
+		},
+	};
+}

package/src/core/backends/types.ts

@@ -0,0 +1,61 @@
+// backend seam — the contract every result source (searxng | tavily-compat |
+// custom) implements. A Backend is HANDED a proxied `http` helper (bound to the
+// configured egress dispatcher) so it physically cannot bypass the egress.
+
+/** A single search hit. */
+export interface SearchResult {
+	title: string;
+	url: string;
+	snippet?: string;
+}
+
+/** A fetched, extracted page as budget-bounded markdown. */
+export interface FetchResult {
+	url: string;
+	title?: string;
+	markdown: string;
+	truncated: boolean;
+}
+
+export interface SearchOptions {
+	maxResults?: number;
+	signal?: AbortSignal;
+}
+
+export interface FetchOptions {
+	size?: 's' | 'm' | 'l' | 'f';
+	signal?: AbortSignal;
+}
+
+/** Options the http helper accepts for a single request. */
+export interface HttpRequestOptions {
+	method?: string;
+	headers?: Record<string, string>;
+	body?: string;
+	/** Per-request timeout in ms (the helper aborts past this). */
+	timeoutMs?: number;
+	signal?: AbortSignal;
+}
+
+/**
+ * The proxied http helper handed to backends. Both methods route through the
+ * egress dispatcher; a backend never gets un-proxied transport of its own.
+ */
+export interface Http {
+	fetchJson<T = unknown>(url: string, options?: HttpRequestOptions): Promise<T>;
+	fetchText(url: string, options?: HttpRequestOptions): Promise<string>;
+}
+
+/**
+ * A result/content source. `search` is required; `fetch` is optional (a backend
+ * may override the default distilly Extractor with its own `/extract`). Both are
+ * given the proxied `http` helper so they cannot escape the configured egress.
+ */
+export interface Backend {
+	search(
+		query: string,
+		http: Http,
+		options?: SearchOptions,
+	): Promise<SearchResult[]>;
+	fetch?(url: string, http: Http, options?: FetchOptions): Promise<FetchResult>;
+}

package/src/core/baseurl.ts

@@ -0,0 +1,104 @@
+// baseUrl transport parsing — the small, transport-AWARE helper that classifies
+// a resolved `baseUrl` into either a normal TCP HTTP base or a Unix-domain-socket
+// form, and (for the socket form) rewrites it into a synthetic `http://localhost`
+// base that the transport-UNAWARE backends can build their request URL on top of.
+//
+// WHY THIS LIVES HERE (and not in egress.ts): the egress dispatcher
+// (`buildDispatcher`) is built ONCE from config and SHARED by both the backend
+// hop (search.ts) AND the arbitrary-public-URL fetch (fetch.ts). Binding a socket
+// `Agent` into that shared dispatcher would route every `web_fetch` into the
+// SearXNG socket. So the socket transport is scoped to the BACKEND `baseUrl` hop
+// only: search.ts asks this helper to translate the baseUrl, gets back a real
+// `http://localhost…` base (which the searxng backend's `new URL('search', base)`
+// still works on, unchanged) plus the per-hop socket `Agent`, and leaves the
+// fetch/SSRF egress path untouched.
+//
+// GRAMMAR (recorded decision, see the task's done record):
+//   unix:<socketPath>[:<httpPath>]
+//     - `<socketPath>`: the absolute path to the uWSGI Unix domain socket
+//       (e.g. /usr/local/searxng/run/socket). Must not contain a `:` (the parse
+//       splits on the FIRST `:` after the `unix:` scheme; conventional socket
+//       paths never contain a colon).
+//     - `<httpPath>`: OPTIONAL base path the SearXNG app is mounted under,
+//       defaulting to `/`. It is the SAME thing the TCP `baseUrl` encodes as its
+//       path: the backend appends `search` to it (`new URL('search', base + '/')`),
+//       so the install default is just `unix:/usr/local/searxng/run/socket` (the
+//       backend then requests `/search`). A non-root mount is `…/socket:/searxng`.
+//   A raw `unix:…` string is NOT a valid base for `new URL('search', …)`, so this
+//   translation MUST run BEFORE the backend builds its URL.
+
+import {Agent, type Dispatcher} from 'undici';
+
+/** The `unix:` scheme prefix this helper recognizes on a `baseUrl`. */
+const UNIX_PREFIX = 'unix:';
+
+/** A parsed Unix-socket baseUrl: the socket file path + the app's base path. */
+export interface UnixBaseUrl {
+	socketPath: string;
+	/** The app's base path (mount point); the backend appends `search` to it. */
+	httpPath: string;
+}
+
+/** Is this resolved `baseUrl` a Unix-domain-socket form (`unix:…`)? */
+export function isUnixBaseUrl(baseUrl: string): boolean {
+	return baseUrl.startsWith(UNIX_PREFIX);
+}
+
+/**
+ * Parse a `unix:<socketPath>[:<httpPath>]` baseUrl into `{socketPath, httpPath}`.
+ * Splits on the FIRST `:` after the `unix:` scheme (socket paths conventionally
+ * carry no colon); an absent/empty `<httpPath>` defaults to `/`.
+ *
+ * Throws if the socket path is empty (there is nothing to connect to).
+ */
+export function parseUnixBaseUrl(baseUrl: string): UnixBaseUrl {
+	const rest = baseUrl.slice(UNIX_PREFIX.length);
+	const sep = rest.indexOf(':');
+	const socketPath = sep === -1 ? rest : rest.slice(0, sep);
+	const rawHttpPath = sep === -1 ? '' : rest.slice(sep + 1);
+	if (!socketPath)
+		throw new Error(
+			`webveil: malformed unix baseUrl ${JSON.stringify(baseUrl)} — ` +
+				`expected unix:<socketPath>[:<httpPath>] with a non-empty socket path`,
+		);
+	const httpPath = rawHttpPath
+		? rawHttpPath.startsWith('/')
+			? rawHttpPath
+			: '/' + rawHttpPath
+		: '/';
+	return {socketPath, httpPath};
+}
+
+/**
+ * The result of resolving a `baseUrl` for the BACKEND hop: the (possibly
+ * rewritten) HTTP base the backend builds its request URL on, plus an OPTIONAL
+ * undici `Dispatcher` to carry that hop. For a normal TCP baseUrl the dispatcher
+ * is `undefined` (the caller uses the config-wide egress dispatcher); for a
+ * `unix:` baseUrl it is a socket-bound `Agent` and the base is a synthetic
+ * `http://localhost<httpPath>`.
+ */
+export interface BackendTransport {
+	baseUrl: string;
+	dispatcher?: Dispatcher;
+}
+
+/**
+ * Resolve a `baseUrl` into a backend-hop transport. For a `unix:` baseUrl this
+ * builds a socket-bound `Agent({connect:{socketPath}})` and a synthetic
+ * `http://localhost<httpPath>` base (the URL host is irrelevant to routing — the
+ * socket decides — and only becomes the `Host` header). For any other baseUrl it
+ * returns the baseUrl unchanged with NO dispatcher (the caller keeps using the
+ * shared config-wide egress dispatcher).
+ *
+ * NOTE: this is the BACKEND-hop transport only. It is never bound into the
+ * shared egress dispatcher, so `web_fetch`/SSRF egress is unaffected.
+ */
+export function resolveBackendTransport(baseUrl: string): BackendTransport {
+	if (!isUnixBaseUrl(baseUrl)) return {baseUrl};
+	const {socketPath, httpPath} = parseUnixBaseUrl(baseUrl);
+	const dispatcher = new Agent({connect: {socketPath}});
+	return {
+		baseUrl: `http://localhost${httpPath === '/' ? '' : httpPath}`,
+		dispatcher,
+	};
+}

package/src/core/config.ts

@@ -0,0 +1,106 @@
+// config seam — per-folder resolution. Precedence (highest wins):
+//   env > nearest .pi/webveil.json (walking up from cwd) > global
+//   ~/.pi/agent/webveil.json > defaults.
+// "Per folder = per account/egress." Each layer is a partial; later (lower)
+// layers fill gaps the higher layers leave.
+
+import {readFileSync} from 'node:fs';
+import {homedir} from 'node:os';
+import {dirname, join, parse} from 'node:path';
+
+/** How outbound HTTP leaves the machine. See egress.ts. */
+export type Egress =
+	| {mode: 'direct'}
+	| {mode: 'http'; url: string}
+	| {mode: 'socks5'; url: string};
+
+/** Page-size budget preset for fetch (passed through to distilly). */
+export type FetchSize = 's' | 'm' | 'l' | 'f';
+
+/** The fully-resolved config every webveil module consumes. */
+export interface Config {
+	backend: string;
+	baseUrl: string;
+	apiKey?: string;
+	egress: Egress;
+	fetchSize: FetchSize;
+}
+
+/** A config file / env layer: any subset of the resolved shape. */
+export type PartialConfig = Partial<Config>;
+
+export interface ResolveOptions {
+	/** Directory the per-folder walk starts from. Defaults to process.cwd(). */
+	cwd?: string;
+	/** Environment to read overrides from. Defaults to process.env. */
+	env?: Record<string, string | undefined>;
+	/**
+	 * Path to the global config file. Defaults to ~/.pi/agent/webveil.json.
+	 * Tests point this at a temp dir to isolate the real home directory.
+	 */
+	globalPath?: string;
+}
+
+const DEFAULTS: Config = {
+	backend: 'searxng',
+	baseUrl: 'http://127.0.0.1:8080',
+	egress: {mode: 'direct'},
+	fetchSize: 'm',
+};
+
+const PROJECT_FILE = join('.pi', 'webveil.json');
+
+function readJson(path: string): PartialConfig | undefined {
+	let text: string;
+	try {
+		text = readFileSync(path, 'utf8');
+	} catch {
+		return undefined; // absent file is fine; missing layers are expected
+	}
+	return JSON.parse(text) as PartialConfig;
+}
+
+/** The nearest `.pi/webveil.json` walking up from `cwd` (first found wins). */
+function readProjectChain(cwd: string): PartialConfig | undefined {
+	let dir = cwd;
+	const {root} = parse(dir);
+	for (;;) {
+		const found = readJson(join(dir, PROJECT_FILE));
+		if (found) return found;
+		if (dir === root) return undefined;
+		dir = dirname(dir);
+	}
+}
+
+function readEnv(env: Record<string, string | undefined>): PartialConfig {
+	const layer: PartialConfig = {};
+	if (env.WEBVEIL_BACKEND) layer.backend = env.WEBVEIL_BACKEND;
+	if (env.WEBVEIL_BASE_URL) layer.baseUrl = env.WEBVEIL_BASE_URL;
+	if (env.WEBVEIL_API_KEY) layer.apiKey = env.WEBVEIL_API_KEY;
+	if (env.WEBVEIL_FETCH_SIZE)
+		layer.fetchSize = env.WEBVEIL_FETCH_SIZE as FetchSize;
+	const mode = env.WEBVEIL_EGRESS;
+	if (mode === 'direct') layer.egress = {mode: 'direct'};
+	else if (mode === 'http' || mode === 'socks5')
+		layer.egress = {mode, url: env.WEBVEIL_EGRESS_URL ?? ''};
+	return layer;
+}
+
+/**
+ * Resolve the effective config. Higher-precedence layers override lower ones,
+ * key by key: env > project chain > global file > defaults.
+ */
+export function resolveConfig(options: ResolveOptions = {}): Config {
+	const cwd = options.cwd ?? process.cwd();
+	const env = options.env ?? process.env;
+	const globalPath =
+		options.globalPath ?? join(homedir(), '.pi', 'agent', 'webveil.json');
+
+	const layers: PartialConfig[] = [
+		DEFAULTS,
+		readJson(globalPath) ?? {},
+		readProjectChain(cwd) ?? {},
+		readEnv(env),
+	];
+	return Object.assign({}, ...layers) as Config;
+}

package/src/core/egress.ts

@@ -0,0 +1,134 @@
+// egress seam — how outbound HTTP leaves the machine. Yields TWO artifacts off
+// the SAME undici dispatcher: the proxied `http` helper (see http.ts, handed to
+// backends) and an egress-bound WHATWG `fetch` (injected into distilly/fetch).
+//
+// CRITICAL anonymity invariant (docs/adr/0001): egress is fail-loud. A
+// configured proxy that cannot be built MUST throw — it must NEVER silently
+// fall back to un-proxied (direct) transport.
+
+import {Agent, type Dispatcher, ProxyAgent, fetch as undiciFetch} from 'undici';
+import {socksDispatcher} from 'fetch-socks';
+import type {Config, Egress} from './config.js';
+import {isUnixBaseUrl} from './baseurl.js';
+
+/** Thrown when a configured egress proxy cannot be built. Never swallowed. */
+export class EgressError extends Error {
+	constructor(message: string, options?: {cause?: unknown}) {
+		super(message, options);
+		this.name = 'EgressError';
+	}
+}
+
+/**
+ * Fail-loud guard for the false-confidence combo: a `unix:` (local-socket)
+ * backend `baseUrl` configured with a NON-direct egress (`http`/`socks5`). A
+ * Unix socket is inherently local, so proxying that hop is the same fake-
+ * anonymity footgun as proxying a loopback TCP baseUrl: webveil would route a
+ * pointless local call through the proxy while the backend (SearXNG) crawls the
+ * public web from the real IP, OUTSIDE webveil's egress. Refuse it and point at
+ * the real fix.
+ *
+ * OVERLAP SEAM (recorded): this is the loopback false-confidence family. The
+ * sibling task `fail-loud-on-proxied-loopback-backend` adds the broader guard
+ * for loopback TCP baseUrls (127.0.0.0/8, ::1, localhost). When it lands, fold
+ * THIS `unix:`-is-loopback-equivalent case into that single guard instead of
+ * keeping a parallel check here.
+ */
+export function assertEgressAllowsBaseUrl(cfg: Config): void {
+	if (cfg.egress.mode === 'direct') return;
+	if (isUnixBaseUrl(cfg.baseUrl))
+		throw new EgressError(
+			`egress ${cfg.egress.mode}: a unix: (local socket) baseUrl cannot be ` +
+				`proxied — it is inherently local, so proxying it gives fake ` +
+				`anonymity (SearXNG still crawls the web from your real IP). Set ` +
+				`egress=direct and proxy the backend itself (SearXNG's ` +
+				`outgoing.proxies), or use a remote backend.`,
+		);
+}
+
+function socksFromUrl(raw: string): Dispatcher {
+	const url = new URL(raw); // throws on a malformed proxy URL → fail loud
+	const protocol = url.protocol.replace(':', '');
+	if (protocol !== 'socks5' && protocol !== 'socks' && protocol !== 'socks5h')
+		throw new EgressError(
+			`egress socks5: expected a socks5:// proxy url, got ${raw}`,
+		);
+	const port = Number(url.port);
+	if (!url.hostname || !Number.isInteger(port) || port <= 0)
+		throw new EgressError(`egress socks5: invalid host/port in ${raw}`);
+	return socksDispatcher({
+		type: 5,
+		host: url.hostname,
+		port,
+		userId: url.username || undefined,
+		password: url.password || undefined,
+	});
+}
+
+/**
+ * Build the undici Dispatcher for the config's egress mode:
+ *   - direct → undefined (undici uses its default, un-proxied transport)
+ *   - http   → ProxyAgent
+ *   - socks5 → socks dispatcher (undici Agent over a socks connector)
+ *
+ * Throws (fail loud) if a configured http/socks5 proxy cannot be built. It
+ * NEVER returns `undefined` (direct) as a fallback for a broken proxy.
+ */
+export function buildDispatcher(cfg: Config): Dispatcher | undefined {
+	const egress: Egress = cfg.egress;
+	switch (egress.mode) {
+		case 'direct':
+			return undefined;
+		case 'http':
+			try {
+				if (!egress.url) throw new Error('missing proxy url');
+				return new ProxyAgent(egress.url);
+			} catch (cause) {
+				throw new EgressError(
+					`egress http: could not build proxy for ${egress.url}`,
+					{cause},
+				);
+			}
+		case 'socks5':
+			try {
+				if (!egress.url) throw new Error('missing proxy url');
+				return socksFromUrl(egress.url);
+			} catch (cause) {
+				if (cause instanceof EgressError) throw cause;
+				throw new EgressError(
+					`egress socks5: could not build proxy for ${egress.url}`,
+					{cause},
+				);
+			}
+		default: {
+			const exhaustive: never = egress;
+			throw new EgressError(
+				`egress: unknown mode ${JSON.stringify(exhaustive)}`,
+			);
+		}
+	}
+}
+
+/** A WHATWG-compatible fetch bound to a specific egress dispatcher. */
+export type EgressFetch = typeof globalThis.fetch;
+
+/**
+ * Build an egress-bound WHATWG `fetch`: undici's `fetch` closed over the
+ * dispatcher from buildDispatcher(cfg). This is the `fetch` injected into
+ * distilly/fetch so distilly never has egress of its own. Same fail-loud
+ * guarantee: a broken proxy throws HERE (before any I/O), never goes un-proxied.
+ */
+export function createEgressFetch(cfg: Config): EgressFetch {
+	const dispatcher = buildDispatcher(cfg);
+	return ((input: RequestInfo | URL, init?: RequestInit) =>
+		undiciFetch(
+			input as never,
+			{
+				...((init ?? {}) as Record<string, unknown>),
+				dispatcher,
+			} as never,
+		)) as EgressFetch;
+}
+
+export type {Dispatcher};
+export {Agent, ProxyAgent};

package/src/core/extract.ts

@@ -0,0 +1,82 @@
+// Extractor seam — turn a URL into clean, size-bounded markdown by calling
+// distilly's NETWORKED `urlToMarkdown` (the `distilly/fetch` entrypoint),
+// INJECTING webveil's egress-bound `fetch` as the only transport. distilly's
+// network Rules (github/mdn/react.dev/vuejs.org) rewrite a matching URL to its
+// raw `.md`/API source and fetch THAT over our egress — shorter, cleaner output;
+// non-matching URLs run through distilly's pure core. See docs/adr/0001.
+//
+// THE HARD INVARIANT (load-bearing for anonymity): webveil ALWAYS injects its
+// egress-bound `fetch` here and NEVER lets distilly use a global/default fetch.
+// distilly throws if none is injected — the desired fail-loud. And the egress
+// fetch itself throws (before any I/O) when a configured proxy is unbuildable
+// (egress.ts), so a broken proxy can never become an un-proxied request.
+//
+// This is the DEFAULT extractor; a backend's own `/extract` (tavily-compat) may
+// override it (wired in the core-fetch task).
+
+import {urlToMarkdown as distillyUrlToMarkdown} from 'distilly/fetch';
+import type {Config, FetchSize} from './config.js';
+import {createEgressFetch, type EgressFetch} from './egress.js';
+import type {FetchResult} from './backends/types.js';
+
+/** The shape distilly's `urlToMarkdown` returns (the bits we surface). */
+interface UrlToMarkdownResult {
+	markdown: string;
+	truncated: boolean;
+}
+
+/** distilly's networked entrypoint, narrowed to what the seam injects/uses. */
+type UrlToMarkdown = (
+	url: string | URL,
+	options: {fetch: EgressFetch; size?: FetchSize},
+) => Promise<UrlToMarkdownResult>;
+
+/** Per-call extractor options. */
+export interface ExtractOptions {
+	/**
+	 * Page-size budget for THIS call. Overrides the config's `fetchSize` when
+	 * given. webveil's `s`/`m`/`l`/`f` preset maps STRAIGHT to distilly's `size`
+	 * (the two enums are identical), so this is passed through verbatim.
+	 */
+	size?: FetchSize;
+}
+
+/**
+ * Seams the extractor's collaborators so it is testable WITHOUT real network or
+ * undici: tests inject a spy `urlToMarkdown` and/or a spy egress fetch to assert
+ * distilly is called with the egress fetch (never a global). Defaults wire the
+ * real `distilly/fetch` + `createEgressFetch`.
+ */
+export interface ExtractDeps {
+	/** distilly's networked `urlToMarkdown` (default: the real `distilly/fetch`). */
+	urlToMarkdown?: UrlToMarkdown;
+	/** Builds the egress-bound fetch from config (default: createEgressFetch). */
+	createEgressFetch?: (config: Config) => EgressFetch;
+}
+
+/**
+ * Extract a URL to clean, budget-bounded markdown via distilly over webveil's
+ * egress. Builds the egress-bound `fetch` (fail-loud on an unbuildable proxy),
+ * injects it into distilly's `urlToMarkdown`, maps the `s`/`m`/`l`/`f` preset to
+ * distilly's `size`, and surfaces distilly's `truncated`.
+ *
+ * @returns `{ url, markdown, truncated }` (a `FetchResult` without a `title`).
+ */
+export async function extract(
+	url: string,
+	config: Config,
+	options: ExtractOptions = {},
+	deps: ExtractDeps = {},
+): Promise<FetchResult> {
+	const urlToMarkdown = deps.urlToMarkdown ?? distillyUrlToMarkdown;
+	const buildFetch = deps.createEgressFetch ?? createEgressFetch;
+
+	// Build the egress-bound fetch FIRST: a configured-but-unbuildable proxy
+	// throws here, before any network access (never an un-proxied request).
+	const fetch = buildFetch(config);
+
+	const size = options.size ?? config.fetchSize;
+
+	const {markdown, truncated} = await urlToMarkdown(url, {fetch, size});
+	return {url, markdown, truncated};
+}