webveil 0.0.0 → 0.1.0

package/src/core/backends/custom.ts

@@ -0,0 +1,159 @@
+// custom backend — the local-command escape hatch (contract lifted from
+// pi-web-providers' custom-wrapper). Instead of an HTTP source, it spawns a
+// configured local command, writes the request as JSON to its stdin, and parses
+// `SearchResult[]` from its stdout. This lets any local script be a backend.
+//
+// Egress note: this backend owns its own I/O (the spawned command does whatever
+// it wants), so the handed `http` helper is unused here — there is no outbound
+// HTTP for webveil to proxy. It still returns the normalized SearchResult shape.
+//
+// Command source: the configured `baseUrl` carries the command line, parsed as a
+// whitespace-separated argv (first token = executable, rest = args), matching how
+// the other backends read `baseUrl` as "where results come from". (Recorded
+// decision; see the task's Decisions block.)
+//
+// Contract:
+//   stdin  <- JSON: {"query": string, "maxResults"?: number}
+//   stdout -> JSON: SearchResult[]  (each {title, url, snippet?})
+// Malformed stdout (non-JSON, not an array, or entries missing url/title) FAILS
+// CLEARLY — it never silently returns an empty list.
+
+import {spawn as defaultSpawn} from 'node:child_process';
+import type {Config} from '../config.js';
+import type {Backend, Http, SearchOptions, SearchResult} from './types.js';
+
+/** The JSON request written to the command's stdin. */
+interface CustomRequest {
+	query: string;
+	maxResults?: number;
+}
+
+/** The result of running the command: its stdout text (and exit status). */
+interface CommandRun {
+	stdout: string;
+	stderr: string;
+	code: number | null;
+}
+
+/**
+ * Minimal `spawn` shape this backend needs, seamed so a test can inject a fake
+ * without a real subprocess. Defaults to `node:child_process` `spawn`.
+ */
+export type SpawnFn = typeof defaultSpawn;
+
+function str(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Parse the configured command line into [executable, ...args]. */
+function parseCommand(baseUrl: string): [string, string[]] {
+	const parts = baseUrl.trim().split(/\s+/).filter(Boolean);
+	if (parts.length === 0)
+		throw new Error(
+			'custom: no command configured (set baseUrl to the command to run)',
+		);
+	return [parts[0]!, parts.slice(1)];
+}
+
+/**
+ * Normalize one stdout entry into a SearchResult, FAILING CLEARLY on a malformed
+ * entry rather than dropping it — the custom contract is explicit, so a missing
+ * url/title is a contract violation the user should see, not a silent skip.
+ */
+function toResult(entry: unknown, index: number): SearchResult {
+	if (typeof entry !== 'object' || entry === null)
+		throw new Error(
+			`custom: malformed output — result[${index}] is not an object`,
+		);
+	const hit = entry as Record<string, unknown>;
+	const url = str(hit.url);
+	const title = str(hit.title);
+	if (!url || !title)
+		throw new Error(
+			`custom: malformed output — result[${index}] is missing a url or title`,
+		);
+	const snippet = str(hit.snippet);
+	return snippet ? {title, url, snippet} : {title, url};
+}
+
+/** Parse the command's stdout into SearchResult[], failing clearly on garbage. */
+function parseOutput(stdout: string): SearchResult[] {
+	const trimmed = stdout.trim();
+	if (trimmed.length === 0)
+		throw new Error('custom: command produced no output');
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(trimmed);
+	} catch (cause) {
+		throw new Error(
+			`custom: malformed output — stdout is not valid JSON: ${(cause as Error).message}`,
+		);
+	}
+	if (!Array.isArray(parsed))
+		throw new Error(
+			'custom: malformed output — expected a JSON array of results',
+		);
+	return parsed.map(toResult);
+}
+
+/** Spawn the command, write the request to stdin, and collect stdout/stderr. */
+function runCommand(
+	spawn: SpawnFn,
+	exe: string,
+	args: string[],
+	request: CustomRequest,
+	signal?: AbortSignal,
+): Promise<CommandRun> {
+	return new Promise<CommandRun>((resolve, reject) => {
+		const child = spawn(exe, args, {
+			stdio: ['pipe', 'pipe', 'pipe'],
+			signal,
+		});
+		let stdout = '';
+		let stderr = '';
+		child.stdout?.on('data', (chunk) => (stdout += String(chunk)));
+		child.stderr?.on('data', (chunk) => (stderr += String(chunk)));
+		child.on('error', (err) =>
+			reject(new Error(`custom: failed to spawn '${exe}': ${err.message}`)),
+		);
+		child.on('close', (code) => resolve({stdout, stderr, code}));
+		child.stdin?.on('error', () => {
+			// A command that exits before reading stdin closes the pipe; ignore the
+			// EPIPE here and let the close handler report via exit code/stderr.
+		});
+		child.stdin?.end(JSON.stringify(request));
+	});
+}
+
+/**
+ * Build a custom backend bound to the configured command. The command owns its
+ * own I/O; webveil hands it the request as JSON on stdin and parses
+ * SearchResult[] from stdout, failing clearly on malformed output.
+ */
+export function createCustomBackend(
+	config: Config,
+	spawn: SpawnFn = defaultSpawn,
+): Backend {
+	const [exe, args] = parseCommand(config.baseUrl);
+	return {
+		async search(
+			query: string,
+			_http: Http,
+			options: SearchOptions = {},
+		): Promise<SearchResult[]> {
+			const request: CustomRequest = {query};
+			if (options.maxResults !== undefined)
+				request.maxResults = options.maxResults;
+			const run = await runCommand(spawn, exe, args, request, options.signal);
+			if (run.code !== 0)
+				throw new Error(
+					`custom: command '${exe}' exited with code ${run.code}` +
+						(run.stderr.trim() ? `: ${run.stderr.trim()}` : ''),
+				);
+			const results = parseOutput(run.stdout);
+			return options.maxResults !== undefined
+				? results.slice(0, options.maxResults)
+				: results;
+		},
+	};
+}

package/src/core/backends/registry.ts

@@ -0,0 +1,41 @@
+// backend registry — a tiny `name -> Backend` dispatcher (concept trimmed from
+// pi-search-hub's registry). Each backend registers a factory keyed by its config
+// `backend` name; `getBackend` resolves the name to a constructed Backend (handed
+// the resolved config so it knows its instance baseUrl / apiKey) and fails clearly
+// on an unknown name. Later backend tasks (tavily-compat, custom) append their own
+// registrations to FACTORIES below.
+
+import type {Config} from '../config.js';
+import type {Backend} from './types.js';
+import {createSearxngBackend} from './searxng.js';
+import {createTavilyCompatBackend} from './tavily-compat.js';
+import {createCustomBackend} from './custom.js';
+
+/** Builds a Backend from the resolved config (knows its baseUrl / apiKey). */
+export type BackendFactory = (config: Config) => Backend;
+
+/** name -> factory. New backends add an entry here. */
+const FACTORIES: Record<string, BackendFactory> = {
+	searxng: createSearxngBackend,
+	'tavily-compat': createTavilyCompatBackend,
+	custom: createCustomBackend,
+};
+
+/** The backend names the registry can resolve. */
+export function backendNames(): string[] {
+	return Object.keys(FACTORIES);
+}
+
+/**
+ * Resolve a backend name to a constructed Backend. Throws clearly on an unknown
+ * name (listing the known ones) so a misconfigured `backend` fails loud, never
+ * silently no-ops.
+ */
+export function getBackend(name: string, config: Config): Backend {
+	const factory = FACTORIES[name];
+	if (!factory)
+		throw new Error(
+			`webveil: unknown backend '${name}' (known: ${backendNames().join(', ')})`,
+		);
+	return factory(config);
+}

package/src/core/backends/searxng.ts

@@ -0,0 +1,70 @@
+// searxng backend — the keyless, self-hosted metasearch default. Queries a
+// SearXNG instance's JSON API (`/search?format=json`) THROUGH the handed `http`
+// helper (never a direct fetch, so egress is not bypassable) and normalizes the
+// response into SearchResult[].
+
+import type {Config} from '../config.js';
+import type {Backend, Http, SearchOptions, SearchResult} from './types.js';
+
+/** The shape of one entry in a SearXNG JSON `results` array (subset we use). */
+interface SearxngResult {
+	url?: unknown;
+	title?: unknown;
+	content?: unknown;
+}
+
+/** The SearXNG JSON API response (subset we use). */
+interface SearxngResponse {
+	results?: SearxngResult[];
+}
+
+function str(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Normalize one SearXNG hit; drop entries without a usable url + title. */
+function toResult(hit: SearxngResult): SearchResult | undefined {
+	const url = str(hit.url);
+	const title = str(hit.title);
+	if (!url || !title) return undefined;
+	const snippet = str(hit.content);
+	return snippet ? {title, url, snippet} : {title, url};
+}
+
+/** Build the SearXNG JSON search URL for a query against the instance baseUrl. */
+function buildUrl(baseUrl: string, query: string): string {
+	const url = new URL(
+		'search',
+		baseUrl.endsWith('/') ? baseUrl : baseUrl + '/',
+	);
+	url.searchParams.set('q', query);
+	url.searchParams.set('format', 'json');
+	return url.toString();
+}
+
+/**
+ * Build a SearXNG backend bound to the configured instance. The returned backend
+ * only ever touches the network via the injected `http` helper.
+ */
+export function createSearxngBackend(config: Config): Backend {
+	const baseUrl = config.baseUrl;
+	return {
+		async search(
+			query: string,
+			http: Http,
+			options: SearchOptions = {},
+		): Promise<SearchResult[]> {
+			const body = await http.fetchJson<SearxngResponse>(
+				buildUrl(baseUrl, query),
+				{headers: {accept: 'application/json'}, signal: options.signal},
+			);
+			const results = Array.isArray(body.results) ? body.results : [];
+			const normalized = results
+				.map(toResult)
+				.filter((r): r is SearchResult => r !== undefined);
+			return options.maxResults !== undefined
+				? normalized.slice(0, options.maxResults)
+				: normalized;
+		},
+	};
+}

package/src/core/backends/tavily-compat.ts

@@ -0,0 +1,156 @@
+// tavily-compat backend — a generic Tavily-shaped client (POST `/search` and an
+// optional POST `/extract`) selected purely by `baseUrl`, so it covers
+// orio-search / searcharvester / agent-search and any other Tavily-API-shaped
+// instance. Both endpoints go THROUGH the handed `http` helper (never a direct
+// fetch, so egress is not bypassable). `/search` normalizes to SearchResult[];
+// `/extract` is exposed as the optional `Backend.fetch` a later task uses to
+// override the distilly Extractor.
+//
+// Auth: a Bearer header is sent only when an apiKey is configured. The covered
+// self-hosted instances are typically keyless, so a missing key is normal, not
+// an error.
+
+import type {Config} from '../config.js';
+import type {
+	Backend,
+	FetchOptions,
+	FetchResult,
+	Http,
+	HttpRequestOptions,
+	SearchOptions,
+	SearchResult,
+} from './types.js';
+
+/** One entry in a Tavily `/search` `results` array (subset we use). */
+interface TavilySearchHit {
+	title?: unknown;
+	url?: unknown;
+	content?: unknown;
+}
+
+/** The Tavily `/search` response (subset we use). */
+interface TavilySearchResponse {
+	results?: TavilySearchHit[];
+}
+
+/** One entry in a Tavily `/extract` `results` array (subset we use). */
+interface TavilyExtractHit {
+	url?: unknown;
+	raw_content?: unknown;
+}
+
+/** One entry in a Tavily `/extract` `failed_results` array (subset we use). */
+interface TavilyExtractFailure {
+	url?: unknown;
+	error?: unknown;
+}
+
+/** The Tavily `/extract` response (subset we use). */
+interface TavilyExtractResponse {
+	results?: TavilyExtractHit[];
+	failed_results?: TavilyExtractFailure[];
+}
+
+function str(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+/** Normalize one Tavily search hit; drop entries without a usable url + title. */
+function toResult(hit: TavilySearchHit): SearchResult | undefined {
+	const url = str(hit.url);
+	const title = str(hit.title);
+	if (!url || !title) return undefined;
+	const snippet = str(hit.content);
+	return snippet ? {title, url, snippet} : {title, url};
+}
+
+/** Resolve an endpoint path against the instance baseUrl. */
+function endpoint(baseUrl: string, path: string): string {
+	return new URL(
+		path,
+		baseUrl.endsWith('/') ? baseUrl : baseUrl + '/',
+	).toString();
+}
+
+/**
+ * Build a Tavily-compat backend bound to the configured instance. The returned
+ * backend only ever touches the network via the injected `http` helper. A Bearer
+ * header is added only when an apiKey is set (the covered instances are usually
+ * keyless).
+ */
+export function createTavilyCompatBackend(config: Config): Backend {
+	const baseUrl = config.baseUrl;
+	const apiKey = config.apiKey;
+
+	function headers(): Record<string, string> {
+		const h: Record<string, string> = {
+			'content-type': 'application/json',
+			accept: 'application/json',
+		};
+		if (apiKey) h.authorization = `Bearer ${apiKey}`;
+		return h;
+	}
+
+	function post(
+		path: string,
+		payload: unknown,
+		signal?: AbortSignal,
+	): HttpRequestOptions {
+		return {
+			method: 'POST',
+			headers: headers(),
+			body: JSON.stringify(payload),
+			signal,
+		};
+	}
+
+	return {
+		async search(
+			query: string,
+			http: Http,
+			options: SearchOptions = {},
+		): Promise<SearchResult[]> {
+			const payload: Record<string, unknown> = {query};
+			if (options.maxResults !== undefined)
+				payload.max_results = options.maxResults;
+			const body = await http.fetchJson<TavilySearchResponse>(
+				endpoint(baseUrl, 'search'),
+				post('search', payload, options.signal),
+			);
+			const results = Array.isArray(body.results) ? body.results : [];
+			const normalized = results
+				.map(toResult)
+				.filter((r): r is SearchResult => r !== undefined);
+			return options.maxResults !== undefined
+				? normalized.slice(0, options.maxResults)
+				: normalized;
+		},
+
+		async fetch(
+			url: string,
+			http: Http,
+			options: FetchOptions = {},
+		): Promise<FetchResult> {
+			// Tavily `/extract` has no `s/m/l/f` size knob (it has `format` /
+			// `extract_depth` instead); always request markdown. The default
+			// distilly Extractor owns webveil's size presets.
+			const body = await http.fetchJson<TavilyExtractResponse>(
+				endpoint(baseUrl, 'extract'),
+				post('extract', {urls: url, format: 'markdown'}, options.signal),
+			);
+			const failure = (body.failed_results ?? []).find(
+				(f) => str(f.url) === url,
+			);
+			if (failure)
+				throw new Error(
+					`tavily-compat: /extract failed for ${url}: ${str(failure.error) ?? 'unknown error'}`,
+				);
+			const hit = (body.results ?? [])[0];
+			const markdown = hit ? str(hit.raw_content) : undefined;
+			if (markdown === undefined)
+				throw new Error(`tavily-compat: no extract result for ${url}`);
+			// Tavily `/extract` returns no `truncated` flag and no page title.
+			return {url, markdown, truncated: false};
+		},
+	};
+}

package/src/core/backends/types.ts

@@ -0,0 +1,61 @@
+// backend seam — the contract every result source (searxng | tavily-compat |
+// custom) implements. A Backend is HANDED a proxied `http` helper (bound to the
+// configured egress dispatcher) so it physically cannot bypass the egress.
+
+/** A single search hit. */
+export interface SearchResult {
+	title: string;
+	url: string;
+	snippet?: string;
+}
+
+/** A fetched, extracted page as budget-bounded markdown. */
+export interface FetchResult {
+	url: string;
+	title?: string;
+	markdown: string;
+	truncated: boolean;
+}
+
+export interface SearchOptions {
+	maxResults?: number;
+	signal?: AbortSignal;
+}
+
+export interface FetchOptions {
+	size?: 's' | 'm' | 'l' | 'f';
+	signal?: AbortSignal;
+}
+
+/** Options the http helper accepts for a single request. */
+export interface HttpRequestOptions {
+	method?: string;
+	headers?: Record<string, string>;
+	body?: string;
+	/** Per-request timeout in ms (the helper aborts past this). */
+	timeoutMs?: number;
+	signal?: AbortSignal;
+}
+
+/**
+ * The proxied http helper handed to backends. Both methods route through the
+ * egress dispatcher; a backend never gets un-proxied transport of its own.
+ */
+export interface Http {
+	fetchJson<T = unknown>(url: string, options?: HttpRequestOptions): Promise<T>;
+	fetchText(url: string, options?: HttpRequestOptions): Promise<string>;
+}
+
+/**
+ * A result/content source. `search` is required; `fetch` is optional (a backend
+ * may override the default distilly Extractor with its own `/extract`). Both are
+ * given the proxied `http` helper so they cannot escape the configured egress.
+ */
+export interface Backend {
+	search(
+		query: string,
+		http: Http,
+		options?: SearchOptions,
+	): Promise<SearchResult[]>;
+	fetch?(url: string, http: Http, options?: FetchOptions): Promise<FetchResult>;
+}

package/src/core/config.ts

@@ -0,0 +1,106 @@
+// config seam — per-folder resolution. Precedence (highest wins):
+//   env > nearest .pi/webveil.json (walking up from cwd) > global
+//   ~/.pi/agent/webveil.json > defaults.
+// "Per folder = per account/egress." Each layer is a partial; later (lower)
+// layers fill gaps the higher layers leave.
+
+import {readFileSync} from 'node:fs';
+import {homedir} from 'node:os';
+import {dirname, join, parse} from 'node:path';
+
+/** How outbound HTTP leaves the machine. See egress.ts. */
+export type Egress =
+	| {mode: 'direct'}
+	| {mode: 'http'; url: string}
+	| {mode: 'socks5'; url: string};
+
+/** Page-size budget preset for fetch (passed through to distilly). */
+export type FetchSize = 's' | 'm' | 'l' | 'f';
+
+/** The fully-resolved config every webveil module consumes. */
+export interface Config {
+	backend: string;
+	baseUrl: string;
+	apiKey?: string;
+	egress: Egress;
+	fetchSize: FetchSize;
+}
+
+/** A config file / env layer: any subset of the resolved shape. */
+export type PartialConfig = Partial<Config>;
+
+export interface ResolveOptions {
+	/** Directory the per-folder walk starts from. Defaults to process.cwd(). */
+	cwd?: string;
+	/** Environment to read overrides from. Defaults to process.env. */
+	env?: Record<string, string | undefined>;
+	/**
+	 * Path to the global config file. Defaults to ~/.pi/agent/webveil.json.
+	 * Tests point this at a temp dir to isolate the real home directory.
+	 */
+	globalPath?: string;
+}
+
+const DEFAULTS: Config = {
+	backend: 'searxng',
+	baseUrl: 'http://127.0.0.1:8080',
+	egress: {mode: 'direct'},
+	fetchSize: 'm',
+};
+
+const PROJECT_FILE = join('.pi', 'webveil.json');
+
+function readJson(path: string): PartialConfig | undefined {
+	let text: string;
+	try {
+		text = readFileSync(path, 'utf8');
+	} catch {
+		return undefined; // absent file is fine; missing layers are expected
+	}
+	return JSON.parse(text) as PartialConfig;
+}
+
+/** The nearest `.pi/webveil.json` walking up from `cwd` (first found wins). */
+function readProjectChain(cwd: string): PartialConfig | undefined {
+	let dir = cwd;
+	const {root} = parse(dir);
+	for (;;) {
+		const found = readJson(join(dir, PROJECT_FILE));
+		if (found) return found;
+		if (dir === root) return undefined;
+		dir = dirname(dir);
+	}
+}
+
+function readEnv(env: Record<string, string | undefined>): PartialConfig {
+	const layer: PartialConfig = {};
+	if (env.WEBVEIL_BACKEND) layer.backend = env.WEBVEIL_BACKEND;
+	if (env.WEBVEIL_BASE_URL) layer.baseUrl = env.WEBVEIL_BASE_URL;
+	if (env.WEBVEIL_API_KEY) layer.apiKey = env.WEBVEIL_API_KEY;
+	if (env.WEBVEIL_FETCH_SIZE)
+		layer.fetchSize = env.WEBVEIL_FETCH_SIZE as FetchSize;
+	const mode = env.WEBVEIL_EGRESS;
+	if (mode === 'direct') layer.egress = {mode: 'direct'};
+	else if (mode === 'http' || mode === 'socks5')
+		layer.egress = {mode, url: env.WEBVEIL_EGRESS_URL ?? ''};
+	return layer;
+}
+
+/**
+ * Resolve the effective config. Higher-precedence layers override lower ones,
+ * key by key: env > project chain > global file > defaults.
+ */
+export function resolveConfig(options: ResolveOptions = {}): Config {
+	const cwd = options.cwd ?? process.cwd();
+	const env = options.env ?? process.env;
+	const globalPath =
+		options.globalPath ?? join(homedir(), '.pi', 'agent', 'webveil.json');
+
+	const layers: PartialConfig[] = [
+		DEFAULTS,
+		readJson(globalPath) ?? {},
+		readProjectChain(cwd) ?? {},
+		readEnv(env),
+	];
+	return Object.assign({}, ...layers) as Config;
+}