pi-wsearch 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 K2
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,147 @@
+# pi-wsearch: web tools for pi
+
+## Not the web tools you've used
+
+### web.search
+
+**One search, many judges.** Most web tools hand you a single engine's
+ranking. `web.search` runs every engine you've configured in parallel and
+merges their results by reciprocal rank fusion (RRF). A hit only earns its
+place by surviving several independent judgments. The response shows how
+many: `← 3 engines agree` means three separate indexes ranked that page.
+Agreement is the spam defense; generative-SEO can stuff one index, not three.
+
+**Pick your engines.** Nine are supported; three or four is plenty. Use
+whatever keys you have, even one, because what follows doesn't depend on how
+many engines answer:
+
+- **Rare strings match literally.** `pgvector`, CVE numbers, error codes match
+  as exact words, not fuzzy prose.
+- **Junk is downranked, not censored.** Known junk domains and thin snippets
+  lose points at scoring time.
+- **Same page, one entry.** Tracking-param variants and near-duplicate titles
+  collapse into one row.
+- **Two results per domain, max.** One site's SEO army can't flood the
+  answers.
+
+**Supported providers:** Serper, Tavily, Exa, Brave, Jina, Kagi, You.com,
+Firecrawl, TinyFish
+
+### web.fetch
+
+**Fetch reads a map, not a dump.** Most web tools hand you raw HTML and tell
+you to find the signal. `web.fetch` breaks any page into an outline of every
+section and answers by number: `{url, sections: [0, 3]}` fetches just two
+sections of the fetched page. The agent surveys the map, picks what matters,
+and reads only those. No
+full reads, no 40 KB dumps. Truncation is layered: sections over 3,000
+characters are cut with the tail marked unreachable; pages that overflow are
+capped; the final response is clipped. Every cut is marked; the truncated
+part is named, not lost.
+
+**Reads are free.** Page parsing is built in, nothing billed per read, no key
+required. Real headings become the map, TOC echoes get dropped, so reads come
+back as clean as a paid one at zero cost. Paid readers only step in for
+JavaScript heavy pages or when the local read fails.
+
+**Tracking links cache as one page.** `?utm_source=…` variants of the same URL
+share one 24 hour cache entry. Faster repeats, smaller disk.
+
+**Guarded against your localhost.** A fetched page can't aim the agent at
+your own machine: private and loopback addresses (hex encoded, shorthand, and
+mapped IPv6 forms included) are rejected before any request is made.
+
+**Distrust is the default.** The agent is prompted to treat fetched content
+as untrusted input: a claim only becomes a verdict after a second source
+corroborates it.
+
+**Optional readers:** Firecrawl, Tavily, Exa, Jina, fallback only
+
+## Install
+
+From npm:
+
+```
+pi install npm:pi-wsearch
+```
+
+Or from git:
+
+```
+pi install git:github.com/k3-2o/pi-wsearch
+```
+
+Either way pi picks up the package on next start and registers `web.search`,
+`web.fetch`, and the `/websearch` command. To try it once without installing:
+
+```
+pi -e npm:pi-wsearch
+```
+
+## Keys
+
+Search needs at least one provider key; fetch needs none. Two ways to set one:
+
+- **In your shell.** Export it in the terminal session running pi, or park it
+  in your shell config. The extension reads `.zshrc`, `.zprofile`, `.bashrc`,
+  `.profile`, fish config, and nushell env based on your `$SHELL` at startup,
+  so a key in any of those is discovered on its own.
+- **Via `/websearch`.** Pick a provider from the list, paste the key. It's
+  written to `~/.pi/agent/wsearch/env` with 0600 permissions.
+
+Either place accepts a `!command` value, so the key can come from Keychain or
+1Password at use time and never sit on disk. Session env wins over the file,
+the file over shell config. `/websearch` shows which providers are armed and
+which are dormant. No keys yet? Tavily, Exa, Firecrawl, and TinyFish all have
+free tiers.
+
+## Commands
+
+
+One command, five subcommands:
+
+| Command | Does |
+|---|---|
+| `/websearch` | Status: what's armed, what's dormant, where each key lives. |
+| `/websearch login [provider]` | Store a key: pick from the provider list, paste it, confirm. Accepts `!command` values. |
+| `/websearch logout [provider]` | Remove a key from `wsearch/env`. Keys set in your shell or env are reported, but can't be removed from here. |
+| `/websearch off` | Hide both tools from the agent until you turn them back on. |
+| `/websearch on` | Bring them back. |
+
+> [!NOTE]
+> Two knobs also live in `wsearch/env` (or your shell env, same precedence as
+> keys). `WSEARCH_ENGINES=tavily,exa` makes search use only Tavily and Exa,
+> with Tavily consulted first. `WSEARCH_FETCH_CHAIN=tavily,jina` makes failed
+> reads try Tavily first, then Jina, skipping the rest.
+
+## Quick start
+
+1. Get a key from any provider: Tavily, Exa, and Firecrawl have free tiers.
+2. Set it one of three ways:
+
+   ```
+   /websearch login            # pick a provider, paste the key
+   /websearch login tavily     # or name it directly
+   export TAVILY_API_KEY=...   # or just the terminal session
+   ```
+
+   Shell config works too (.zshrc, .bashrc, fish config): park the export
+   there and the extension finds it on startup.
+3. Verify:
+
+   ```
+   /websearch
+   ```
+
+   Your provider should appear under `armed:`.
+4. Ask the agent something current:
+
+   > What's the current Node.js LTS version?
+
+   It will `web.search` it, `web.fetch` the answer's page by section, and
+   cite where each claim came from.
+
+## License
+
+MIT
+

package/index.ts

@@ -0,0 +1,8 @@
+/** Entry: registers web.search + web.fetch (wiring lives in tools.ts). */
+import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
+import { registerWebTools, registerWebCommand } from './src/tools';
+
+export default function (pi: ExtensionAPI) {
+	registerWebTools(pi);
+	registerWebCommand(pi);
+}

package/justfile

@@ -0,0 +1,31 @@
+export PATH := "node_modules/.bin:" + env_var_or_default("PATH", "")
+
+default: check
+
+check: lint format-check typecheck knip test
+
+lint:
+	oxlint
+
+format:
+	oxfmt
+
+format-check:
+	oxfmt --check
+
+typecheck:
+	tsc --noEmit
+
+knip:
+	knip
+
+coverage:
+	bun test --jobs 1 test/ --coverage
+
+test:
+	bun test --jobs 1 test/
+
+install:
+	mkdir -p $HOME/.pi/agent/extensions/pi-wsearch/src
+	cp index.ts $HOME/.pi/agent/extensions/pi-wsearch/
+	cp src/*.ts $HOME/.pi/agent/extensions/pi-wsearch/src/

package/package.json

@@ -0,0 +1,63 @@
+{
+  "name": "pi-wsearch",
+  "version": "0.1.0",
+  "description": "Pi extension: web.search + web.fetch",
+  "license": "MIT",
+  "type": "module",
+  "keywords": [
+    "pi-package",
+    "pi",
+    "extension",
+    "coding-agent",
+    "agent",
+    "web-search",
+    "web-search-api",
+    "search-engine",
+    "multi-engine",
+    "search-api",
+    "web-fetch",
+    "scraper",
+    "reader",
+    "rrf",
+    "typescript",
+    "bun",
+    "ai"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/k3-2o/pi-wsearch.git"
+  },
+  "homepage": "https://github.com/k3-2o/pi-wsearch#readme",
+  "bugs": "https://github.com/k3-2o/pi-wsearch/issues",
+  "scripts": {
+    "lint": "oxlint",
+    "format": "oxfmt",
+    "format:check": "oxfmt --check",
+    "check": "oxlint && oxfmt --check && tsc --noEmit && knip && bun test --jobs 1 test/",
+    "test": "bun test --jobs 1 test/",
+    "typecheck": "tsc --noEmit",
+    "knip": "knip",
+    "coverage": "bun test --jobs 1 test/ --coverage"
+  },
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-agent-core": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.4.2",
+    "fast-check": "^4.10.2",
+    "knip": "^6.39.0",
+    "oxfmt": "^0.71.0",
+    "oxlint": "^1.86.0",
+    "typescript": "^7.0.2"
+  },
+  "dependencies": {}
+}

package/src/cache.ts

@@ -0,0 +1,91 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+const DEFAULT_CAP = 400;
+
+interface Entry {
+	exp: number;
+	value: unknown;
+}
+
+interface Store {
+	entries: Record<string, Entry>;
+	order: string[];
+}
+
+export class TtlCache {
+	private store: Store = { entries: {}, order: [] };
+	constructor(
+		private file: string,
+		private cap = DEFAULT_CAP,
+	) {
+		this.load();
+	}
+	private load() {
+		try {
+			if (!existsSync(this.file)) return;
+			const raw = JSON.parse(readFileSync(this.file, 'utf8'));
+			if (raw && typeof raw === 'object' && typeof raw.entries === 'object') {
+				this.store = raw;
+				const now = Date.now();
+				// purge expired entries AND their order slots: dead keys must not grow across restarts
+				for (const k of Object.keys(this.store.entries)) {
+					if ((this.store.entries[k]?.exp ?? 0) <= now) delete this.store.entries[k];
+				}
+				this.store.order = this.store.order.filter((k) => this.store.entries[k] !== undefined);
+			}
+		} catch {
+			this.store = { entries: {}, order: [] };
+		}
+	}
+	private persist() {
+		try {
+			mkdirSync(dirname(this.file), { recursive: true });
+			const tmp = this.file + '.tmp';
+			writeFileSync(tmp, JSON.stringify(this.store), 'utf8');
+			renameSync(tmp, this.file);
+		} catch {
+			/* cache writes are best-effort */
+		}
+	}
+	get(key: string): unknown | undefined {
+		const e = this.store.entries[key];
+		if (!e) return undefined;
+		if (e.exp < Date.now()) {
+			delete this.store.entries[key];
+			this.store.order = this.store.order.filter((k) => k !== key);
+			return undefined;
+		}
+		const i = this.store.order.indexOf(key);
+		if (i >= 0) this.store.order.splice(i, 1);
+		this.store.order.push(key);
+		return e.value;
+	}
+	set(key: string, value: unknown, ttlMs: number) {
+		this.store.entries[key] = { exp: Date.now() + ttlMs, value };
+		const i = this.store.order.indexOf(key);
+		if (i >= 0) this.store.order.splice(i, 1);
+		this.store.order.push(key);
+		while (this.store.order.length > this.cap) {
+			const old = this.store.order.shift()!;
+			delete this.store.entries[old];
+		}
+		this.persist();
+	}
+}
+
+export function cacheKey(parts: string[]): string {
+	return createHash('sha256').update(parts.join('|')).digest('hex');
+}
+
+/** Search results are NOT cached: exact-repeat reuse is rare; staleness risk is not worth it. */
+export const CACHE_TTL_HOURS = 24;
+
+function defaultCacheDir(): string {
+	return join(process.env.HOME ?? process.env.USERPROFILE ?? '.', '.pi', 'agent', 'wsearch');
+}
+
+export function openCache(file?: string): TtlCache {
+	return new TtlCache(file ?? join(defaultCacheDir(), 'cache.json'));
+}

package/src/config.ts

@@ -0,0 +1,263 @@
+/**
+ * Key sources, in order (first hit wins): process.env → the dedicated
+ * ~/.pi/agent/wsearch/env file → the user's detected shell config. Any value
+ * may be `!command` (runs once per process; stdout = key). Values are never
+ * logged, returned, or serialized.
+ */
+import { readFileSync, existsSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+export interface WebConfig {
+	serperKey?: string;
+	tavilyKey?: string;
+	exaKey?: string;
+	firecrawlKey?: string;
+	braveKey?: string;
+	jinaKey?: string;
+	kagiKey?: string;
+	youKey?: string;
+	tinyfishKey?: string;
+	cacheDir: string;
+	allowPrivate?: boolean;
+	searchEngines?: string;
+	fetchChain?: string;
+}
+
+export type ProviderKey =
+	| 'serperKey'
+	| 'tavilyKey'
+	| 'exaKey'
+	| 'firecrawlKey'
+	| 'braveKey'
+	| 'jinaKey'
+	| 'kagiKey'
+	| 'youKey'
+	| 'tinyfishKey';
+
+interface KeySpec {
+	id: string;
+	key: ProviderKey;
+	envNames: string[];
+}
+
+export const SPECS: KeySpec[] = [
+	{ id: 'serper', key: 'serperKey', envNames: ['SERPER_API_KEY', 'SERPER_KEY'] },
+	{ id: 'tavily', key: 'tavilyKey', envNames: ['TAVILY_API_KEY', 'TAVILY_KEY'] },
+	{ id: 'exa', key: 'exaKey', envNames: ['EXA_API_KEY', 'EXA_KEY'] },
+	{ id: 'firecrawl', key: 'firecrawlKey', envNames: ['FIRECRAWL_API_KEY', 'FIRECRAWL_KEY'] },
+	{ id: 'brave', key: 'braveKey', envNames: ['BRAVE_API_KEY', 'BRAVE_KEY'] },
+	{ id: 'jina', key: 'jinaKey', envNames: ['JINA_API_KEY', 'JINA_KEY'] },
+	{ id: 'kagi', key: 'kagiKey', envNames: ['KAGI_API_KEY', 'KAGI_KEY'] },
+	{ id: 'you', key: 'youKey', envNames: ['YOUCOM_API_KEY', 'YDC_API_KEY'] },
+	{ id: 'tinyfish', key: 'tinyfishKey', envNames: ['TINYFISH_API_KEY', 'TINYFISH_KEY'] },
+];
+
+export interface ProviderInfo {
+	id: string;
+	label: string;
+	envName: string;
+	aliases: string[];
+}
+
+const PROVIDER_LABELS: Record<string, string> = {
+	serper: 'Serper (Google)',
+	tavily: 'Tavily',
+	exa: 'Exa',
+	firecrawl: 'Firecrawl',
+	brave: 'Brave',
+	jina: 'Jina',
+	kagi: 'Kagi',
+	you: 'You.com',
+	tinyfish: 'TinyFish',
+};
+
+export const PROVIDERS: ProviderInfo[] = SPECS.map((s) => ({
+	id: s.id,
+	label: PROVIDER_LABELS[s.id] ?? s.id,
+	envName: s.envNames[0],
+	aliases: s.envNames.slice(1),
+}));
+
+const SHELL_FILES: Record<string, string[]> = {
+	zsh: ['.zshrc', '.zprofile'],
+	bash: ['.bashrc', '.bash_profile', '.profile'],
+	sh: ['.profile'],
+	dash: ['.profile'],
+	fish: ['.config/fish/config.fish'],
+	nu: ['.config/nushell/env.nu'],
+	nushell: ['.config/nushell/env.nu'],
+};
+const DEFAULT_SHELL_FILES = SHELL_FILES.bash;
+
+export function shellConfigFiles(shell = process.env.SHELL): string[] {
+	if (!shell) return DEFAULT_SHELL_FILES;
+	const base = shell.split('/').pop()?.toLowerCase() ?? '';
+	return SHELL_FILES[base] ?? DEFAULT_SHELL_FILES;
+}
+
+// A '#' starts a comment only at a word boundary, outside quotes: 'K=a#b'
+// keeps the hash; 'K=a # comment' drops it (runs before unquoting).
+function stripComment(v: string): string {
+	let inS = false;
+	let inD = false;
+	for (let i = 0; i < v.length; i++) {
+		const c = v[i];
+		if (c === '\\' && (inS || inD)) i++;
+		else if (c === "'" && !inD) inS = !inS;
+		else if (c === '"' && !inS) inD = !inD;
+		else if (c === '#' && !inS && !inD && (i === 0 || /\s/.test(v[i - 1] as string))) return v.slice(0, i);
+	}
+	return v;
+}
+
+function unquote(v: string): string {
+	let s = stripComment(v).trim();
+	if (s.length >= 2 && ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'")))) {
+		s = s.slice(1, -1);
+	}
+	return s;
+}
+
+export function parseEnvLine(line: string): [string, string] | null {
+	const t = line.trim();
+	if (!t || t.startsWith('#')) return null;
+	let m = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(t);
+	if (m) return [m[1], unquote(m[2])];
+	m = /^\s*set\s+(?:-gx|-x)\s+([A-Za-z_][A-Za-z0-9_]*)\s+(.*)$/.exec(t);
+	if (m) return [m[1], unquote(m[2])];
+	m = /^\s*\$env\.([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(t);
+	if (m) return [m[1], unquote(m[2])];
+	return null;
+}
+
+export function parseEnvText(text: string): Record<string, string> {
+	const out: Record<string, string> = {};
+	for (const raw of text.split('\n')) {
+		const pair = parseEnvLine(raw);
+		if (pair && !(pair[0] in out)) out[pair[0]] = pair[1];
+	}
+	return out;
+}
+
+function readEnvText(path: string): Record<string, string> {
+	try {
+		if (!existsSync(path)) return {};
+		return parseEnvText(readFileSync(path, 'utf8'));
+	} catch {
+		return {};
+	}
+}
+
+export function resolveStoredValue(raw: string): string | undefined {
+	const v = raw.trim();
+	if (!v.startsWith('!')) return v.length > 4 ? v : undefined;
+	const cmd = v.slice(1).trim();
+	if (!cmd) return undefined;
+	try {
+		const r = spawnSync('sh', ['-c', cmd], {
+			stdio: ['ignore', 'pipe', 'pipe'],
+			timeout: 5_000,
+			killSignal: 'SIGTERM',
+		});
+		if (r.status !== 0 || r.signal || !r.stdout) return undefined;
+		const out = r.stdout.toString().trim();
+		return out.length > 4 ? out : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+export function keyFilePath(): string {
+	return join(wsearchDir(), 'env');
+}
+
+export function disabledFlagPath(): string {
+	return join(wsearchDir(), 'off');
+}
+
+function wsearchDir(): string {
+	return process.env.WSEARCH_CACHE_DIR ?? join(homedir(), '.pi', 'agent', 'wsearch');
+}
+
+let cached: WebConfig | undefined;
+let origins: Record<string, 'env' | 'wsearch' | 'shell'> = {};
+export function resetConfigCache(): void {
+	cached = undefined;
+	origins = {};
+}
+
+// !command resolution is synchronous (spawnSync, 5s each, once per process).
+export function getConfig(): WebConfig {
+	if (cached) return cached;
+	const env = process.env;
+
+	const keysFile = readEnvText(keyFilePath());
+	const shell = {} as Record<string, string>;
+	for (const f of shellConfigFiles()) {
+		const map = readEnvText(join(homedir(), f));
+		for (const [k, v] of Object.entries(map)) if (!(k in shell)) shell[k] = v;
+	}
+
+	const find = (spec: KeySpec): string | undefined => {
+		for (const name of spec.envNames) {
+			if (env[name] && env[name].length > 4) {
+				const v = resolveStoredValue(env[name]);
+				if (v) {
+					origins[spec.id] = 'env';
+					return v;
+				}
+			}
+			if (keysFile[name]) {
+				const v = resolveStoredValue(keysFile[name]);
+				if (v) {
+					origins[spec.id] = 'wsearch';
+					return v;
+				}
+			}
+			if (shell[name]) {
+				const v = resolveStoredValue(shell[name]);
+				if (v) {
+					origins[spec.id] = 'shell';
+					return v;
+				}
+			}
+		}
+		return undefined;
+	};
+
+	cached = {
+		cacheDir: process.env.WSEARCH_CACHE_DIR ?? join(homedir(), '.pi', 'agent', 'wsearch'),
+		allowPrivate: process.env.WSEARCH_ALLOW_PRIVATE === '1',
+		searchEngines: (env.WSEARCH_ENGINES ?? keysFile.WSEARCH_ENGINES ?? shell.WSEARCH_ENGINES)?.trim() || undefined,
+		fetchChain:
+			(env.WSEARCH_FETCH_CHAIN ?? keysFile.WSEARCH_FETCH_CHAIN ?? shell.WSEARCH_FETCH_CHAIN)?.trim() || undefined,
+	};
+	// iterate the catalog keyed on spec.key: a SPECS reorder can never mis-map keys
+	for (const spec of SPECS) cached[spec.key] = find(spec);
+	return cached;
+}
+
+export function keyStatus(): Record<string, boolean> {
+	const c = getConfig();
+	const out: Record<string, boolean> = {};
+	for (const spec of SPECS) out[spec.id] = !!c[spec.key];
+	return out;
+}
+
+export function keyOrigins(): Record<string, 'env' | 'wsearch' | 'shell'> {
+	getConfig();
+	return { ...origins };
+}
+
+export function sanitizeError(e: unknown): string {
+	let msg = e instanceof Error ? e.message : String(e);
+	const cfg = getConfig();
+	for (const spec of SPECS) {
+		const secret = cfg[spec.key];
+		if (secret && secret.length > 8) msg = msg.split(secret).join('***');
+	}
+	msg = msg.replace(/\b(?:sk-[A-Za-z0-9_-]{12,}|Bearer\s+[A-Za-z0-9._-]{12,}|X-API-KEY[^\n]*)\b/gi, '***');
+	return msg.slice(0, 240);
+}

package/src/constants.ts

@@ -0,0 +1,6 @@
+/** Shared constants: single home for facts that would otherwise drift across modules. */
+
+export const DEFAULT_MAX_RESULTS = 8;
+
+/** The abort message: a UX contract (pi renders isError red), one home. */
+export const ABORT_ERROR = 'aborted (user interrupt)';