pi-wsearch 0.1.0 → 0.2.1

package/README.md

@@ -34,10 +34,10 @@ you to find the signal. `web.fetch` breaks any page into an outline of every
 section and answers by number: `{url, sections: [0, 3]}` fetches just two
 sections of the fetched page. The agent surveys the map, picks what matters,
 and reads only those. No
-full reads, no 40 KB dumps. Truncation is layered: sections over 3,000
-characters are cut with the tail marked unreachable; pages that overflow are
-capped; the final response is clipped. Every cut is marked; the truncated
-part is named, not lost.
+full reads, no 40 KB dumps. Sections over 3,000 characters split into
+addressable continuation entries (`Section (cont. 1)`), so no tail is ever
+unreachable; pages that overflow are capped; the final response is clipped.
+Every cut is marked — nothing is silently lost.
 
 **Reads are free.** Page parsing is built in, nothing billed per read, no key
 required. Real headings become the map, TOC echoes get dropped, so reads come

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Pi extension: web.search + web.fetch",
   "license": "MIT",
   "type": "module",
@@ -59,5 +59,16 @@
     "oxlint": "^1.86.0",
     "typescript": "^7.0.2"
   },
-  "dependencies": {}
-}
+  "dependencies": {},
+  "main": "index.ts",
+  "files": [
+    "index.ts",
+    "src/",
+    "README.md",
+    "LICENSE",
+    "!CHANGELOG.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/cache.ts

@@ -28,11 +28,6 @@ export class TtlCache {
 			const raw = JSON.parse(readFileSync(this.file, 'utf8'));
 			if (raw && typeof raw === 'object' && typeof raw.entries === 'object') {
 				this.store = raw;
-				const now = Date.now();
-				// purge expired entries AND their order slots: dead keys must not grow across restarts
-				for (const k of Object.keys(this.store.entries)) {
-					if ((this.store.entries[k]?.exp ?? 0) <= now) delete this.store.entries[k];
-				}
 				this.store.order = this.store.order.filter((k) => this.store.entries[k] !== undefined);
 			}
 		} catch {
@@ -51,17 +46,21 @@ export class TtlCache {
 	}
 	get(key: string): unknown | undefined {
 		const e = this.store.entries[key];
-		if (!e) return undefined;
-		if (e.exp < Date.now()) {
-			delete this.store.entries[key];
-			this.store.order = this.store.order.filter((k) => k !== key);
-			return undefined;
-		}
+		if (!e || e.exp < Date.now()) return undefined;
 		const i = this.store.order.indexOf(key);
 		if (i >= 0) this.store.order.splice(i, 1);
 		this.store.order.push(key);
 		return e.value;
 	}
+
+	/** Expired copy WITHOUT deleting it (stale-if-error reserve). */
+	peekStale(key: string): { value: unknown; ageHours: number } | undefined {
+		const e = this.store.entries[key];
+		if (!e) return undefined;
+		const ageHours = (Date.now() - e.exp) / 3600_000;
+		if (ageHours <= 0) return undefined;
+		return { value: e.value, ageHours };
+	}
 	set(key: string, value: unknown, ttlMs: number) {
 		this.store.entries[key] = { exp: Date.now() + ttlMs, value };
 		const i = this.store.order.indexOf(key);

package/src/config.ts

@@ -181,6 +181,8 @@ function wsearchDir(): string {
 	return process.env.WSEARCH_CACHE_DIR ?? join(homedir(), '.pi', 'agent', 'wsearch');
 }
 
+export { wsearchDir };
+
 let cached: WebConfig | undefined;
 let origins: Record<string, 'env' | 'wsearch' | 'shell'> = {};
 export function resetConfigCache(): void {

package/src/engines.ts

@@ -75,6 +75,18 @@ async function request<T>(
 	}
 }
 
+function extractSiteOperator(query: string): { query: string; domain?: string } {
+	const m = /(?:^|\s)site:([a-z0-9.-]+[a-z0-9])(?:\s|$)/i.exec(query);
+	if (!m) return { query };
+	return {
+		query: query
+			.replace(m[0], ' ')
+			.replace(/\s{2,}/g, ' ')
+			.trim(),
+		domain: m[1].toLowerCase(),
+	};
+}
+
 async function serper(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
 	const t0 = Date.now();
 	try {
@@ -104,12 +116,14 @@ async function serper(key: string, opts: SearchOptions, signal?: AbortSignal): P
 async function tavily(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
 	const t0 = Date.now();
 	try {
+		const { query, domain } = extractSiteOperator(opts.query);
 		const body: Record<string, unknown> = {
-			query: opts.query,
+			query,
 			max_results: opts.maxResults ?? DEFAULT_MAX_RESULTS,
 			search_depth: 'basic',
 			topic: 'general',
 		};
+		if (domain) body.include_domains = [domain];
 		// Docs: auth = Authorization: Bearer header
 		const d = await postJson<{
 			results?: { title?: string; url?: string; content?: string; published_date?: string }[];
@@ -132,11 +146,13 @@ async function tavily(key: string, opts: SearchOptions, signal?: AbortSignal): P
 async function exa(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
 	const t0 = Date.now();
 	try {
+		const { query, domain } = extractSiteOperator(opts.query);
 		const body: Record<string, unknown> = {
-			query: opts.query,
+			query,
 			numResults: opts.maxResults ?? DEFAULT_MAX_RESULTS,
 			contents: { highlights: true },
 		};
+		if (domain) body.includeDomains = [domain];
 		const d = await postJson<{
 			results?: { title?: string; url?: string; highlights?: string[]; publishedDate?: string }[];
 		}>('https://api.exa.ai/search', body, jsonHeaders({ 'x-api-key': key }), signal);
@@ -360,6 +376,13 @@ function parseEngineList(raw: string | undefined): EngineName[] | undefined {
 	return out.length ? out : undefined;
 }
 
+const deadEngines = new Set<string>();
+const STABLE_ENGINE_ERROR = /HTTP 40[123]/;
+
+export function clearDeadEngines(): void {
+	deadEngines.clear();
+}
+
 function selectEngines(opts: SearchOptions): EngineDef[] {
 	const cfg = getConfig();
 	const wanted = opts.engines ?? parseEngineList(cfg.searchEngines);
@@ -372,7 +395,7 @@ function selectEngines(opts: SearchOptions): EngineDef[] {
 
 export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]> {
 	const cfg = getConfig();
-	const chosen = selectEngines(opts);
+	const chosen = selectEngines(opts).filter((def) => !deadEngines.has(def.id));
 	if (chosen.length === 0) {
 		const wanted = opts.engines ?? parseEngineList(getConfig().searchEngines);
 		const error = wanted?.length
@@ -380,5 +403,11 @@ export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]>
 			: 'no search engine keys configured; run /websearch to add one';
 		return [{ engine: 'none', hits: [], error, latencyMs: 0 }];
 	}
-	return Promise.all(chosen.map((def) => def.search(cfg[def.key] as string, opts, opts.signal)));
+	return Promise.all(
+		chosen.map(async (def) => {
+			const outcome = await def.search(cfg[def.key] as string, opts, opts.signal);
+			if (outcome.error && STABLE_ENGINE_ERROR.test(outcome.error)) deadEngines.add(def.id);
+			return outcome;
+		}),
+	);
 }

package/src/fuse.ts

@@ -148,6 +148,36 @@ function exactMatchBoost(hit: { title: string; snippet: string }, query: string
 	return 0.3 * Math.min(1, matched / terms.length);
 }
 
+function bm25Score(hit: { title: string; snippet: string }, terms: string[]): number {
+	if (!terms.length) return 0;
+	const title = hit.title.toLowerCase();
+	const snip = hit.snippet.toLowerCase();
+	let score = 0;
+	for (const t of terms) {
+		const tfTitle = countTerm(t, title);
+		const tfSnip = countTerm(t, snip);
+		if (!tfTitle && !tfSnip) continue;
+		const idf = 1 + Math.min(1, 12 / Math.max(t.length, 1));
+		const saturate = (tf: number) => (tf * 2.2) / (tf + 1.2);
+		score += idf * (2 * saturate(tfTitle) + saturate(tfSnip));
+	}
+	return score;
+}
+
+function countTerm(term: string, text: string): number {
+	let n = 0;
+	let i = 0;
+	for (;;) {
+		const m = text.indexOf(term, i);
+		if (m < 0) break;
+		const before = m === 0 ? ' ' : text[m - 1];
+		const after = m + term.length >= text.length ? ' ' : text[m + term.length];
+		if (!/[a-z0-9]/.test(before) && !/[a-z0-9]/.test(after)) n++;
+		i = m + term.length;
+	}
+	return n;
+}
+
 function normTitle(s: string): string {
 	return s
 		.toLowerCase()
@@ -188,6 +218,7 @@ function collapseNearDuplicates(fused: FusedHit[]): FusedHit[] {
 
 export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHit[] {
 	const { query } = opts;
+	const terms = query ? queryTerms(query) : [];
 	const rank: Record<string, { hit: SearchHit; ranks: { engine: string; rank: number }[] }> = {};
 	let answering = 0;
 	for (const oc of outcomes) {
@@ -212,6 +243,8 @@ export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHi
 		const consensus = answering > 0 ? ranks.length / answering : 0;
 		let score = rrf + CONSENSUS_TIEBREAK * consensus;
 		score *= 1 + exactMatchBoost(hit, query);
+		const bm25 = bm25Score(hit, terms);
+		if (bm25 > 0) score *= 1 + Math.min(0.25, 0.045 * bm25);
 		if (hit.snippet.trim().length < 30) score *= 0.92;
 		if (junk) score *= 0.25;
 		fused.push({

package/src/keys.ts

@@ -83,7 +83,9 @@ export function statusText(): string {
 		armed.length ? `  armed:   ${armed.map((id) => `${id} ✓(${origins[id] ?? '?'})`).join(', ')}` : '  armed:   (none)',
 	);
 	lines.push(`  dormant: ${dormant.length ? dormant.join(', ') : '(none)'}`);
-	const chain = cfg.fetchChain ? cfg.fetchChain.replace(/,/g, ', ') : 'firecrawl, tavily, exa, jina';
+	const chain = cfg.fetchChain
+		? cfg.fetchChain.replace(/,/g, ', ')
+		: 'jina, tinyfish, firecrawl, tavily, exa, serper, you, kagi';
 	lines.push(`  fetch:   local → ${chain}`);
 	const knobs = [];
 	if (cfg.searchEngines) knobs.push(`WSEARCH_ENGINES=${cfg.searchEngines}`);
@@ -97,6 +99,7 @@ export function statusText(): string {
 		lines.push('     · exa       → exa.ai        (1,000 free/mo)');
 		lines.push('     · firecrawl → firecrawl.dev (1,000 free credits/mo)');
 		lines.push('     · tinyfish  → tinyfish.ai   (free at any balance)');
+		lines.push('     · jina      → jina.ai       (free tier, search + reader)');
 		lines.push('  2. /websearch login, saved to wsearch/env');
 		lines.push('  3. run /websearch again; it should appear under "armed"');
 	}