pi-wsearch 0.6.0 → 0.7.0

package/README.md

@@ -13,7 +13,7 @@ ranked that page and where each placed it. A hit returned by one engine alone
 shows just that engine and rank: `⇐ exa#3`. Agreement is the spam defense;
 generative-SEO can stuff one index, not three.
 
-**Pick your engines.** Nine are supported; three or four is plenty. Use
+**Pick your engines.** Ten are supported; three or four is plenty. Use
 whatever keys you have, even one:
 
 - **Rare strings match literally.** `pgvector`, CVE numbers, error codes match
@@ -26,7 +26,7 @@ whatever keys you have, even one:
   answers.
 
 **Supported providers:** Serper, Tavily, Exa, Brave, Jina, Kagi, You.com,
-Firecrawl, TinyFish
+Firecrawl, TinyFish, Parallel, Keenable
 
 ### WebFetch
 
@@ -46,11 +46,8 @@ local budget cuts.
 **Reads are free.** Page parsing is built in, bills nothing per read, and
 needs no key. Real headings become the map, TOC echoes get dropped. Paid
 readers only step in for JavaScript-heavy pages or when the local read fails.
-Reader output is laundered: standalone photo
-credits (REUTERS/, AP Photo/, Getty), video
-durations, carousel markers and relative-time chips are stripped from news
-pages (whole-line rules only; prose is untouched), and a reader that returns
-a 404-page template is skipped so the chain moves on to the next backend.
+A reader that returns a 404-page template is skipped so the chain moves on to
+the next backend.
 
 **Tracking links cache as one page.** `?utm_source=…` variants of the same URL
 share one 24 hour cache entry. Faster repeats, smaller disk.
@@ -63,7 +60,7 @@ mapped IPv6 forms included) are rejected before any request is made.
 as untrusted input: a claim only becomes a verdict after a second source
 corroborates it.
 
-**Optional readers:** Firecrawl, Tavily, Exa, Jina, fallback only
+**Optional readers:** Firecrawl, Tavily, Exa, Jina, Parallel, Keenable, fallback only
 
 ## Install
 
@@ -100,8 +97,8 @@ Search needs at least one provider key; fetch needs none. Two ways to set one:
 Either place accepts a `!command` value, so the key can come from Keychain or
 1Password at use time and never sit on disk. Session env wins over the file,
 the file over shell config. `/websearch` shows which providers are armed and
-which are dormant. No keys yet? Tavily, Exa, Firecrawl, and TinyFish all have
-free tiers.
+which are dormant. No keys yet? Tavily, Exa, Firecrawl, TinyFish, Parallel, and
+Keenable have free tiers; Keenable's starts at 100K requests/mo.
 
 ## Commands
 
@@ -138,7 +135,7 @@ One command, five subcommands:
    /websearch
    ```
 
-   Your provider should appear under `armed:`.
+   Notifies you on your current configured providers.
 4. Ask the agent something current:
 
    > What's the current Node.js LTS version?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Pi extension: WebSearch + WebFetch",
   "license": "MIT",
   "type": "module",

package/src/cache.ts

@@ -1,79 +1,111 @@
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { rename, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 
 const DEFAULT_CAP = 400;
+const STALE_RESERVE_MS = 24 * 3600_000;
 
 interface Entry {
 	exp: number;
 	value: unknown;
 }
 
-interface Store {
-	entries: Record<string, Entry>;
-	order: string[];
-}
-
+/** Map iteration order = insertion order, so delete+re-set IS the LRU move;
+ * no parallel order array to splice. */
 export class TtlCache {
-	private store: Store = { entries: {}, order: [] };
+	private entries = new Map<string, Entry>();
+	private loaded = false;
+	private write = Promise.resolve();
 	constructor(
 		private file: string,
 		private cap = DEFAULT_CAP,
-	) {
-		this.load();
-	}
-	private load() {
+	) {}
+
+	private ensureLoaded() {
+		if (this.loaded) return;
+		this.loaded = true;
 		try {
 			if (!existsSync(this.file)) return;
-			const raw = JSON.parse(readFileSync(this.file, 'utf8'));
-			if (raw && typeof raw === 'object' && typeof raw.entries === 'object') {
-				this.store = raw;
-				this.store.order = this.store.order.filter((k) => this.store.entries[k] !== undefined);
+			const raw = JSON.parse(readFileSync(this.file, 'utf8')) as {
+				entries?: Record<string, Entry>;
+				order?: string[];
+			} | null;
+			if (!raw || typeof raw !== 'object' || typeof raw.entries !== 'object') return;
+			const total = Object.keys(raw.entries).length;
+			const cutoff = Date.now() - STALE_RESERVE_MS;
+			// Legacy files carried an explicit LRU `order` array: seed the Map in
+			// that order so eviction under the cap starts at the true oldest.
+			const order: string[] = Array.isArray(raw.order)
+				? [...raw.order.filter((k): k is string => typeof k === 'string'), ...Object.keys(raw.entries)]
+				: Object.keys(raw.entries);
+			for (const k of order) {
+				const e = raw.entries[k];
+				if (typeof e?.exp !== 'number' || !Number.isFinite(e.exp)) continue; // corrupt: dead
+				if (e.exp <= cutoff) continue; // beyond the stale reserve: dead
+				this.entries.set(k, e);
 			}
+			// Rewrite only when a stale/corrupt entry was actually dropped; a
+			// healthy file must never be rewritten on load.
+			if (this.entries.size !== total) this.persist();
 		} catch {
-			this.store = { entries: {}, order: [] };
+			this.entries.clear();
 		}
 	}
+
 	private persist() {
 		try {
 			mkdirSync(dirname(this.file), { recursive: true });
-			const tmp = this.file + '.tmp';
-			writeFileSync(tmp, JSON.stringify(this.store), 'utf8');
-			renameSync(tmp, this.file);
+			const tmp = `${this.file}.tmp`;
+			this.write = this.write
+				.then(() => JSON.stringify({ entries: Object.fromEntries(this.entries) }))
+				.then((data) => writeFile(tmp, data, 'utf8'))
+				.then(() => rename(tmp, this.file))
+				.catch(() => {});
 		} catch {
 			/* cache writes are best-effort */
 		}
 	}
-	/** Entry valid while unexpired; non-numeric/NaN exp (corrupt store) must NOT
-	 * count as "never expires" — treat it as expired so a poisoned entry dies. */
+
+	flush(): Promise<void> {
+		return this.write;
+	}
+
+	/** Entry valid while unexpired; corrupt exp (poisoned store) counts as
+	 * expired, never "never expires". */
 	get(key: string): unknown | undefined {
-		const e = this.store.entries[key];
-		if (!e || typeof e.exp !== 'number' || !Number.isFinite(e.exp) || e.exp < Date.now()) return undefined;
-		const i = this.store.order.indexOf(key);
-		if (i >= 0) this.store.order.splice(i, 1);
-		this.store.order.push(key);
+		this.ensureLoaded();
+		const e = this.entries.get(key);
+		if (!e || e.exp < Date.now()) return undefined;
+		this.touch(key, e);
 		return e.value;
 	}
 
 	/** Expired copy WITHOUT deleting it (stale-if-error reserve). */
 	peekStale(key: string): { value: unknown; ageHours: number } | undefined {
-		const e = this.store.entries[key];
+		this.ensureLoaded();
+		const e = this.entries.get(key);
 		if (!e || typeof e.exp !== 'number' || !Number.isFinite(e.exp)) return undefined;
 		const ageHours = (Date.now() - e.exp) / 3600_000;
 		if (ageHours <= 0) return undefined;
 		return { value: e.value, ageHours };
 	}
+
 	set(key: string, value: unknown, ttlMs: number) {
-		this.store.entries[key] = { exp: Date.now() + ttlMs, value };
-		const i = this.store.order.indexOf(key);
-		if (i >= 0) this.store.order.splice(i, 1);
-		this.store.order.push(key);
-		while (this.store.order.length > this.cap) {
-			const old = this.store.order.shift()!;
-			delete this.store.entries[old];
+		this.ensureLoaded();
+		this.touch(key, { exp: Date.now() + ttlMs, value });
+		while (this.entries.size > this.cap) {
+			const oldest = this.entries.keys().next().value;
+			if (oldest === undefined) break;
+			this.entries.delete(oldest);
 		}
 		this.persist();
 	}
+
+	private touch(key: string, e: Entry) {
+		this.entries.delete(key);
+		this.entries.set(key, e);
+	}
 }
 
 export function cacheKey(parts: string[]): string {

package/src/config.ts

@@ -9,6 +9,11 @@ import { spawnSync } from 'node:child_process';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 
+export const DEFAULT_MAX_RESULTS = 8;
+
+/** The abort message: a UX contract (pi renders isError red), one home. */
+export const ABORT_ERROR = 'aborted (user interrupt)';
+
 export interface WebConfig {
 	serperKey?: string;
 	tavilyKey?: string;
@@ -19,6 +24,8 @@ export interface WebConfig {
 	kagiKey?: string;
 	youKey?: string;
 	tinyfishKey?: string;
+	parallelKey?: string;
+	keenableKey?: string;
 	cacheDir: string;
 	allowPrivate?: boolean;
 	searchEngines?: string;
@@ -34,7 +41,9 @@ export type ProviderKey =
 	| 'jinaKey'
 	| 'kagiKey'
 	| 'youKey'
-	| 'tinyfishKey';
+	| 'tinyfishKey'
+	| 'parallelKey'
+	| 'keenableKey';
 
 interface KeySpec {
 	id: string;
@@ -52,6 +61,8 @@ export const SPECS: KeySpec[] = [
 	{ id: 'kagi', key: 'kagiKey', envNames: ['KAGI_API_KEY', 'KAGI_KEY'] },
 	{ id: 'you', key: 'youKey', envNames: ['YOUCOM_API_KEY', 'YDC_API_KEY'] },
 	{ id: 'tinyfish', key: 'tinyfishKey', envNames: ['TINYFISH_API_KEY', 'TINYFISH_KEY'] },
+	{ id: 'parallel', key: 'parallelKey', envNames: ['PARALLEL_API_KEY', 'PARALLEL_KEY'] },
+	{ id: 'keenable', key: 'keenableKey', envNames: ['KEENABLE_API_KEY', 'KEENABLE_KEY'] },
 ];
 
 export interface ProviderInfo {
@@ -71,6 +82,8 @@ const PROVIDER_LABELS: Record<string, string> = {
 	kagi: 'Kagi',
 	you: 'You.com',
 	tinyfish: 'TinyFish',
+	parallel: 'Parallel',
+	keenable: 'Keenable',
 };
 
 export const PROVIDERS: ProviderInfo[] = SPECS.map((s) => ({

package/src/dom.ts

@@ -215,15 +215,15 @@ function renderBlock(n: DomNode, out: string[]): void {
 	}
 }
 
-const DATE_KEYWORDS = ['datePublished', 'dateCreated', 'uploadDate', 'publishedAt'];
-const DATE_META_NAMES = [
+const DATE_KEYWORDS = new Set(['datePublished', 'dateCreated', 'uploadDate', 'publishedAt']);
+const DATE_META_NAMES = new Set([
 	'article:published_time',
 	'datePublished',
 	'publish-date',
 	'pubdate',
 	'parsely-pub-date',
 	'date',
-];
+]);
 
 function toIso(v: string | undefined): string | undefined {
 	if (!v) return undefined;
@@ -232,14 +232,15 @@ function toIso(v: string | undefined): string | undefined {
 	return new Date(ms).toISOString().slice(0, 10);
 }
 
+const pickIsoDate = (value: unknown): string | undefined => {
+	if (typeof value === 'string') {
+		const iso = toIso(value);
+		if (iso) return iso;
+	}
+	return undefined;
+};
+
 function jsonLdDate(nodes: DomNode[]): string | undefined {
-	const pick = (value: unknown): string | undefined => {
-		if (typeof value === 'string') {
-			const iso = toIso(value);
-			if (iso) return iso;
-		}
-		return undefined;
-	};
 	const walk = (value: unknown): string | undefined => {
 		if (Array.isArray(value)) {
 			for (const v of value) {
@@ -250,7 +251,7 @@ function jsonLdDate(nodes: DomNode[]): string | undefined {
 		}
 		if (typeof value === 'object' && value !== null) {
 			for (const key of DATE_KEYWORDS) {
-				const hit = pick((value as Record<string, unknown>)[key]);
+				const hit = pickIsoDate((value as Record<string, unknown>)[key]);
 				if (hit) return hit;
 			}
 			for (const v of Object.values(value as Record<string, unknown>)) {
@@ -317,7 +318,7 @@ export function headScan(html: string): HeadScan {
 		if (name === 'meta') {
 			const key = (attribs['property'] ?? attribs['name'] ?? '').toLowerCase().trim();
 			if (!ogTitle && key === 'og:title') ogTitle = (attribs['content'] ?? '').trim().slice(0, 200);
-			if (!metaDate && DATE_META_NAMES.includes(key)) metaDate = toIso(attribs['content']);
+			if (!metaDate && DATE_META_NAMES.has(key)) metaDate = toIso(attribs['content']);
 		}
 		if (name === 'time' && !timeDate && attribs['datetime']) timeDate = toIso(attribs['datetime']);
 		for (const c of n.children ?? []) visit(c);