pi-wsearch 0.1.0 → 0.2.0

package/CHANGELOG.md

@@ -0,0 +1,62 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+## [Unreleased]
+
+### Added
+
+- Over-cap sections split into addressable `(cont. N)` outline entries; no section tail is ever unreachable.
+- Promo/nag blocks (premium upsells, newsletter signups) filtered from outline and section bodies.
+- Adjacent verbatim-duplicate paragraphs dropped.
+- Headingless pages get a synthesized `(lead)` outline entry.
+- Heading-only stub sections (page chrome like "## Latest commit", "### Stars" with no body) dropped — a real section carries prose under its heading.
+- GitHub repo pages serve the raw README under the original URL (repo pages are ~90% UI chrome); filename candidates tried in order, graceful fall-through to the normal page path on failure; non-root paths (blob/tree/issues) unaffected.
+- Local extraction parity with paid readers: HTML tables → markdown pipe tables (structure preserved instead of flattened to word soup); MathML formulas → embedded TeX when annotated; KaTeX span soup (client-rendered, no TeX source in HTML) dropped instead of linearizing into scrambled fragments.
+- News-page hard-sample cleanup: per-line junk filtering (photo credits like "Chip Somodevilla / Getty Images file", orphan "38d ago" card fragments, IE-browser nags, "Skip to Content" links, "FOR SUBSCRIBERS" nags), ALL-CAPS card kickers, empty `##` heading debris, bullet leader-line collapsing, and `<title>`-echo blocks dropped — NBC/CBS-class hub pages now open directly on their story list.
+- Broader hard-sample sweep (forums, docs SPAs, helpdesks, recipes, e-commerce): newsletter signup modules ("#### Email Newsletter" + email-field trust lines) dropped; access-notice pages (People Inc-style HTTP 402 with an HTML notice) treated as bot-walls — UA retry + chain fallback, and their notice text can never pass as content.
+- Keyless jina reader tier: runs without a key when jina is explicitly named in `WSEARCH_FETCH_CHAIN` (opt-in, rate-limited ~20 req/min); a configured key also enables it.
+- Reader backends for kagi (Universal Summarizer — last in chain: lossy summaries never pre-empt full extraction; covers PDFs/office docs/YouTube), TinyFish (Fetch API, free at any balance), serper (Webpage Scrape) and You.com (Contents API) — key-gated, docs-verified contracts.
+- Junk list covers social-syndication mirrors (twiscan, twstalker, unrollnow, threadreaderapp, nitter, upd.dev); any numeric `status|thread/<id>` path on any host is junk (uniform rule, no favored-host allowlist).
+- `engines_errored` structured field plus `⚠ engines:` diagnostics line on search output; engines with a stable 401/402/403 error are skipped for the rest of the process (memo cleared on `/websearch login|logout`).
+- Reader-chain exhaustion now reports every backend's error, not just the last one.
+- Intrinsic BM25-flavored lexical rescoring in the fuse stage (title hits 2x, IDF-lite rarity via token length, bounded 0.25 multiplier): single-engine hits with strong query overlap reorder near-ties; multi-engine agreement still dominates. Deterministic, model-free.
+- `/websearch` onboarding lists jina's free tier among the cheapest paths.
+- Observe-only fetch-through ledger (`wsearch/fetchtrace.jsonl`): search records each result's rank/engine-agreement; a successful first fetch of a search-surfaced URL appends the tuple (query, rank, engines, url). Pure measurement — zero behavior change — the ground-truth layer for judging future ranking changes (top-3 fetch-decision precision).
+- URL canonicalization depth-up: paths case-folded and duplicate slashes collapsed (server-equivalent spellings no longer split RRF agreement across "different" URLs).
+- Mojibake detection and repair in the byte-decode path: UTF-8-read-as-CP1252 double-encoding (declared charset lies, upstream proxy mangling) is detected by signature density and re-decoded; clean and declared-latin1 pages pass through untouched.
+- Stale-if-error cache reserve: expired entries survive as a reserve instead of being purged; when a previously-readable page's live fetch fails (wall tightened, host down), the expired copy is served with a `[STALE — ~Nh old]` title marker rather than an error. `get()` never serves expired copies; the reserve is replaced on overwrite and capped by the LRU cap.
+- FineWeb/Gopher repetition gates on extracted text: >30% duplicate lines or >20% repeated unique 5-grams marks the output low-quality (catches link-farms and menu-soup that survive every pattern filter; published thresholds, not invented ones).
+- Conditional revalidation (ETag/If-None-Match, Last-Modified/If-Modified-Since): cached copies store their validators; an expired copy revalidates before re-download — a 304 re-marks it fresh for free (local path only; reader-chain results carry no validators).
+- Fetched pages now carry `publishedAt` (JSON-LD datePublished → article:published_time meta → <time datetime>, first match wins) and a `paywalled: true` flag when soft-paywall overlay signatures (fc-ab-root, metering-modal, isAccessibleForFree:false) are present — detection only, never bypass.
+- GitHub blob URLs (single source files) serve the raw file content under the original URL, same contract as the README preference — the repo-page path buried files under navigation UI.
+- Cookie-consent banner boilerplate ("We use cookies to improve…", "By continuing to browse…", "Accept all", "Manage preferences") filtered.
+- Page-wide duplicate-paragraph removal: paragraphs repeated verbatim anywhere on a page are dropped from their later occurrences (adjacent-dedupe caught only back-to-back copies; hub pages re-serve card text across sections — pure token waste).
+- Accept header prefers `text/markdown` over HTML: sites with server-side content negotiation (agent-readability ecosystem) now hand over their clean markdown rendition; non-negotiating servers ignore the header entirely.
+- Zero-result recovery: when a search returns nothing, one automatic retry runs with operators stripped (`site:`, quotes, `-term`) or the rarest term dropped — the agent no longer burns a round-trip to do it manually; the retry is noted in the output.
+- `site:` operator normalization: routed to native domain-filter params where supported (tavily `include_domains`, exa `includeDomains`), passed through verbatim for engines with native operator support (serper, brave, kagi) — no more diluting domain-restricted queries into full-web text search.
+- Search result titles strip the site-name suffix ("Page Title | Site Name") — display noise and title-token pollution.
+- Search results display a normalized date (ISO, from engine date fields incl. relative forms like "3 days ago") when the engine provides one.
+- Archive fallback for dead pages: a 404 (dead host, removed docs) tries archive.org's Wayback availability API and serves the snapshot through the normal local path, title-marked `[ARCHIVED copy <date> — live page is gone]`. Doctrine line: 404/dead-host only, never 403-walls; best-effort (lookup failure → plain error as before).
+
+### Changed
+
+- Fetch cache key bumped to `v3` (extraction pipeline shape changed).
+- `OUTLINE_CAP` raised from 14 to 20.
+- Reader chain default order: jina (free tier) → tinyfish (free at any balance) → firecrawl → tavily → exa → serper → you → kagi (lossy summarizer last).
+- Permanent local-failure guard narrowed to 404/oversized/empty: 403 bot-walls now advance to the chain (renderers exist to read them); `r.jina.ai`/`s.jina.ai` proxy URLs never enter the chain.
+- Search engines pin `WSEARCH_ENGINES` and readers pin `WSEARCH_FETCH_CHAIN` at runtime as before; chain status display mirrors the new default order.
+
+### Fixed
+
+- `splitSection` infinite loop on space-free runs (base64 blobs, packed table cells) — cut enforces a hard progress floor.
+- Dropped tiny preambles (e.g. leaked `<title>` text) no longer shift heading names off by one.
+- Reader chain was spent on permanent local failures (404/oversized/empty URLs) — now returns the real status instantly instead of burning paid backends.
+- Jina reader gate contradicted its keyless-capable read implementation (blocked keyless reads despite tolerant code) — gate aligned to the explicit opt-in.
+- Dead engines (e.g. firecrawl with exhausted credits) dragged every search fan-out ~7s — skipped after the first stable failure.
+- `/websearch off|on` withdrew/restored tools at registration only, leaving them declared to the model until a manual reload — the active tool set is now synced via `pi.setActiveTools()` so the change applies on the next model request.
+- Fetching `r.jina.ai/<target>` sent paid renderers at a proxy URL (a renderer cannot render a proxy) — now fails fast with the proxy's own status.
+
+### Removed
+
+- `CAP_BREAK_GRACE` boundary heuristic (subsumed by the progress floor).

package/README.md

@@ -34,10 +34,10 @@ you to find the signal. `web.fetch` breaks any page into an outline of every
 section and answers by number: `{url, sections: [0, 3]}` fetches just two
 sections of the fetched page. The agent surveys the map, picks what matters,
 and reads only those. No
-full reads, no 40 KB dumps. Truncation is layered: sections over 3,000
-characters are cut with the tail marked unreachable; pages that overflow are
-capped; the final response is clipped. Every cut is marked; the truncated
-part is named, not lost.
+full reads, no 40 KB dumps. Sections over 3,000 characters split into
+addressable continuation entries (`Section (cont. 1)`), so no tail is ever
+unreachable; pages that overflow are capped; the final response is clipped.
+Every cut is marked — nothing is silently lost.
 
 **Reads are free.** Page parsing is built in, nothing billed per read, no key
 required. Real headings become the map, TOC echoes get dropped, so reads come

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-wsearch",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Pi extension: web.search + web.fetch",
   "license": "MIT",
   "type": "module",
@@ -59,5 +59,16 @@
     "oxlint": "^1.86.0",
     "typescript": "^7.0.2"
   },
-  "dependencies": {}
+  "dependencies": {},
+  "main": "index.ts",
+  "files": [
+    "index.ts",
+    "src/",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  }
 }

package/src/cache.ts

@@ -28,11 +28,6 @@ export class TtlCache {
 			const raw = JSON.parse(readFileSync(this.file, 'utf8'));
 			if (raw && typeof raw === 'object' && typeof raw.entries === 'object') {
 				this.store = raw;
-				const now = Date.now();
-				// purge expired entries AND their order slots: dead keys must not grow across restarts
-				for (const k of Object.keys(this.store.entries)) {
-					if ((this.store.entries[k]?.exp ?? 0) <= now) delete this.store.entries[k];
-				}
 				this.store.order = this.store.order.filter((k) => this.store.entries[k] !== undefined);
 			}
 		} catch {
@@ -51,17 +46,21 @@ export class TtlCache {
 	}
 	get(key: string): unknown | undefined {
 		const e = this.store.entries[key];
-		if (!e) return undefined;
-		if (e.exp < Date.now()) {
-			delete this.store.entries[key];
-			this.store.order = this.store.order.filter((k) => k !== key);
-			return undefined;
-		}
+		if (!e || e.exp < Date.now()) return undefined;
 		const i = this.store.order.indexOf(key);
 		if (i >= 0) this.store.order.splice(i, 1);
 		this.store.order.push(key);
 		return e.value;
 	}
+
+	/** Expired copy WITHOUT deleting it (stale-if-error reserve). */
+	peekStale(key: string): { value: unknown; ageHours: number } | undefined {
+		const e = this.store.entries[key];
+		if (!e) return undefined;
+		const ageHours = (Date.now() - e.exp) / 3600_000;
+		if (ageHours <= 0) return undefined;
+		return { value: e.value, ageHours };
+	}
 	set(key: string, value: unknown, ttlMs: number) {
 		this.store.entries[key] = { exp: Date.now() + ttlMs, value };
 		const i = this.store.order.indexOf(key);

package/src/config.ts

@@ -181,6 +181,8 @@ function wsearchDir(): string {
 	return process.env.WSEARCH_CACHE_DIR ?? join(homedir(), '.pi', 'agent', 'wsearch');
 }
 
+export { wsearchDir };
+
 let cached: WebConfig | undefined;
 let origins: Record<string, 'env' | 'wsearch' | 'shell'> = {};
 export function resetConfigCache(): void {

package/src/engines.ts

@@ -75,6 +75,18 @@ async function request<T>(
 	}
 }
 
+function extractSiteOperator(query: string): { query: string; domain?: string } {
+	const m = /(?:^|\s)site:([a-z0-9.-]+[a-z0-9])(?:\s|$)/i.exec(query);
+	if (!m) return { query };
+	return {
+		query: query
+			.replace(m[0], ' ')
+			.replace(/\s{2,}/g, ' ')
+			.trim(),
+		domain: m[1].toLowerCase(),
+	};
+}
+
 async function serper(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
 	const t0 = Date.now();
 	try {
@@ -104,12 +116,14 @@ async function serper(key: string, opts: SearchOptions, signal?: AbortSignal): P
 async function tavily(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
 	const t0 = Date.now();
 	try {
+		const { query, domain } = extractSiteOperator(opts.query);
 		const body: Record<string, unknown> = {
-			query: opts.query,
+			query,
 			max_results: opts.maxResults ?? DEFAULT_MAX_RESULTS,
 			search_depth: 'basic',
 			topic: 'general',
 		};
+		if (domain) body.include_domains = [domain];
 		// Docs: auth = Authorization: Bearer header
 		const d = await postJson<{
 			results?: { title?: string; url?: string; content?: string; published_date?: string }[];
@@ -132,11 +146,13 @@ async function tavily(key: string, opts: SearchOptions, signal?: AbortSignal): P
 async function exa(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
 	const t0 = Date.now();
 	try {
+		const { query, domain } = extractSiteOperator(opts.query);
 		const body: Record<string, unknown> = {
-			query: opts.query,
+			query,
 			numResults: opts.maxResults ?? DEFAULT_MAX_RESULTS,
 			contents: { highlights: true },
 		};
+		if (domain) body.includeDomains = [domain];
 		const d = await postJson<{
 			results?: { title?: string; url?: string; highlights?: string[]; publishedDate?: string }[];
 		}>('https://api.exa.ai/search', body, jsonHeaders({ 'x-api-key': key }), signal);
@@ -360,6 +376,13 @@ function parseEngineList(raw: string | undefined): EngineName[] | undefined {
 	return out.length ? out : undefined;
 }
 
+const deadEngines = new Set<string>();
+const STABLE_ENGINE_ERROR = /HTTP 40[123]/;
+
+export function clearDeadEngines(): void {
+	deadEngines.clear();
+}
+
 function selectEngines(opts: SearchOptions): EngineDef[] {
 	const cfg = getConfig();
 	const wanted = opts.engines ?? parseEngineList(cfg.searchEngines);
@@ -372,7 +395,7 @@ function selectEngines(opts: SearchOptions): EngineDef[] {
 
 export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]> {
 	const cfg = getConfig();
-	const chosen = selectEngines(opts);
+	const chosen = selectEngines(opts).filter((def) => !deadEngines.has(def.id));
 	if (chosen.length === 0) {
 		const wanted = opts.engines ?? parseEngineList(getConfig().searchEngines);
 		const error = wanted?.length
@@ -380,5 +403,11 @@ export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]>
 			: 'no search engine keys configured; run /websearch to add one';
 		return [{ engine: 'none', hits: [], error, latencyMs: 0 }];
 	}
-	return Promise.all(chosen.map((def) => def.search(cfg[def.key] as string, opts, opts.signal)));
+	return Promise.all(
+		chosen.map(async (def) => {
+			const outcome = await def.search(cfg[def.key] as string, opts, opts.signal);
+			if (outcome.error && STABLE_ENGINE_ERROR.test(outcome.error)) deadEngines.add(def.id);
+			return outcome;
+		}),
+	);
 }

package/src/fuse.ts

@@ -148,6 +148,36 @@ function exactMatchBoost(hit: { title: string; snippet: string }, query: string
 	return 0.3 * Math.min(1, matched / terms.length);
 }
 
+function bm25Score(hit: { title: string; snippet: string }, terms: string[]): number {
+	if (!terms.length) return 0;
+	const title = hit.title.toLowerCase();
+	const snip = hit.snippet.toLowerCase();
+	let score = 0;
+	for (const t of terms) {
+		const tfTitle = countTerm(t, title);
+		const tfSnip = countTerm(t, snip);
+		if (!tfTitle && !tfSnip) continue;
+		const idf = 1 + Math.min(1, 12 / Math.max(t.length, 1));
+		const saturate = (tf: number) => (tf * 2.2) / (tf + 1.2);
+		score += idf * (2 * saturate(tfTitle) + saturate(tfSnip));
+	}
+	return score;
+}
+
+function countTerm(term: string, text: string): number {
+	let n = 0;
+	let i = 0;
+	for (;;) {
+		const m = text.indexOf(term, i);
+		if (m < 0) break;
+		const before = m === 0 ? ' ' : text[m - 1];
+		const after = m + term.length >= text.length ? ' ' : text[m + term.length];
+		if (!/[a-z0-9]/.test(before) && !/[a-z0-9]/.test(after)) n++;
+		i = m + term.length;
+	}
+	return n;
+}
+
 function normTitle(s: string): string {
 	return s
 		.toLowerCase()
@@ -188,6 +218,7 @@ function collapseNearDuplicates(fused: FusedHit[]): FusedHit[] {
 
 export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHit[] {
 	const { query } = opts;
+	const terms = query ? queryTerms(query) : [];
 	const rank: Record<string, { hit: SearchHit; ranks: { engine: string; rank: number }[] }> = {};
 	let answering = 0;
 	for (const oc of outcomes) {
@@ -212,6 +243,8 @@ export function fuse(outcomes: EngineOutcome[], opts: FuseOptions = {}): FusedHi
 		const consensus = answering > 0 ? ranks.length / answering : 0;
 		let score = rrf + CONSENSUS_TIEBREAK * consensus;
 		score *= 1 + exactMatchBoost(hit, query);
+		const bm25 = bm25Score(hit, terms);
+		if (bm25 > 0) score *= 1 + Math.min(0.25, 0.045 * bm25);
 		if (hit.snippet.trim().length < 30) score *= 0.92;
 		if (junk) score *= 0.25;
 		fused.push({

package/src/keys.ts

@@ -83,7 +83,9 @@ export function statusText(): string {
 		armed.length ? `  armed:   ${armed.map((id) => `${id} ✓(${origins[id] ?? '?'})`).join(', ')}` : '  armed:   (none)',
 	);
 	lines.push(`  dormant: ${dormant.length ? dormant.join(', ') : '(none)'}`);
-	const chain = cfg.fetchChain ? cfg.fetchChain.replace(/,/g, ', ') : 'firecrawl, tavily, exa, jina';
+	const chain = cfg.fetchChain
+		? cfg.fetchChain.replace(/,/g, ', ')
+		: 'jina, tinyfish, firecrawl, tavily, exa, serper, you, kagi';
 	lines.push(`  fetch:   local → ${chain}`);
 	const knobs = [];
 	if (cfg.searchEngines) knobs.push(`WSEARCH_ENGINES=${cfg.searchEngines}`);
@@ -97,6 +99,7 @@ export function statusText(): string {
 		lines.push('     · exa       → exa.ai        (1,000 free/mo)');
 		lines.push('     · firecrawl → firecrawl.dev (1,000 free credits/mo)');
 		lines.push('     · tinyfish  → tinyfish.ai   (free at any balance)');
+		lines.push('     · jina      → jina.ai       (free tier, search + reader)');
 		lines.push('  2. /websearch login, saved to wsearch/env');
 		lines.push('  3. run /websearch again; it should appear under "armed"');
 	}