pi-webfind 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,228 @@
1
+ /**
2
+ * Site adapters — route known URL shapes to a cheaper, cleaner source
3
+ * instead of scraping HTML. Each adapter returns markdown or null
4
+ * (null = no adapter / adapter failed → caller falls back to generic fetch).
5
+ *
6
+ * All free, no keys (GitHub optionally honours GITHUB_TOKEN for rate limits).
7
+ */
8
+ import { getJson } from "./apis.ts";
9
+
10
+ const UA = "pi-webfind/0.5 (free web research toolkit for pi coding agent; +https://github.com/jawwadzafar/pi-webfind)";
11
+
12
+ async function getText(url: string, signal?: AbortSignal, headers?: Record<string, string>): Promise<string> {
13
+ const timeout = AbortSignal.timeout(20_000);
14
+ const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
15
+ const res = await fetch(url, {
16
+ headers: { "User-Agent": UA, Accept: "text/plain, text/markdown, */*", ...headers },
17
+ signal: combined,
18
+ redirect: "follow",
19
+ });
20
+ if (!res.ok) throw new Error(`HTTP ${res.status} from ${new URL(url).host}`);
21
+ return res.text();
22
+ }
23
+
24
+ export interface AdapterResult {
25
+ text: string;
26
+ source: string; // e.g. "github-api"
27
+ }
28
+
29
+ type Adapter = (url: URL, signal?: AbortSignal) => Promise<AdapterResult | null>;
30
+
31
+ // ------------------------------------------------------------------- github
32
+
33
+ function ghHeaders(): Record<string, string> {
34
+ const token = process.env.GITHUB_TOKEN;
35
+ return token ? { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" } : { Accept: "application/vnd.github+json" };
36
+ }
37
+
38
+ const githubAdapter: Adapter = async (url, signal) => {
39
+ const parts = url.pathname.split("/").filter(Boolean); // [o, r, (blob|tree|issues|pull), ...]
40
+ if (parts.length < 2) return null;
41
+ const [owner, repo, kind, ...rest] = parts;
42
+
43
+ // raw file: /blob/{ref}/{path...} or /raw/{ref}/{path...}
44
+ if ((kind === "blob" || kind === "raw") && rest.length >= 2) {
45
+ const [, ref, ...path] = [kind, rest[0], ...rest.slice(1)];
46
+ const raw = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${path.join("/")}`;
47
+ const text = await getText(raw, signal);
48
+ return { text: text.slice(0, 200_000), source: "github-raw" };
49
+ }
50
+
51
+ // issue / pull: API → markdown
52
+ if (kind === "issues" || kind === "pull") {
53
+ const num = Number(rest[0]);
54
+ if (!num) return null;
55
+ const isPr = kind === "pull";
56
+ const base = `https://api.github.com/repos/${owner}/${repo}/${isPr ? "pulls" : "issues"}/${num}`;
57
+ const item = await getJson<any>(base, signal, ghHeaders());
58
+ let md = `# ${item.title ?? `${owner}/${repo}#${num}`}\n\n${isPr && item.message ? "" : ""}${item.body ?? "(no body)"}\n`;
59
+ md = md.replace(/^(# .*\n\n)+/, `# ${item.title}\n\n`); // collapse stray dupes
60
+ const comments = await getJson<any[]>(`https://api.github.com/repos/${owner}/${repo}/issues/${num}/comments?per_page=30`, signal, ghHeaders()).catch(() => []);
61
+ if (Array.isArray(comments) && comments.length > 0) {
62
+ md += `\n---\n\n## Comments\n\n` + comments
63
+ .map((c) => `**${c.user?.login ?? "?"}** (${(c.created_at ?? "").slice(0, 10)}):\n\n${c.body ?? ""}`)
64
+ .join("\n\n---\n\n");
65
+ }
66
+ return { text: md.slice(0, 100_000), source: isPr ? "github-pr-api" : "github-issue-api" };
67
+ }
68
+
69
+ // repo root: metadata + README
70
+ if (parts.length === 2) {
71
+ const meta = await getJson<any>(`https://api.github.com/repos/${owner}/${repo}`, signal, ghHeaders());
72
+ let md = `# ${meta.full_name}\n\n${meta.description ?? ""}\n\n`;
73
+ md += `★ ${meta.stargazers_count} · ${meta.language ?? "?"} · updated ${(meta.updated_at ?? "").slice(0, 10)}\n`;
74
+ if (meta.license?.spdx_id) md += `License: ${meta.license.spdx_id}\n`;
75
+ if (Array.isArray(meta.topics) && meta.topics.length) md += `Tags: ${meta.topics.slice(0, 8).join(", ")}\n`;
76
+ const branches = ["HEAD", meta.default_branch ?? "main"];
77
+ for (const b of branches) {
78
+ try {
79
+ const readme = await getText(`https://raw.githubusercontent.com/${owner}/${repo}/${b}/README.md`, signal);
80
+ md += `\n---\n\n${readme}`;
81
+ break;
82
+ } catch {
83
+ /* try next branch name */
84
+ }
85
+ }
86
+ return { text: md.slice(0, 120_000), source: "github-api" };
87
+ }
88
+
89
+ return null;
90
+ };
91
+
92
+ // ------------------------------------------------------- stackoverflow
93
+
94
+ function stripHtml(s: string): string {
95
+ return s
96
+ .replace(/<pre><code>([\s\S]*?)<\/code><\/pre>/g, (_, c) => `\n\`\`\`\n${c.replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&")}\n\`\`\`\n`)
97
+ .replace(/<code>([\s\S]*?)<\/code>/g, (_, c) => `\`${c.replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&")}\``)
98
+ .replace(/<(p|br|div|li|h[1-6])[^>]*>/gi, "\n")
99
+ .replace(/<[^>]+>/g, "")
100
+ .replace(/</g, "<")
101
+ .replace(/>/g, ">")
102
+ .replace(/"/g, '"')
103
+ .replace(/&#39;/g, "'")
104
+ .replace(/&/g, "&")
105
+ .replace(/\n{3,}/g, "\n\n")
106
+ .trim();
107
+ }
108
+
109
+ const stackOverflowAdapter: Adapter = async (url, signal) => {
110
+ const m = url.pathname.match(/\/questions\/(\d+)/);
111
+ if (!m) return null;
112
+ const id = m[1];
113
+ const q = await getJson<any>(
114
+ `https://api.stackexchange.com/2.3/questions/${id}/answers?order=desc&sort=votes&site=stackoverflow&filter=withbody&pagesize=5`,
115
+ signal,
116
+ );
117
+ const qData = await getJson<any>(`https://api.stackexchange.com/2.3/questions/${id}?site=stackoverflow&filter=!9Z(-wwYGT`, signal).catch(() => null);
118
+ const question = qData?.items?.[0];
119
+ let md = question ? `# ${question.title}\n\n${stripHtml(question.body ?? "")}\n` : "";
120
+ const answers = (q.items ?? []) as any[];
121
+ if (answers.length > 0) {
122
+ md += `\n---\n\n## Answers\n\n` + answers
123
+ .map((a) => `${a.is_accepted ? "**Accepted answer**\n\n" : ""}${stripHtml(a.body ?? "")}`)
124
+ .join("\n\n---\n\n");
125
+ } else {
126
+ md += "\n(no answers yet)\n";
127
+ }
128
+ return { text: md.slice(0, 100_000), source: "stackexchange-api" };
129
+ };
130
+
131
+ function stripTags(s: string): string {
132
+ return s.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
133
+ }
134
+
135
+ // ----------------------------------------------------------------- hackernews
136
+
137
+ const hnAdapter: Adapter = async (url, signal) => {
138
+ if (!/(^|\.)news\.ycombinator\.com$/.test(url.hostname)) return null;
139
+ const id = url.searchParams.get("id");
140
+ if (!id) return null;
141
+ const item = await getJson<any>(`https://hn.algolia.com/api/v1/items/${id}`, signal);
142
+ const flat = (n: any, depth: number): string => {
143
+ let s = "";
144
+ if (n.text) s += `${" ".repeat(depth)}- ${String(n.text).replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim()}\n`;
145
+ for (const c of n.children ?? []) s += flat(c, depth + 1);
146
+ return s;
147
+ };
148
+ const md = `# ${item.title ?? "HN thread " + (item.id ?? "")}\n\n${item.points ? `${item.points} points · u/${item.author ?? "?"}\n\n` : ""}${flat(item, 0)}`;
149
+ return { text: md.slice(0, 100_000), source: "hn-algolia" };
150
+ };
151
+
152
+ // ------------------------------------------------------------------ reddit
153
+
154
+ const redditAdapter: Adapter = async (url, signal) => {
155
+ if (!/(^|\.)reddit\.com$/.test(url.hostname) || !url.pathname.includes("/comments/")) return null;
156
+ const data = (await getJson<any>(url.protocol + "//" + url.host + url.pathname.replace(/\/$/, "") + ".json?limit=30", signal)) as any;
157
+ const post = Array.isArray(data) ? data[0]?.data?.children?.[0]?.data : null;
158
+ if (!post) return null;
159
+ let md = `# ${post.title}\n\nr/${post.subreddit} · ↑${post.ups} · u/${post.author}\n\n${post.selftext ?? ""}\n`;
160
+ const comments = Array.isArray(data) ? data[1]?.data?.children ?? [] : [];
161
+ const flat = (children: any[], depth = 0): string => {
162
+ let s = "";
163
+ for (const c of children) {
164
+ const d = c.data;
165
+ if (!d || d.kind === "more") continue;
166
+ s += `${" ".repeat(depth)}- **u/${d.author}** (↑${d.ups}): ${String(d.body ?? "").replace(/\n+/g, " ").slice(0, 500)}\n`;
167
+ if (d.replies?.data?.children) s += flat(d.replies.data.children, depth + 1);
168
+ }
169
+ return s;
170
+ };
171
+ md += `\n## Top comments\n\n` + flat(comments);
172
+ return { text: md.slice(0, 80_000), source: "reddit-json" };
173
+ };
174
+
175
+ // ------------------------------------------------------------- wikipedia
176
+
177
+ const wikipediaAdapter: Adapter = async (url, signal) => {
178
+ if (!/(^|\.)wikipedia\.org$/.test(url.hostname)) return null;
179
+ const m = url.pathname.match(/^\/wiki\/([^/:#]+)$/);
180
+ if (!m) return null;
181
+ const title = decodeURIComponent(m[1]);
182
+ const summary = await getJson<any>(`https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`, signal).catch(() => null);
183
+ const apiHost = url.hostname.startsWith("en.") ? url.hostname : "en.wikipedia.org";
184
+ const raw = await getText(`https://${apiHost}/api/rest_v1/page/html/${encodeURIComponent(title)}`, signal).catch(() => "");
185
+ if (!raw) return null;
186
+ // reuse the extractor's markdown conversion
187
+ const { htmlToMarkdown } = await import("./extract.ts");
188
+ const converted = htmlToMarkdown(raw, url.href, 200_000);
189
+ let md = converted.text;
190
+ if (summary?.extract && !md.includes(summary.extract.slice(0, 80))) {
191
+ md = `${summary.extract}\n\n${md}`;
192
+ }
193
+ return { text: md.slice(0, 200_000), source: "wikipedia-rest" };
194
+ };
195
+
196
+ // ------------------------------------------------------------------ router
197
+
198
+ const ADAPTERS: Array<[RegExp, Adapter]> = [
199
+ [/^github\.com$/, githubAdapter],
200
+ [/^(www\.)?stackoverflow\.com$/, stackOverflowAdapter],
201
+ [/^(www\.)?reddit\.com$/, redditAdapter],
202
+ [/^news\.ycombinator\.com$/, hnAdapter],
203
+ [/^(en\.)?wikipedia\.org$/, wikipediaAdapter],
204
+ ];
205
+
206
+ /**
207
+ * Try site adapters for a URL. Returns null when no adapter matches or all
208
+ * fail — caller falls back to the generic HTML pipeline.
209
+ */
210
+ export async function trySiteAdapter(url: string, signal?: AbortSignal): Promise<AdapterResult | null> {
211
+ let u: URL;
212
+ try {
213
+ u = new URL(url);
214
+ } catch {
215
+ return null;
216
+ }
217
+ if (u.protocol !== "http:" && u.protocol !== "https:") return null;
218
+ for (const [hostRe, adapter] of ADAPTERS) {
219
+ if (!hostRe.test(u.hostname)) continue;
220
+ try {
221
+ const r = await adapter(u, signal);
222
+ if (r && r.text.length > 80) return r;
223
+ } catch {
224
+ return null; // adapter exists but failed — generic path is more honest than an error
225
+ }
226
+ }
227
+ return null;
228
+ }
package/lib/apis.ts ADDED
@@ -0,0 +1,221 @@
1
+ /**
2
+ * Specialized search APIs — all free, no API keys required.
3
+ * - Stack Exchange (Stack Overflow): 300 req/day per IP, no key
4
+ * - Wikipedia (MediaWiki OpenSearch): unlimited reasonable use
5
+ * - npm registry: unlimited, no key
6
+ * - GitHub search: 10 req/min unauthenticated
7
+ * - Hacker News (Algolia): unlimited, no key
8
+ */
9
+ import { createDiskBackedCache } from "./cache.ts";
10
+
11
+ const UA =
12
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
13
+ const TIMEOUT_MS = 15_000;
14
+ const CACHE = createDiskBackedCache({ name: "apis", maxEntries: 256, ttlMs: 10 * 60 * 1000 });
15
+
16
+ export interface ApiResult {
17
+ title: string;
18
+ url: string;
19
+ snippet: string;
20
+ meta?: string;
21
+ }
22
+
23
+ export async function getJson<T>(url: string, signal?: AbortSignal, headers?: Record<string, string>): Promise<T> {
24
+ const timeout = AbortSignal.timeout(TIMEOUT_MS);
25
+ const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
26
+ const res = await fetch(url, {
27
+ headers: { "User-Agent": UA, Accept: "application/json", ...headers },
28
+ signal: combined,
29
+ });
30
+ if (!res.ok) throw new Error(`HTTP ${res.status} from ${new URL(url).host}`);
31
+ return (await res.json()) as T;
32
+ }
33
+
34
+ // ------------------------------------------------------------- stackoverflow
35
+
36
+ interface SeItem {
37
+ title: string;
38
+ link: string;
39
+ score: number;
40
+ answer_count: number;
41
+ is_answered: boolean;
42
+ is_accepted?: boolean;
43
+ tags?: string[];
44
+ excerpt?: string;
45
+ creation_date: number;
46
+ }
47
+
48
+ export async function searchStackOverflow(
49
+ query: string,
50
+ max: number,
51
+ signal?: AbortSignal,
52
+ ): Promise<ApiResult[]> {
53
+ const key = `so:${max}:${query}`;
54
+ const cached = CACHE.get(key);
55
+ if (cached) return cached as ApiResult[];
56
+ const u = new URL("https://api.stackexchange.com/2.3/search/advanced");
57
+ u.searchParams.set("order", "desc");
58
+ u.searchParams.set("sort", "relevance");
59
+ u.searchParams.set("q", query);
60
+ u.searchParams.set("site", "stackoverflow");
61
+ u.searchParams.set("pagesize", String(Math.min(Math.max(max, 1), 30)));
62
+ u.searchParams.set("filter", "!nNPvSNdWme"); // includes excerpt
63
+ const data = await getJson<{ items?: SeItem[]; quota_remaining?: number; error_message?: string }>(
64
+ u.toString(),
65
+ signal,
66
+ );
67
+ if (data.error_message) throw new Error(`stackexchange: ${data.error_message}`);
68
+ const results = (data.items ?? []).map((it) => ({
69
+ title: it.title.replace(/"/g, '"').replace(/&#39;/g, "'").replace(/&/g, "&"),
70
+ url: `https://stackoverflow.com/questions/${it.question_id}`,
71
+ snippet: (it.excerpt ?? "").trim(),
72
+ meta: `▲${it.score} · ${it.answer_count} answers${it.is_accepted ? " · ✓accepted" : ""} · [${(it.tags ?? []).slice(0, 4).join(", ")}]`,
73
+ }));
74
+ CACHE.set(key, results);
75
+ return results;
76
+ }
77
+
78
+ // ---------------------------------------------------------------- wikipedia
79
+
80
+ interface WikiSearchItem {
81
+ title: string;
82
+ snippet: string; // contains <span class="searchmatch"> html
83
+ pageid: number;
84
+ }
85
+
86
+ export async function searchWikipedia(
87
+ query: string,
88
+ max: number,
89
+ signal?: AbortSignal,
90
+ ): Promise<ApiResult[]> {
91
+ const key = `wiki:${max}:${query}`;
92
+ const cached = CACHE.get(key);
93
+ if (cached) return cached as ApiResult[];
94
+ const u = new URL("https://en.wikipedia.org/w/api.php");
95
+ u.searchParams.set("action", "query");
96
+ u.searchParams.set("list", "search");
97
+ u.searchParams.set("srsearch", query);
98
+ u.searchParams.set("srlimit", String(Math.min(Math.max(max, 1), 30)));
99
+ u.searchParams.set("format", "json");
100
+ const data = await getJson<{ query?: { search?: WikiSearchItem[] } }>(u.toString(), signal);
101
+ const results = (data.query?.search ?? []).map((it) => ({
102
+ title: it.title,
103
+ url: `https://en.wikipedia.org/wiki/${encodeURIComponent(it.title.replace(/ /g, "_"))}`,
104
+ snippet: it.snippet.replace(/<[^>]+>/g, "").trim(),
105
+ meta: "wikipedia",
106
+ }));
107
+ CACHE.set(key, results);
108
+ return results;
109
+ }
110
+
111
+ // ---------------------------------------------------------------------- npm
112
+
113
+ interface NpmObject {
114
+ package: {
115
+ name: string;
116
+ version: string;
117
+ description?: string;
118
+ links?: { npm?: string };
119
+ publisher?: { username?: string };
120
+ date?: string;
121
+ };
122
+ score?: { final: number; detail: { popularity: number; quality: number; maintenance: number } };
123
+ }
124
+
125
+ export async function searchNpm(query: string, max: number, signal?: AbortSignal): Promise<ApiResult[]> {
126
+ const key = `npm:${max}:${query}`;
127
+ const cached = CACHE.get(key);
128
+ if (cached) return cached as ApiResult[];
129
+ const u = new URL("https://registry.npmjs.org/-/v1/search");
130
+ u.searchParams.set("text", query);
131
+ u.searchParams.set("size", String(Math.min(Math.max(max, 1), 20)));
132
+ const data = await getJson<{ objects?: NpmObject[] }>(u.toString(), signal);
133
+ const results = (data.objects ?? []).map((o) => ({
134
+ title: `${o.package.name} v${o.package.version}`,
135
+ url: o.package.links?.npm ?? `https://www.npmjs.com/package/${o.package.name}`,
136
+ snippet: o.package.description ?? "",
137
+ meta: `⭐quality ${(100 * (o.score?.detail.quality ?? 0)).toFixed(0)} · pop ${(100 * (o.score?.detail.popularity ?? 0)).toFixed(0)}`,
138
+ }));
139
+ CACHE.set(key, results);
140
+ return results;
141
+ }
142
+
143
+ // -------------------------------------------------------------------- github
144
+
145
+ interface GhRepo {
146
+ full_name: string;
147
+ html_url: string;
148
+ description: string | null;
149
+ stargazers_count: number;
150
+ language: string | null;
151
+ updated_at: string;
152
+ }
153
+
154
+ interface GhSearchResponse {
155
+ items?: GhRepo[];
156
+ message?: string;
157
+ }
158
+
159
+ export async function searchGithubRepos(
160
+ query: string,
161
+ max: number,
162
+ signal?: AbortSignal,
163
+ token?: string,
164
+ ): Promise<ApiResult[]> {
165
+ const key = `gh:${max}:${query}`;
166
+ const cached = CACHE.get(key);
167
+ if (cached) return cached as ApiResult[];
168
+ const u = new URL("https://api.github.com/search/repositories");
169
+ u.searchParams.set("q", query);
170
+ u.searchParams.set("per_page", String(Math.min(Math.max(max, 1), 30)));
171
+ u.searchParams.set("sort", "best-match");
172
+ const data = await getJson<GhSearchResponse>(u.toString(), signal, token ? { Authorization: `Bearer ${token}` } : {});
173
+ if (data.message) throw new Error(`github: ${data.message}`);
174
+ const results = (data.items ?? []).map((r) => ({
175
+ title: r.full_name,
176
+ url: r.html_url,
177
+ snippet: r.description ?? "",
178
+ meta: `★${r.stargazers_count} · ${r.language ?? "?"} · updated ${new Date(r.updated_at).toISOString().slice(0, 10)}`,
179
+ }));
180
+ CACHE.set(key, results);
181
+ return results;
182
+ }
183
+
184
+ // ---------------------------------------------------------------- hackernews
185
+
186
+ interface HnHit {
187
+ title: string | null;
188
+ url: string | null;
189
+ objectID: string;
190
+ points: number | null;
191
+ num_comments: number | null;
192
+ story_text?: string | null;
193
+ comment_text?: string | null;
194
+ created_at: string;
195
+ }
196
+
197
+ export async function searchHackerNews(
198
+ query: string,
199
+ max: number,
200
+ signal?: AbortSignal,
201
+ ): Promise<ApiResult[]> {
202
+ const key = `hn:${max}:${query}`;
203
+ const cached = CACHE.get(key);
204
+ if (cached) return cached as ApiResult[];
205
+ const u = new URL("https://hn.algolia.com/api/v1/search");
206
+ u.searchParams.set("query", query);
207
+ u.searchParams.set("hitsPerPage", String(Math.min(Math.max(max, 1), 30)));
208
+ const data = await getJson<{ hits?: HnHit[] }>(u.toString(), signal);
209
+ const results = (data.hits ?? [])
210
+ .filter((h) => h.title || h.story_title)
211
+ .map((h) => ({
212
+ title: h.title ?? h.story_title ?? "(untitled)",
213
+ url:
214
+ h.url ??
215
+ (h.story_text ? `https://news.ycombinator.com/item?id=${h.objectID}` : `https://news.ycombinator.com/item?id=${h.objectID}`),
216
+ snippet: (h.story_text ? h.story_text.replace(/<[^>]+>/g, "").slice(0, 200) : "").trim(),
217
+ meta: `▲${h.points ?? 0} · ${h.num_comments ?? 0} comments · ${h.created_at.slice(0, 10)}`,
218
+ }));
219
+ CACHE.set(key, results);
220
+ return results;
221
+ }
package/lib/cache.ts ADDED
@@ -0,0 +1,160 @@
1
+ /**
2
+ * Generic TTL + LRU cache used across engines, APIs and the fetcher.
3
+ *
4
+ * `createDiskBackedCache` adds a JSON file layer under `~/.pi/agent/cache/webfind`:
5
+ * memory stays the hot path; disk survives restarts (search pages, fetched
6
+ * article text). Writes are debounced and flushed on a timer + process exit.
7
+ */
8
+ import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
9
+ import { join } from "node:path";
10
+ import { homedir } from "node:os";
11
+
12
+ export function createTtlCache(maxEntries: number, ttlMs: number) {
13
+ const map = new Map<string, { at: number; value: unknown }>();
14
+ return {
15
+ get(key: string): unknown | null {
16
+ const hit = map.get(key);
17
+ if (!hit) return null;
18
+ if (Date.now() - hit.at > ttlMs) {
19
+ map.delete(key);
20
+ return null;
21
+ }
22
+ // LRU refresh
23
+ map.delete(key);
24
+ map.set(key, hit);
25
+ return hit.value;
26
+ },
27
+ set(key: string, value: unknown) {
28
+ if (map.size >= maxEntries) {
29
+ const oldest = map.keys().next().value;
30
+ if (oldest) map.delete(oldest);
31
+ }
32
+ map.set(key, { at: Date.now(), value });
33
+ },
34
+ };
35
+ }
36
+
37
+ const DISK_LIMIT_BYTES = 8 * 1024 * 1024; // keep each snapshot small
38
+
39
+ export interface DiskBackedCache {
40
+ get(key: string): unknown | null;
41
+ set(key: string, value: unknown): void;
42
+ /** Force a synchronous write (used on process exit). */
43
+ flushSync(): void;
44
+ }
45
+
46
+ /**
47
+ * Memory-first cache with a JSON disk snapshot. On first use, loads the disk
48
+ * file (pruning expired entries). `set` updates memory and schedules a
49
+ * debounced flush; process exit flushes synchronously.
50
+ */
51
+ export function createDiskBackedCache(opts: {
52
+ name: string; // file name without extension
53
+ maxEntries: number;
54
+ ttlMs: number;
55
+ flushMs?: number;
56
+ }): DiskBackedCache {
57
+ const map = new Map<string, { at: number; value: unknown }>();
58
+ const dir = join(homedir(), ".pi", "agent", "cache", "webfind");
59
+ const file = join(dir, `${opts.name}.json`);
60
+ let loaded = false;
61
+ let dirty = false;
62
+ let maxAt = 0;
63
+
64
+ const load = () => {
65
+ if (loaded) return;
66
+ loaded = true;
67
+ try {
68
+ if (!existsSync(file)) return;
69
+ const entries = JSON.parse(readFileSync(file, "utf8")) as Array<[string, { at: number; value: unknown }]>;
70
+ const now = Date.now();
71
+ for (const [k, v] of entries) {
72
+ if (typeof v?.at === "number" && typeof k === "string" && now - v.at <= opts.ttlMs) {
73
+ map.set(k, v);
74
+ if (v.at > maxAt) maxAt = v.at;
75
+ }
76
+ }
77
+ } catch {
78
+ // corrupt snapshot — ignore, start fresh
79
+ }
80
+ };
81
+
82
+ const evict = () => {
83
+ while (map.size >= opts.maxEntries) {
84
+ // drop the oldest entry
85
+ let oldestKey: string | null = null;
86
+ let oldestAt = Infinity;
87
+ for (const [k, v] of map) {
88
+ if (v.at < oldestAt) {
89
+ oldestAt = v.at;
90
+ oldestKey = k;
91
+ }
92
+ }
93
+ if (!oldestKey) break;
94
+ map.delete(oldestKey);
95
+ }
96
+ };
97
+
98
+ const flush = () => {
99
+ if (!dirty) return;
100
+ dirty = false;
101
+ try {
102
+ mkdirSync(dir, { recursive: true });
103
+ const entries = [...map.entries()];
104
+ const body = JSON.stringify(entries);
105
+ if (body.length > DISK_LIMIT_BYTES) {
106
+ // over budget: keep the newest half
107
+ entries.sort((a, b) => b[1].at - a[1].at);
108
+ writeFileSync(file, JSON.stringify(entries.slice(0, Math.ceil(entries.length / 2))));
109
+ } else {
110
+ writeFileSync(file, body);
111
+ }
112
+ } catch {
113
+ // disk full/readonly — cache silently degrades to memory-only
114
+ }
115
+ };
116
+
117
+ // debounced background flush
118
+ let timer: ReturnType<typeof setTimeout> | null = null;
119
+ const schedule = () => {
120
+ if (timer) return;
121
+ timer = setTimeout(() => {
122
+ timer = null;
123
+ flush();
124
+ }, opts.flushMs ?? 3_000);
125
+ if (typeof timer === "object" && "unref" in (timer as any)) (timer as any).unref?.();
126
+ };
127
+
128
+ // flush on exit (best effort)
129
+ process.on("exit", () => flush());
130
+ try {
131
+ process.on("SIGINT", () => {
132
+ flush();
133
+ });
134
+ } catch {
135
+ /* not always available */
136
+ }
137
+
138
+ return {
139
+ get(key: string) {
140
+ load();
141
+ const hit = map.get(key);
142
+ if (!hit) return null;
143
+ if (Date.now() - hit.at > opts.ttlMs) {
144
+ map.delete(key);
145
+ return null;
146
+ }
147
+ return hit.value;
148
+ },
149
+ set(key: string, value: unknown) {
150
+ load();
151
+ evict();
152
+ const at = Math.max(Date.now(), maxAt + 1); // strictly increasing → stable eviction order
153
+ maxAt = at;
154
+ map.set(key, { at, value });
155
+ dirty = true;
156
+ schedule();
157
+ },
158
+ flushSync: flush,
159
+ };
160
+ }