pi-webfind 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +126 -0
- package/extensions/web-search.ts +662 -0
- package/lib/adapters.ts +228 -0
- package/lib/apis.ts +221 -0
- package/lib/cache.ts +160 -0
- package/lib/engine.ts +418 -0
- package/lib/extract.ts +320 -0
- package/lib/fetcher.ts +475 -0
- package/lib/pdf.ts +181 -0
- package/lib/rank.ts +169 -0
- package/package.json +56 -0
package/lib/adapters.ts
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Site adapters — route known URL shapes to a cheaper, cleaner source
|
|
3
|
+
* instead of scraping HTML. Each adapter returns markdown or null
|
|
4
|
+
* (null = no adapter / adapter failed → caller falls back to generic fetch).
|
|
5
|
+
*
|
|
6
|
+
* All free, no keys (GitHub optionally honours GITHUB_TOKEN for rate limits).
|
|
7
|
+
*/
|
|
8
|
+
import { getJson } from "./apis.ts";
|
|
9
|
+
|
|
10
|
+
const UA = "pi-webfind/0.5 (free web research toolkit for pi coding agent; +https://github.com/jawwadzafar/pi-webfind)";
|
|
11
|
+
|
|
12
|
+
async function getText(url: string, signal?: AbortSignal, headers?: Record<string, string>): Promise<string> {
|
|
13
|
+
const timeout = AbortSignal.timeout(20_000);
|
|
14
|
+
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
15
|
+
const res = await fetch(url, {
|
|
16
|
+
headers: { "User-Agent": UA, Accept: "text/plain, text/markdown, */*", ...headers },
|
|
17
|
+
signal: combined,
|
|
18
|
+
redirect: "follow",
|
|
19
|
+
});
|
|
20
|
+
if (!res.ok) throw new Error(`HTTP ${res.status} from ${new URL(url).host}`);
|
|
21
|
+
return res.text();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface AdapterResult {
|
|
25
|
+
text: string;
|
|
26
|
+
source: string; // e.g. "github-api"
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
type Adapter = (url: URL, signal?: AbortSignal) => Promise<AdapterResult | null>;
|
|
30
|
+
|
|
31
|
+
// ------------------------------------------------------------------- github
|
|
32
|
+
|
|
33
|
+
function ghHeaders(): Record<string, string> {
|
|
34
|
+
const token = process.env.GITHUB_TOKEN;
|
|
35
|
+
return token ? { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" } : { Accept: "application/vnd.github+json" };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const githubAdapter: Adapter = async (url, signal) => {
|
|
39
|
+
const parts = url.pathname.split("/").filter(Boolean); // [o, r, (blob|tree|issues|pull), ...]
|
|
40
|
+
if (parts.length < 2) return null;
|
|
41
|
+
const [owner, repo, kind, ...rest] = parts;
|
|
42
|
+
|
|
43
|
+
// raw file: /blob/{ref}/{path...} or /raw/{ref}/{path...}
|
|
44
|
+
if ((kind === "blob" || kind === "raw") && rest.length >= 2) {
|
|
45
|
+
const [, ref, ...path] = [kind, rest[0], ...rest.slice(1)];
|
|
46
|
+
const raw = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${path.join("/")}`;
|
|
47
|
+
const text = await getText(raw, signal);
|
|
48
|
+
return { text: text.slice(0, 200_000), source: "github-raw" };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// issue / pull: API → markdown
|
|
52
|
+
if (kind === "issues" || kind === "pull") {
|
|
53
|
+
const num = Number(rest[0]);
|
|
54
|
+
if (!num) return null;
|
|
55
|
+
const isPr = kind === "pull";
|
|
56
|
+
const base = `https://api.github.com/repos/${owner}/${repo}/${isPr ? "pulls" : "issues"}/${num}`;
|
|
57
|
+
const item = await getJson<any>(base, signal, ghHeaders());
|
|
58
|
+
let md = `# ${item.title ?? `${owner}/${repo}#${num}`}\n\n${isPr && item.message ? "" : ""}${item.body ?? "(no body)"}\n`;
|
|
59
|
+
md = md.replace(/^(# .*\n\n)+/, `# ${item.title}\n\n`); // collapse stray dupes
|
|
60
|
+
const comments = await getJson<any[]>(`https://api.github.com/repos/${owner}/${repo}/issues/${num}/comments?per_page=30`, signal, ghHeaders()).catch(() => []);
|
|
61
|
+
if (Array.isArray(comments) && comments.length > 0) {
|
|
62
|
+
md += `\n---\n\n## Comments\n\n` + comments
|
|
63
|
+
.map((c) => `**${c.user?.login ?? "?"}** (${(c.created_at ?? "").slice(0, 10)}):\n\n${c.body ?? ""}`)
|
|
64
|
+
.join("\n\n---\n\n");
|
|
65
|
+
}
|
|
66
|
+
return { text: md.slice(0, 100_000), source: isPr ? "github-pr-api" : "github-issue-api" };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// repo root: metadata + README
|
|
70
|
+
if (parts.length === 2) {
|
|
71
|
+
const meta = await getJson<any>(`https://api.github.com/repos/${owner}/${repo}`, signal, ghHeaders());
|
|
72
|
+
let md = `# ${meta.full_name}\n\n${meta.description ?? ""}\n\n`;
|
|
73
|
+
md += `★ ${meta.stargazers_count} · ${meta.language ?? "?"} · updated ${(meta.updated_at ?? "").slice(0, 10)}\n`;
|
|
74
|
+
if (meta.license?.spdx_id) md += `License: ${meta.license.spdx_id}\n`;
|
|
75
|
+
if (Array.isArray(meta.topics) && meta.topics.length) md += `Tags: ${meta.topics.slice(0, 8).join(", ")}\n`;
|
|
76
|
+
const branches = ["HEAD", meta.default_branch ?? "main"];
|
|
77
|
+
for (const b of branches) {
|
|
78
|
+
try {
|
|
79
|
+
const readme = await getText(`https://raw.githubusercontent.com/${owner}/${repo}/${b}/README.md`, signal);
|
|
80
|
+
md += `\n---\n\n${readme}`;
|
|
81
|
+
break;
|
|
82
|
+
} catch {
|
|
83
|
+
/* try next branch name */
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return { text: md.slice(0, 120_000), source: "github-api" };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return null;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
// ------------------------------------------------------- stackoverflow
|
|
93
|
+
|
|
94
|
+
function stripHtml(s: string): string {
|
|
95
|
+
return s
|
|
96
|
+
.replace(/<pre><code>([\s\S]*?)<\/code><\/pre>/g, (_, c) => `\n\`\`\`\n${c.replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&")}\n\`\`\`\n`)
|
|
97
|
+
.replace(/<code>([\s\S]*?)<\/code>/g, (_, c) => `\`${c.replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&")}\``)
|
|
98
|
+
.replace(/<(p|br|div|li|h[1-6])[^>]*>/gi, "\n")
|
|
99
|
+
.replace(/<[^>]+>/g, "")
|
|
100
|
+
.replace(/</g, "<")
|
|
101
|
+
.replace(/>/g, ">")
|
|
102
|
+
.replace(/"/g, '"')
|
|
103
|
+
.replace(/'/g, "'")
|
|
104
|
+
.replace(/&/g, "&")
|
|
105
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
106
|
+
.trim();
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const stackOverflowAdapter: Adapter = async (url, signal) => {
|
|
110
|
+
const m = url.pathname.match(/\/questions\/(\d+)/);
|
|
111
|
+
if (!m) return null;
|
|
112
|
+
const id = m[1];
|
|
113
|
+
const q = await getJson<any>(
|
|
114
|
+
`https://api.stackexchange.com/2.3/questions/${id}/answers?order=desc&sort=votes&site=stackoverflow&filter=withbody&pagesize=5`,
|
|
115
|
+
signal,
|
|
116
|
+
);
|
|
117
|
+
const qData = await getJson<any>(`https://api.stackexchange.com/2.3/questions/${id}?site=stackoverflow&filter=!9Z(-wwYGT`, signal).catch(() => null);
|
|
118
|
+
const question = qData?.items?.[0];
|
|
119
|
+
let md = question ? `# ${question.title}\n\n${stripHtml(question.body ?? "")}\n` : "";
|
|
120
|
+
const answers = (q.items ?? []) as any[];
|
|
121
|
+
if (answers.length > 0) {
|
|
122
|
+
md += `\n---\n\n## Answers\n\n` + answers
|
|
123
|
+
.map((a) => `${a.is_accepted ? "**Accepted answer**\n\n" : ""}${stripHtml(a.body ?? "")}`)
|
|
124
|
+
.join("\n\n---\n\n");
|
|
125
|
+
} else {
|
|
126
|
+
md += "\n(no answers yet)\n";
|
|
127
|
+
}
|
|
128
|
+
return { text: md.slice(0, 100_000), source: "stackexchange-api" };
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
function stripTags(s: string): string {
|
|
132
|
+
return s.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ----------------------------------------------------------------- hackernews
|
|
136
|
+
|
|
137
|
+
const hnAdapter: Adapter = async (url, signal) => {
|
|
138
|
+
if (!/(^|\.)news\.ycombinator\.com$/.test(url.hostname)) return null;
|
|
139
|
+
const id = url.searchParams.get("id");
|
|
140
|
+
if (!id) return null;
|
|
141
|
+
const item = await getJson<any>(`https://hn.algolia.com/api/v1/items/${id}`, signal);
|
|
142
|
+
const flat = (n: any, depth: number): string => {
|
|
143
|
+
let s = "";
|
|
144
|
+
if (n.text) s += `${" ".repeat(depth)}- ${String(n.text).replace(/<[^>]+>/g, "").replace(/\s+/g, " ").trim()}\n`;
|
|
145
|
+
for (const c of n.children ?? []) s += flat(c, depth + 1);
|
|
146
|
+
return s;
|
|
147
|
+
};
|
|
148
|
+
const md = `# ${item.title ?? "HN thread " + (item.id ?? "")}\n\n${item.points ? `${item.points} points · u/${item.author ?? "?"}\n\n` : ""}${flat(item, 0)}`;
|
|
149
|
+
return { text: md.slice(0, 100_000), source: "hn-algolia" };
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
// ------------------------------------------------------------------ reddit
|
|
153
|
+
|
|
154
|
+
const redditAdapter: Adapter = async (url, signal) => {
|
|
155
|
+
if (!/(^|\.)reddit\.com$/.test(url.hostname) || !url.pathname.includes("/comments/")) return null;
|
|
156
|
+
const data = (await getJson<any>(url.protocol + "//" + url.host + url.pathname.replace(/\/$/, "") + ".json?limit=30", signal)) as any;
|
|
157
|
+
const post = Array.isArray(data) ? data[0]?.data?.children?.[0]?.data : null;
|
|
158
|
+
if (!post) return null;
|
|
159
|
+
let md = `# ${post.title}\n\nr/${post.subreddit} · ↑${post.ups} · u/${post.author}\n\n${post.selftext ?? ""}\n`;
|
|
160
|
+
const comments = Array.isArray(data) ? data[1]?.data?.children ?? [] : [];
|
|
161
|
+
const flat = (children: any[], depth = 0): string => {
|
|
162
|
+
let s = "";
|
|
163
|
+
for (const c of children) {
|
|
164
|
+
const d = c.data;
|
|
165
|
+
if (!d || d.kind === "more") continue;
|
|
166
|
+
s += `${" ".repeat(depth)}- **u/${d.author}** (↑${d.ups}): ${String(d.body ?? "").replace(/\n+/g, " ").slice(0, 500)}\n`;
|
|
167
|
+
if (d.replies?.data?.children) s += flat(d.replies.data.children, depth + 1);
|
|
168
|
+
}
|
|
169
|
+
return s;
|
|
170
|
+
};
|
|
171
|
+
md += `\n## Top comments\n\n` + flat(comments);
|
|
172
|
+
return { text: md.slice(0, 80_000), source: "reddit-json" };
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
// ------------------------------------------------------------- wikipedia
|
|
176
|
+
|
|
177
|
+
const wikipediaAdapter: Adapter = async (url, signal) => {
|
|
178
|
+
if (!/(^|\.)wikipedia\.org$/.test(url.hostname)) return null;
|
|
179
|
+
const m = url.pathname.match(/^\/wiki\/([^/:#]+)$/);
|
|
180
|
+
if (!m) return null;
|
|
181
|
+
const title = decodeURIComponent(m[1]);
|
|
182
|
+
const summary = await getJson<any>(`https://en.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`, signal).catch(() => null);
|
|
183
|
+
const apiHost = url.hostname.startsWith("en.") ? url.hostname : "en.wikipedia.org";
|
|
184
|
+
const raw = await getText(`https://${apiHost}/api/rest_v1/page/html/${encodeURIComponent(title)}`, signal).catch(() => "");
|
|
185
|
+
if (!raw) return null;
|
|
186
|
+
// reuse the extractor's markdown conversion
|
|
187
|
+
const { htmlToMarkdown } = await import("./extract.ts");
|
|
188
|
+
const converted = htmlToMarkdown(raw, url.href, 200_000);
|
|
189
|
+
let md = converted.text;
|
|
190
|
+
if (summary?.extract && !md.includes(summary.extract.slice(0, 80))) {
|
|
191
|
+
md = `${summary.extract}\n\n${md}`;
|
|
192
|
+
}
|
|
193
|
+
return { text: md.slice(0, 200_000), source: "wikipedia-rest" };
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
// ------------------------------------------------------------------ router
|
|
197
|
+
|
|
198
|
+
const ADAPTERS: Array<[RegExp, Adapter]> = [
|
|
199
|
+
[/^github\.com$/, githubAdapter],
|
|
200
|
+
[/^(www\.)?stackoverflow\.com$/, stackOverflowAdapter],
|
|
201
|
+
[/^(www\.)?reddit\.com$/, redditAdapter],
|
|
202
|
+
[/^news\.ycombinator\.com$/, hnAdapter],
|
|
203
|
+
[/^(en\.)?wikipedia\.org$/, wikipediaAdapter],
|
|
204
|
+
];
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Try site adapters for a URL. Returns null when no adapter matches or all
|
|
208
|
+
* fail — caller falls back to the generic HTML pipeline.
|
|
209
|
+
*/
|
|
210
|
+
export async function trySiteAdapter(url: string, signal?: AbortSignal): Promise<AdapterResult | null> {
|
|
211
|
+
let u: URL;
|
|
212
|
+
try {
|
|
213
|
+
u = new URL(url);
|
|
214
|
+
} catch {
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
|
218
|
+
for (const [hostRe, adapter] of ADAPTERS) {
|
|
219
|
+
if (!hostRe.test(u.hostname)) continue;
|
|
220
|
+
try {
|
|
221
|
+
const r = await adapter(u, signal);
|
|
222
|
+
if (r && r.text.length > 80) return r;
|
|
223
|
+
} catch {
|
|
224
|
+
return null; // adapter exists but failed — generic path is more honest than an error
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
return null;
|
|
228
|
+
}
|
package/lib/apis.ts
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Specialized search APIs — all free, no API keys required.
|
|
3
|
+
* - Stack Exchange (Stack Overflow): 300 req/day per IP, no key
|
|
4
|
+
* - Wikipedia (MediaWiki OpenSearch): unlimited reasonable use
|
|
5
|
+
* - npm registry: unlimited, no key
|
|
6
|
+
* - GitHub search: 10 req/min unauthenticated
|
|
7
|
+
* - Hacker News (Algolia): unlimited, no key
|
|
8
|
+
*/
|
|
9
|
+
import { createDiskBackedCache } from "./cache.ts";
|
|
10
|
+
|
|
11
|
+
const UA =
|
|
12
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
|
13
|
+
const TIMEOUT_MS = 15_000;
|
|
14
|
+
const CACHE = createDiskBackedCache({ name: "apis", maxEntries: 256, ttlMs: 10 * 60 * 1000 });
|
|
15
|
+
|
|
16
|
+
export interface ApiResult {
|
|
17
|
+
title: string;
|
|
18
|
+
url: string;
|
|
19
|
+
snippet: string;
|
|
20
|
+
meta?: string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function getJson<T>(url: string, signal?: AbortSignal, headers?: Record<string, string>): Promise<T> {
|
|
24
|
+
const timeout = AbortSignal.timeout(TIMEOUT_MS);
|
|
25
|
+
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
26
|
+
const res = await fetch(url, {
|
|
27
|
+
headers: { "User-Agent": UA, Accept: "application/json", ...headers },
|
|
28
|
+
signal: combined,
|
|
29
|
+
});
|
|
30
|
+
if (!res.ok) throw new Error(`HTTP ${res.status} from ${new URL(url).host}`);
|
|
31
|
+
return (await res.json()) as T;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// ------------------------------------------------------------- stackoverflow
|
|
35
|
+
|
|
36
|
+
interface SeItem {
|
|
37
|
+
title: string;
|
|
38
|
+
link: string;
|
|
39
|
+
score: number;
|
|
40
|
+
answer_count: number;
|
|
41
|
+
is_answered: boolean;
|
|
42
|
+
is_accepted?: boolean;
|
|
43
|
+
tags?: string[];
|
|
44
|
+
excerpt?: string;
|
|
45
|
+
creation_date: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function searchStackOverflow(
|
|
49
|
+
query: string,
|
|
50
|
+
max: number,
|
|
51
|
+
signal?: AbortSignal,
|
|
52
|
+
): Promise<ApiResult[]> {
|
|
53
|
+
const key = `so:${max}:${query}`;
|
|
54
|
+
const cached = CACHE.get(key);
|
|
55
|
+
if (cached) return cached as ApiResult[];
|
|
56
|
+
const u = new URL("https://api.stackexchange.com/2.3/search/advanced");
|
|
57
|
+
u.searchParams.set("order", "desc");
|
|
58
|
+
u.searchParams.set("sort", "relevance");
|
|
59
|
+
u.searchParams.set("q", query);
|
|
60
|
+
u.searchParams.set("site", "stackoverflow");
|
|
61
|
+
u.searchParams.set("pagesize", String(Math.min(Math.max(max, 1), 30)));
|
|
62
|
+
u.searchParams.set("filter", "!nNPvSNdWme"); // includes excerpt
|
|
63
|
+
const data = await getJson<{ items?: SeItem[]; quota_remaining?: number; error_message?: string }>(
|
|
64
|
+
u.toString(),
|
|
65
|
+
signal,
|
|
66
|
+
);
|
|
67
|
+
if (data.error_message) throw new Error(`stackexchange: ${data.error_message}`);
|
|
68
|
+
const results = (data.items ?? []).map((it) => ({
|
|
69
|
+
title: it.title.replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&"),
|
|
70
|
+
url: `https://stackoverflow.com/questions/${it.question_id}`,
|
|
71
|
+
snippet: (it.excerpt ?? "").trim(),
|
|
72
|
+
meta: `▲${it.score} · ${it.answer_count} answers${it.is_accepted ? " · ✓accepted" : ""} · [${(it.tags ?? []).slice(0, 4).join(", ")}]`,
|
|
73
|
+
}));
|
|
74
|
+
CACHE.set(key, results);
|
|
75
|
+
return results;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ---------------------------------------------------------------- wikipedia
|
|
79
|
+
|
|
80
|
+
interface WikiSearchItem {
|
|
81
|
+
title: string;
|
|
82
|
+
snippet: string; // contains <span class="searchmatch"> html
|
|
83
|
+
pageid: number;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export async function searchWikipedia(
|
|
87
|
+
query: string,
|
|
88
|
+
max: number,
|
|
89
|
+
signal?: AbortSignal,
|
|
90
|
+
): Promise<ApiResult[]> {
|
|
91
|
+
const key = `wiki:${max}:${query}`;
|
|
92
|
+
const cached = CACHE.get(key);
|
|
93
|
+
if (cached) return cached as ApiResult[];
|
|
94
|
+
const u = new URL("https://en.wikipedia.org/w/api.php");
|
|
95
|
+
u.searchParams.set("action", "query");
|
|
96
|
+
u.searchParams.set("list", "search");
|
|
97
|
+
u.searchParams.set("srsearch", query);
|
|
98
|
+
u.searchParams.set("srlimit", String(Math.min(Math.max(max, 1), 30)));
|
|
99
|
+
u.searchParams.set("format", "json");
|
|
100
|
+
const data = await getJson<{ query?: { search?: WikiSearchItem[] } }>(u.toString(), signal);
|
|
101
|
+
const results = (data.query?.search ?? []).map((it) => ({
|
|
102
|
+
title: it.title,
|
|
103
|
+
url: `https://en.wikipedia.org/wiki/${encodeURIComponent(it.title.replace(/ /g, "_"))}`,
|
|
104
|
+
snippet: it.snippet.replace(/<[^>]+>/g, "").trim(),
|
|
105
|
+
meta: "wikipedia",
|
|
106
|
+
}));
|
|
107
|
+
CACHE.set(key, results);
|
|
108
|
+
return results;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// ---------------------------------------------------------------------- npm
|
|
112
|
+
|
|
113
|
+
interface NpmObject {
|
|
114
|
+
package: {
|
|
115
|
+
name: string;
|
|
116
|
+
version: string;
|
|
117
|
+
description?: string;
|
|
118
|
+
links?: { npm?: string };
|
|
119
|
+
publisher?: { username?: string };
|
|
120
|
+
date?: string;
|
|
121
|
+
};
|
|
122
|
+
score?: { final: number; detail: { popularity: number; quality: number; maintenance: number } };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export async function searchNpm(query: string, max: number, signal?: AbortSignal): Promise<ApiResult[]> {
|
|
126
|
+
const key = `npm:${max}:${query}`;
|
|
127
|
+
const cached = CACHE.get(key);
|
|
128
|
+
if (cached) return cached as ApiResult[];
|
|
129
|
+
const u = new URL("https://registry.npmjs.org/-/v1/search");
|
|
130
|
+
u.searchParams.set("text", query);
|
|
131
|
+
u.searchParams.set("size", String(Math.min(Math.max(max, 1), 20)));
|
|
132
|
+
const data = await getJson<{ objects?: NpmObject[] }>(u.toString(), signal);
|
|
133
|
+
const results = (data.objects ?? []).map((o) => ({
|
|
134
|
+
title: `${o.package.name} v${o.package.version}`,
|
|
135
|
+
url: o.package.links?.npm ?? `https://www.npmjs.com/package/${o.package.name}`,
|
|
136
|
+
snippet: o.package.description ?? "",
|
|
137
|
+
meta: `⭐quality ${(100 * (o.score?.detail.quality ?? 0)).toFixed(0)} · pop ${(100 * (o.score?.detail.popularity ?? 0)).toFixed(0)}`,
|
|
138
|
+
}));
|
|
139
|
+
CACHE.set(key, results);
|
|
140
|
+
return results;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// -------------------------------------------------------------------- github
|
|
144
|
+
|
|
145
|
+
interface GhRepo {
|
|
146
|
+
full_name: string;
|
|
147
|
+
html_url: string;
|
|
148
|
+
description: string | null;
|
|
149
|
+
stargazers_count: number;
|
|
150
|
+
language: string | null;
|
|
151
|
+
updated_at: string;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
interface GhSearchResponse {
|
|
155
|
+
items?: GhRepo[];
|
|
156
|
+
message?: string;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export async function searchGithubRepos(
|
|
160
|
+
query: string,
|
|
161
|
+
max: number,
|
|
162
|
+
signal?: AbortSignal,
|
|
163
|
+
token?: string,
|
|
164
|
+
): Promise<ApiResult[]> {
|
|
165
|
+
const key = `gh:${max}:${query}`;
|
|
166
|
+
const cached = CACHE.get(key);
|
|
167
|
+
if (cached) return cached as ApiResult[];
|
|
168
|
+
const u = new URL("https://api.github.com/search/repositories");
|
|
169
|
+
u.searchParams.set("q", query);
|
|
170
|
+
u.searchParams.set("per_page", String(Math.min(Math.max(max, 1), 30)));
|
|
171
|
+
u.searchParams.set("sort", "best-match");
|
|
172
|
+
const data = await getJson<GhSearchResponse>(u.toString(), signal, token ? { Authorization: `Bearer ${token}` } : {});
|
|
173
|
+
if (data.message) throw new Error(`github: ${data.message}`);
|
|
174
|
+
const results = (data.items ?? []).map((r) => ({
|
|
175
|
+
title: r.full_name,
|
|
176
|
+
url: r.html_url,
|
|
177
|
+
snippet: r.description ?? "",
|
|
178
|
+
meta: `★${r.stargazers_count} · ${r.language ?? "?"} · updated ${new Date(r.updated_at).toISOString().slice(0, 10)}`,
|
|
179
|
+
}));
|
|
180
|
+
CACHE.set(key, results);
|
|
181
|
+
return results;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ---------------------------------------------------------------- hackernews
|
|
185
|
+
|
|
186
|
+
interface HnHit {
|
|
187
|
+
title: string | null;
|
|
188
|
+
url: string | null;
|
|
189
|
+
objectID: string;
|
|
190
|
+
points: number | null;
|
|
191
|
+
num_comments: number | null;
|
|
192
|
+
story_text?: string | null;
|
|
193
|
+
comment_text?: string | null;
|
|
194
|
+
created_at: string;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export async function searchHackerNews(
|
|
198
|
+
query: string,
|
|
199
|
+
max: number,
|
|
200
|
+
signal?: AbortSignal,
|
|
201
|
+
): Promise<ApiResult[]> {
|
|
202
|
+
const key = `hn:${max}:${query}`;
|
|
203
|
+
const cached = CACHE.get(key);
|
|
204
|
+
if (cached) return cached as ApiResult[];
|
|
205
|
+
const u = new URL("https://hn.algolia.com/api/v1/search");
|
|
206
|
+
u.searchParams.set("query", query);
|
|
207
|
+
u.searchParams.set("hitsPerPage", String(Math.min(Math.max(max, 1), 30)));
|
|
208
|
+
const data = await getJson<{ hits?: HnHit[] }>(u.toString(), signal);
|
|
209
|
+
const results = (data.hits ?? [])
|
|
210
|
+
.filter((h) => h.title || h.story_title)
|
|
211
|
+
.map((h) => ({
|
|
212
|
+
title: h.title ?? h.story_title ?? "(untitled)",
|
|
213
|
+
url:
|
|
214
|
+
h.url ??
|
|
215
|
+
(h.story_text ? `https://news.ycombinator.com/item?id=${h.objectID}` : `https://news.ycombinator.com/item?id=${h.objectID}`),
|
|
216
|
+
snippet: (h.story_text ? h.story_text.replace(/<[^>]+>/g, "").slice(0, 200) : "").trim(),
|
|
217
|
+
meta: `▲${h.points ?? 0} · ${h.num_comments ?? 0} comments · ${h.created_at.slice(0, 10)}`,
|
|
218
|
+
}));
|
|
219
|
+
CACHE.set(key, results);
|
|
220
|
+
return results;
|
|
221
|
+
}
|
package/lib/cache.ts
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic TTL + LRU cache used across engines, APIs and the fetcher.
|
|
3
|
+
*
|
|
4
|
+
* `createDiskBackedCache` adds a JSON file layer under `~/.pi/agent/cache/webfind`:
|
|
5
|
+
* memory stays the hot path; disk survives restarts (search pages, fetched
|
|
6
|
+
* article text). Writes are debounced and flushed on a timer + process exit.
|
|
7
|
+
*/
|
|
8
|
+
import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { homedir } from "node:os";
|
|
11
|
+
|
|
12
|
+
export function createTtlCache(maxEntries: number, ttlMs: number) {
|
|
13
|
+
const map = new Map<string, { at: number; value: unknown }>();
|
|
14
|
+
return {
|
|
15
|
+
get(key: string): unknown | null {
|
|
16
|
+
const hit = map.get(key);
|
|
17
|
+
if (!hit) return null;
|
|
18
|
+
if (Date.now() - hit.at > ttlMs) {
|
|
19
|
+
map.delete(key);
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
// LRU refresh
|
|
23
|
+
map.delete(key);
|
|
24
|
+
map.set(key, hit);
|
|
25
|
+
return hit.value;
|
|
26
|
+
},
|
|
27
|
+
set(key: string, value: unknown) {
|
|
28
|
+
if (map.size >= maxEntries) {
|
|
29
|
+
const oldest = map.keys().next().value;
|
|
30
|
+
if (oldest) map.delete(oldest);
|
|
31
|
+
}
|
|
32
|
+
map.set(key, { at: Date.now(), value });
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const DISK_LIMIT_BYTES = 8 * 1024 * 1024; // keep each snapshot small
|
|
38
|
+
|
|
39
|
+
export interface DiskBackedCache {
|
|
40
|
+
get(key: string): unknown | null;
|
|
41
|
+
set(key: string, value: unknown): void;
|
|
42
|
+
/** Force a synchronous write (used on process exit). */
|
|
43
|
+
flushSync(): void;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Memory-first cache with a JSON disk snapshot. On first use, loads the disk
|
|
48
|
+
* file (pruning expired entries). `set` updates memory and schedules a
|
|
49
|
+
* debounced flush; process exit flushes synchronously.
|
|
50
|
+
*/
|
|
51
|
+
export function createDiskBackedCache(opts: {
|
|
52
|
+
name: string; // file name without extension
|
|
53
|
+
maxEntries: number;
|
|
54
|
+
ttlMs: number;
|
|
55
|
+
flushMs?: number;
|
|
56
|
+
}): DiskBackedCache {
|
|
57
|
+
const map = new Map<string, { at: number; value: unknown }>();
|
|
58
|
+
const dir = join(homedir(), ".pi", "agent", "cache", "webfind");
|
|
59
|
+
const file = join(dir, `${opts.name}.json`);
|
|
60
|
+
let loaded = false;
|
|
61
|
+
let dirty = false;
|
|
62
|
+
let maxAt = 0;
|
|
63
|
+
|
|
64
|
+
const load = () => {
|
|
65
|
+
if (loaded) return;
|
|
66
|
+
loaded = true;
|
|
67
|
+
try {
|
|
68
|
+
if (!existsSync(file)) return;
|
|
69
|
+
const entries = JSON.parse(readFileSync(file, "utf8")) as Array<[string, { at: number; value: unknown }]>;
|
|
70
|
+
const now = Date.now();
|
|
71
|
+
for (const [k, v] of entries) {
|
|
72
|
+
if (typeof v?.at === "number" && typeof k === "string" && now - v.at <= opts.ttlMs) {
|
|
73
|
+
map.set(k, v);
|
|
74
|
+
if (v.at > maxAt) maxAt = v.at;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
} catch {
|
|
78
|
+
// corrupt snapshot — ignore, start fresh
|
|
79
|
+
}
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const evict = () => {
|
|
83
|
+
while (map.size >= opts.maxEntries) {
|
|
84
|
+
// drop the oldest entry
|
|
85
|
+
let oldestKey: string | null = null;
|
|
86
|
+
let oldestAt = Infinity;
|
|
87
|
+
for (const [k, v] of map) {
|
|
88
|
+
if (v.at < oldestAt) {
|
|
89
|
+
oldestAt = v.at;
|
|
90
|
+
oldestKey = k;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
if (!oldestKey) break;
|
|
94
|
+
map.delete(oldestKey);
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const flush = () => {
|
|
99
|
+
if (!dirty) return;
|
|
100
|
+
dirty = false;
|
|
101
|
+
try {
|
|
102
|
+
mkdirSync(dir, { recursive: true });
|
|
103
|
+
const entries = [...map.entries()];
|
|
104
|
+
const body = JSON.stringify(entries);
|
|
105
|
+
if (body.length > DISK_LIMIT_BYTES) {
|
|
106
|
+
// over budget: keep the newest half
|
|
107
|
+
entries.sort((a, b) => b[1].at - a[1].at);
|
|
108
|
+
writeFileSync(file, JSON.stringify(entries.slice(0, Math.ceil(entries.length / 2))));
|
|
109
|
+
} else {
|
|
110
|
+
writeFileSync(file, body);
|
|
111
|
+
}
|
|
112
|
+
} catch {
|
|
113
|
+
// disk full/readonly — cache silently degrades to memory-only
|
|
114
|
+
}
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
// debounced background flush
|
|
118
|
+
let timer: ReturnType<typeof setTimeout> | null = null;
|
|
119
|
+
const schedule = () => {
|
|
120
|
+
if (timer) return;
|
|
121
|
+
timer = setTimeout(() => {
|
|
122
|
+
timer = null;
|
|
123
|
+
flush();
|
|
124
|
+
}, opts.flushMs ?? 3_000);
|
|
125
|
+
if (typeof timer === "object" && "unref" in (timer as any)) (timer as any).unref?.();
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
// flush on exit (best effort)
|
|
129
|
+
process.on("exit", () => flush());
|
|
130
|
+
try {
|
|
131
|
+
process.on("SIGINT", () => {
|
|
132
|
+
flush();
|
|
133
|
+
});
|
|
134
|
+
} catch {
|
|
135
|
+
/* not always available */
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
get(key: string) {
|
|
140
|
+
load();
|
|
141
|
+
const hit = map.get(key);
|
|
142
|
+
if (!hit) return null;
|
|
143
|
+
if (Date.now() - hit.at > opts.ttlMs) {
|
|
144
|
+
map.delete(key);
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
return hit.value;
|
|
148
|
+
},
|
|
149
|
+
set(key: string, value: unknown) {
|
|
150
|
+
load();
|
|
151
|
+
evict();
|
|
152
|
+
const at = Math.max(Date.now(), maxAt + 1); // strictly increasing → stable eviction order
|
|
153
|
+
maxAt = at;
|
|
154
|
+
map.set(key, { at, value });
|
|
155
|
+
dirty = true;
|
|
156
|
+
schedule();
|
|
157
|
+
},
|
|
158
|
+
flushSync: flush,
|
|
159
|
+
};
|
|
160
|
+
}
|