pi-wsearch 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -11
- package/package.json +1 -1
- package/src/cache.ts +66 -34
- package/src/config.ts +14 -1
- package/src/dom.ts +13 -12
- package/src/engines.ts +298 -308
- package/src/fetch.ts +1 -5
- package/src/fuse.ts +27 -15
- package/src/keys.ts +2 -22
- package/src/login.ts +1 -1
- package/src/scrape.ts +58 -481
- package/src/text.ts +418 -0
- package/src/tools.ts +55 -66
- package/src/transport.ts +144 -113
- package/src/view.ts +2 -7
- package/src/constants.ts +0 -6
package/src/engines.ts
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import { getConfig, sanitizeError, type ProviderKey
|
|
2
|
-
import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './
|
|
1
|
+
import { getConfig, sanitizeError, type ProviderKey } from './config';
|
|
2
|
+
import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './config';
|
|
3
3
|
|
|
4
4
|
export interface SearchHit {
|
|
5
5
|
title: string;
|
|
6
6
|
url: string;
|
|
7
7
|
snippet: string;
|
|
8
|
-
engine: string;
|
|
9
8
|
date?: string;
|
|
10
9
|
}
|
|
11
10
|
|
|
@@ -13,11 +12,8 @@ export interface EngineOutcome {
|
|
|
13
12
|
engine: string;
|
|
14
13
|
hits: SearchHit[];
|
|
15
14
|
error?: string;
|
|
16
|
-
latencyMs: number;
|
|
17
15
|
}
|
|
18
16
|
|
|
19
|
-
type EngineName = 'serper' | 'tavily' | 'exa' | 'brave' | 'jina' | 'kagi' | 'you' | 'firecrawl' | 'tinyfish';
|
|
20
|
-
|
|
21
17
|
export interface SearchOptions {
|
|
22
18
|
query: string;
|
|
23
19
|
maxResults?: number;
|
|
@@ -25,12 +21,284 @@ export interface SearchOptions {
|
|
|
25
21
|
signal?: AbortSignal;
|
|
26
22
|
}
|
|
27
23
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
24
|
+
type EngineSearch = (key: string, opts: SearchOptions, signal?: AbortSignal) => Promise<SearchHit[]>;
|
|
25
|
+
|
|
26
|
+
const ENGINES: Record<string, { key: ProviderKey; search: EngineSearch }> = {
|
|
27
|
+
serper: {
|
|
28
|
+
key: 'serperKey',
|
|
29
|
+
search: async (key, opts, signal) => {
|
|
30
|
+
const d = await postJson<{ organic?: { title?: string; link?: string; snippet?: string; date?: string }[] }>(
|
|
31
|
+
'https://google.serper.dev/search',
|
|
32
|
+
{ q: opts.query, num: maxResults(opts), gl: 'us' },
|
|
33
|
+
jsonHeaders({ 'X-API-KEY': key }),
|
|
34
|
+
signal,
|
|
35
|
+
);
|
|
36
|
+
return (d.organic ?? [])
|
|
37
|
+
.filter((r) => r?.link)
|
|
38
|
+
.map((r) => ({
|
|
39
|
+
title: r.title ?? r.link!,
|
|
40
|
+
url: r.link!,
|
|
41
|
+
snippet: r.snippet ?? '',
|
|
42
|
+
date: r.date,
|
|
43
|
+
}));
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
tavily: {
|
|
47
|
+
key: 'tavilyKey',
|
|
48
|
+
search: async (key, opts, signal) => {
|
|
49
|
+
const { query, domain } = extractSiteOperator(opts.query);
|
|
50
|
+
const body: Record<string, unknown> = {
|
|
51
|
+
query,
|
|
52
|
+
max_results: maxResults(opts),
|
|
53
|
+
search_depth: 'basic',
|
|
54
|
+
topic: 'general',
|
|
55
|
+
};
|
|
56
|
+
if (domain) body.include_domains = [domain];
|
|
57
|
+
const d = await postJson<{
|
|
58
|
+
results?: { title?: string; url?: string; content?: string; published_date?: string }[];
|
|
59
|
+
}>('https://api.tavily.com/search', body, jsonHeaders({ Authorization: `Bearer ${key}` }), signal);
|
|
60
|
+
return (d.results ?? [])
|
|
61
|
+
.filter((r) => r?.url)
|
|
62
|
+
.map((r) => ({
|
|
63
|
+
title: r.title ?? r.url!,
|
|
64
|
+
url: r.url!,
|
|
65
|
+
snippet: r.content ?? '',
|
|
66
|
+
date: r.published_date,
|
|
67
|
+
}));
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
exa: {
|
|
71
|
+
key: 'exaKey',
|
|
72
|
+
search: async (key, opts, signal) => {
|
|
73
|
+
const { query, domain } = extractSiteOperator(opts.query);
|
|
74
|
+
const body: Record<string, unknown> = {
|
|
75
|
+
query,
|
|
76
|
+
numResults: maxResults(opts),
|
|
77
|
+
contents: { highlights: true },
|
|
78
|
+
};
|
|
79
|
+
if (domain) body.includeDomains = [domain];
|
|
80
|
+
const d = await postJson<{
|
|
81
|
+
results?: { title?: string; url?: string; highlights?: string[]; publishedDate?: string }[];
|
|
82
|
+
}>('https://api.exa.ai/search', body, jsonHeaders({ 'x-api-key': key }), signal);
|
|
83
|
+
return (d.results ?? [])
|
|
84
|
+
.filter((r) => r?.url)
|
|
85
|
+
.map((r) => ({
|
|
86
|
+
title: r.title ?? r.url!,
|
|
87
|
+
url: r.url!,
|
|
88
|
+
snippet: (r.highlights ?? []).join(' '),
|
|
89
|
+
date: r.publishedDate,
|
|
90
|
+
}));
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
brave: {
|
|
94
|
+
key: 'braveKey',
|
|
95
|
+
search: async (key, opts, signal) => {
|
|
96
|
+
const url = new URL('https://api.search.brave.com/res/v1/web/search');
|
|
97
|
+
url.searchParams.set('q', opts.query);
|
|
98
|
+
url.searchParams.set('count', String(maxResults(opts)));
|
|
99
|
+
url.searchParams.set('extra_snippets', 'true');
|
|
100
|
+
url.searchParams.set('text_decorations', 'false');
|
|
101
|
+
url.searchParams.set('safesearch', 'moderate');
|
|
102
|
+
const d = await getJson<{
|
|
103
|
+
web?: {
|
|
104
|
+
results?: { title?: string; url?: string; description?: string; extra_snippets?: string[]; age?: string }[];
|
|
105
|
+
};
|
|
106
|
+
}>(url.toString(), { Accept: 'application/json', 'X-Subscription-Token': key }, signal);
|
|
107
|
+
return (d.web?.results ?? [])
|
|
108
|
+
.filter((r) => r?.url)
|
|
109
|
+
.map((r) => ({
|
|
110
|
+
title: r.title ?? r.url!,
|
|
111
|
+
url: r.url!,
|
|
112
|
+
snippet: [r.description, ...(r.extra_snippets ?? [])].filter(Boolean).join('\n'),
|
|
113
|
+
date: r.age,
|
|
114
|
+
}));
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
jina: {
|
|
118
|
+
key: 'jinaKey',
|
|
119
|
+
search: async (key, opts, signal) => {
|
|
120
|
+
const url = new URL('https://s.jina.ai/');
|
|
121
|
+
url.searchParams.set('q', opts.query);
|
|
122
|
+
url.searchParams.set('count', String(maxResults(opts)));
|
|
123
|
+
const d = await getJson<{
|
|
124
|
+
data?: { title?: string | null; url?: string | null; description?: string | null; content?: string | null }[];
|
|
125
|
+
}>(
|
|
126
|
+
url.toString(),
|
|
127
|
+
{
|
|
128
|
+
Accept: 'application/json',
|
|
129
|
+
'X-Respond-With': 'no-content',
|
|
130
|
+
'X-Retain-Images': 'none',
|
|
131
|
+
Authorization: `Bearer ${key}`,
|
|
132
|
+
},
|
|
133
|
+
signal,
|
|
134
|
+
);
|
|
135
|
+
return (d.data ?? [])
|
|
136
|
+
.filter((r) => r?.url)
|
|
137
|
+
.map((r) => ({
|
|
138
|
+
title: r.title ?? r.url!,
|
|
139
|
+
url: r.url!,
|
|
140
|
+
snippet: r.description?.trim() || r.content?.trim() || '',
|
|
141
|
+
}));
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
kagi: {
|
|
145
|
+
key: 'kagiKey',
|
|
146
|
+
search: async (key, opts, signal) => {
|
|
147
|
+
const url = new URL('https://kagi.com/api/v1/search');
|
|
148
|
+
url.searchParams.set('q', opts.query);
|
|
149
|
+
url.searchParams.set('limit', String(maxResults(opts)));
|
|
150
|
+
const d = await getJson<{
|
|
151
|
+
data?: {
|
|
152
|
+
search?: {
|
|
153
|
+
url?: string;
|
|
154
|
+
link?: string;
|
|
155
|
+
title?: string;
|
|
156
|
+
t?: string;
|
|
157
|
+
snippet?: string;
|
|
158
|
+
s?: string;
|
|
159
|
+
published?: string;
|
|
160
|
+
}[];
|
|
161
|
+
};
|
|
162
|
+
}>(url.toString(), { Accept: 'application/json', Authorization: `Bot ${key}` }, signal);
|
|
163
|
+
return (d.data?.search ?? [])
|
|
164
|
+
.filter((r) => r?.url || r?.link)
|
|
165
|
+
.map((r) => ({
|
|
166
|
+
title: r.title ?? r.t ?? r.url ?? r.link!,
|
|
167
|
+
url: r.url ?? r.link!,
|
|
168
|
+
snippet: r.snippet ?? r.s ?? '',
|
|
169
|
+
date: r.published,
|
|
170
|
+
}));
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
you: {
|
|
174
|
+
key: 'youKey',
|
|
175
|
+
search: async (key, opts, signal) => {
|
|
176
|
+
const d = await postJson<{
|
|
177
|
+
results?: {
|
|
178
|
+
web?: { url?: string; title?: string; description?: string; snippets?: string[]; page_age?: string }[];
|
|
179
|
+
news?: { url?: string; title?: string; description?: string; page_age?: string }[];
|
|
180
|
+
};
|
|
181
|
+
}>(
|
|
182
|
+
'https://ydc-index.io/v1/search',
|
|
183
|
+
{ query: opts.query, count: maxResults(opts), safesearch: 'moderate' },
|
|
184
|
+
jsonHeaders({ 'X-API-Key': key }),
|
|
185
|
+
signal,
|
|
186
|
+
);
|
|
187
|
+
return [
|
|
188
|
+
...(d.results?.web ?? []).map((w) => ({
|
|
189
|
+
title: w.title ?? w.url ?? '',
|
|
190
|
+
url: w.url ?? '',
|
|
191
|
+
snippet: w.snippets?.join('\n') || w.description || '',
|
|
192
|
+
date: w.page_age,
|
|
193
|
+
})),
|
|
194
|
+
...(d.results?.news ?? []).map((n) => ({
|
|
195
|
+
title: n.title ?? n.url ?? '',
|
|
196
|
+
url: n.url ?? '',
|
|
197
|
+
snippet: n.description || '',
|
|
198
|
+
date: n.page_age,
|
|
199
|
+
})),
|
|
200
|
+
].filter((r) => r.url);
|
|
201
|
+
},
|
|
202
|
+
},
|
|
203
|
+
firecrawl: {
|
|
204
|
+
key: 'firecrawlKey',
|
|
205
|
+
search: async (key, opts, signal) => {
|
|
206
|
+
type FcHit = { url?: string; title?: string; description?: string; snippet?: string; markdown?: string };
|
|
207
|
+
const d = await postJson<{ success?: boolean; data?: { web?: FcHit[] } | FcHit[]; results?: FcHit[] }>(
|
|
208
|
+
'https://api.firecrawl.dev/v2/search',
|
|
209
|
+
{ query: opts.query, limit: maxResults(opts), sources: ['web'] },
|
|
210
|
+
jsonHeaders({ Accept: 'application/json', Authorization: `Bearer ${key}` }),
|
|
211
|
+
signal,
|
|
212
|
+
);
|
|
213
|
+
const web: FcHit[] = Array.isArray(d.data) ? d.data : (d.data?.web ?? d.results ?? []);
|
|
214
|
+
return web
|
|
215
|
+
.filter((r) => r?.url)
|
|
216
|
+
.map((r) => ({
|
|
217
|
+
title: r.title ?? r.url!,
|
|
218
|
+
url: r.url!,
|
|
219
|
+
snippet: r.description ?? r.snippet ?? r.markdown ?? '',
|
|
220
|
+
}));
|
|
221
|
+
},
|
|
222
|
+
},
|
|
223
|
+
tinyfish: {
|
|
224
|
+
key: 'tinyfishKey',
|
|
225
|
+
search: async (key, opts, signal) => {
|
|
226
|
+
const url = new URL('https://api.search.tinyfish.ai');
|
|
227
|
+
url.searchParams.set('query', opts.query);
|
|
228
|
+
if (opts.maxResults) url.searchParams.set('num_results', String(opts.maxResults));
|
|
229
|
+
const d = await getJson<{ results?: { url?: string; title?: string; snippet?: string; position?: number }[] }>(
|
|
230
|
+
url.toString(),
|
|
231
|
+
{ Accept: 'application/json', 'X-API-Key': key },
|
|
232
|
+
signal,
|
|
233
|
+
);
|
|
234
|
+
return (d.results ?? [])
|
|
235
|
+
.filter((r) => r?.url)
|
|
236
|
+
.map((r) => ({ title: r.title ?? r.url!, url: r.url!, snippet: r.snippet ?? '' }));
|
|
237
|
+
},
|
|
238
|
+
},
|
|
239
|
+
parallel: {
|
|
240
|
+
key: 'parallelKey',
|
|
241
|
+
search: async (key, opts, signal) => {
|
|
242
|
+
const { query, domain } = extractSiteOperator(opts.query);
|
|
243
|
+
const advanced_settings: Record<string, unknown> = {
|
|
244
|
+
max_results: maxResults(opts),
|
|
245
|
+
excerpt_settings: { max_chars_per_result: 2_000 },
|
|
246
|
+
};
|
|
247
|
+
if (domain) advanced_settings.source_policy = { include_domains: [domain] };
|
|
248
|
+
const d = await postJson<{
|
|
249
|
+
results?: { url?: string; title?: string | null; publish_date?: string | null; excerpts?: string[] }[];
|
|
250
|
+
}>(
|
|
251
|
+
'https://api.parallel.ai/v1/search',
|
|
252
|
+
{ search_queries: [query], mode: 'fast', advanced_settings },
|
|
253
|
+
jsonHeaders({ 'x-api-key': key }),
|
|
254
|
+
signal,
|
|
255
|
+
);
|
|
256
|
+
return (d.results ?? [])
|
|
257
|
+
.filter((r) => r?.url)
|
|
258
|
+
.slice(0, maxResults(opts))
|
|
259
|
+
.map((r) => {
|
|
260
|
+
const hit: SearchHit = {
|
|
261
|
+
title: r.title ?? r.url!,
|
|
262
|
+
url: r.url!,
|
|
263
|
+
snippet: (r.excerpts ?? []).join(' '),
|
|
264
|
+
};
|
|
265
|
+
if (r.publish_date) hit.date = r.publish_date;
|
|
266
|
+
return hit;
|
|
267
|
+
});
|
|
268
|
+
},
|
|
269
|
+
},
|
|
270
|
+
keenable: {
|
|
271
|
+
key: 'keenableKey',
|
|
272
|
+
search: async (key, opts, signal) => {
|
|
273
|
+
const { query, domain } = extractSiteOperator(opts.query);
|
|
274
|
+
const body: Record<string, unknown> = {
|
|
275
|
+
query,
|
|
276
|
+
mode: 'pro',
|
|
277
|
+
max_results: maxResults(opts),
|
|
278
|
+
snippet_max_length: 2_000,
|
|
279
|
+
disable_cards: true,
|
|
280
|
+
};
|
|
281
|
+
if (domain) body.site = domain;
|
|
282
|
+
const d = await postJson<{
|
|
283
|
+
results?: { title?: string | null; url?: string; snippet?: string; published_at?: string | null }[];
|
|
284
|
+
}>('https://api.keenable.ai/v1/search', body, jsonHeaders({ 'X-API-Key': key }), signal);
|
|
285
|
+
return (d.results ?? [])
|
|
286
|
+
.filter((r) => r?.url)
|
|
287
|
+
.slice(0, maxResults(opts))
|
|
288
|
+
.map((r) => {
|
|
289
|
+
const hit: SearchHit = {
|
|
290
|
+
title: r.title ?? r.url!,
|
|
291
|
+
url: r.url!,
|
|
292
|
+
snippet: r.snippet ?? '',
|
|
293
|
+
};
|
|
294
|
+
if (r.published_at) hit.date = r.published_at;
|
|
295
|
+
return hit;
|
|
296
|
+
});
|
|
297
|
+
},
|
|
298
|
+
},
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
type EngineName = keyof typeof ENGINES;
|
|
34
302
|
|
|
35
303
|
const jsonHeaders = (extra: Record<string, string> = {}) => ({
|
|
36
304
|
'Content-Type': 'application/json',
|
|
@@ -87,283 +355,7 @@ function extractSiteOperator(query: string): { query: string; domain?: string }
|
|
|
87
355
|
};
|
|
88
356
|
}
|
|
89
357
|
|
|
90
|
-
|
|
91
|
-
const t0 = Date.now();
|
|
92
|
-
try {
|
|
93
|
-
const body: Record<string, unknown> = {
|
|
94
|
-
q: opts.query,
|
|
95
|
-
num: opts.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
96
|
-
gl: 'us',
|
|
97
|
-
};
|
|
98
|
-
const d = await postJson<{
|
|
99
|
-
organic?: { title?: string; link?: string; snippet?: string; date?: string }[];
|
|
100
|
-
}>('https://google.serper.dev/search', body, jsonHeaders({ 'X-API-KEY': key }), signal);
|
|
101
|
-
const hits: SearchHit[] = (d.organic ?? [])
|
|
102
|
-
.filter((r) => r?.link)
|
|
103
|
-
.map((r) => ({
|
|
104
|
-
title: r.title ?? r.link!,
|
|
105
|
-
url: r.link!,
|
|
106
|
-
snippet: r.snippet ?? '',
|
|
107
|
-
engine: 'serper',
|
|
108
|
-
date: r.date,
|
|
109
|
-
}));
|
|
110
|
-
return { engine: 'serper', hits, latencyMs: Date.now() - t0 };
|
|
111
|
-
} catch (e) {
|
|
112
|
-
return { engine: 'serper', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
113
|
-
}
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
async function tavily(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
117
|
-
const t0 = Date.now();
|
|
118
|
-
try {
|
|
119
|
-
const { query, domain } = extractSiteOperator(opts.query);
|
|
120
|
-
const body: Record<string, unknown> = {
|
|
121
|
-
query,
|
|
122
|
-
max_results: opts.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
123
|
-
search_depth: 'basic',
|
|
124
|
-
topic: 'general',
|
|
125
|
-
};
|
|
126
|
-
if (domain) body.include_domains = [domain];
|
|
127
|
-
// Docs: auth = Authorization: Bearer header
|
|
128
|
-
const d = await postJson<{
|
|
129
|
-
results?: { title?: string; url?: string; content?: string; published_date?: string }[];
|
|
130
|
-
}>('https://api.tavily.com/search', body, jsonHeaders({ Authorization: `Bearer ${key}` }), signal);
|
|
131
|
-
const hits: SearchHit[] = (d.results ?? [])
|
|
132
|
-
.filter((r) => r?.url)
|
|
133
|
-
.map((r) => ({
|
|
134
|
-
title: r.title ?? r.url!,
|
|
135
|
-
url: r.url!,
|
|
136
|
-
snippet: r.content ?? '',
|
|
137
|
-
engine: 'tavily',
|
|
138
|
-
date: r.published_date,
|
|
139
|
-
}));
|
|
140
|
-
return { engine: 'tavily', hits, latencyMs: Date.now() - t0 };
|
|
141
|
-
} catch (e) {
|
|
142
|
-
return { engine: 'tavily', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
async function exa(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
147
|
-
const t0 = Date.now();
|
|
148
|
-
try {
|
|
149
|
-
const { query, domain } = extractSiteOperator(opts.query);
|
|
150
|
-
const body: Record<string, unknown> = {
|
|
151
|
-
query,
|
|
152
|
-
numResults: opts.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
153
|
-
contents: { highlights: true },
|
|
154
|
-
};
|
|
155
|
-
if (domain) body.includeDomains = [domain];
|
|
156
|
-
const d = await postJson<{
|
|
157
|
-
results?: { title?: string; url?: string; highlights?: string[]; publishedDate?: string }[];
|
|
158
|
-
}>('https://api.exa.ai/search', body, jsonHeaders({ 'x-api-key': key }), signal);
|
|
159
|
-
const hits: SearchHit[] = (d.results ?? [])
|
|
160
|
-
.filter((r) => r?.url)
|
|
161
|
-
.map((r) => ({
|
|
162
|
-
title: r.title ?? r.url!,
|
|
163
|
-
url: r.url!,
|
|
164
|
-
snippet: (r.highlights ?? []).join(' '),
|
|
165
|
-
engine: 'exa',
|
|
166
|
-
date: r.publishedDate,
|
|
167
|
-
}));
|
|
168
|
-
return { engine: 'exa', hits, latencyMs: Date.now() - t0 };
|
|
169
|
-
} catch (e) {
|
|
170
|
-
return { engine: 'exa', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
async function brave(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
175
|
-
const t0 = Date.now();
|
|
176
|
-
try {
|
|
177
|
-
const url = new URL('https://api.search.brave.com/res/v1/web/search');
|
|
178
|
-
url.searchParams.set('q', opts.query);
|
|
179
|
-
url.searchParams.set('count', String(opts.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
180
|
-
url.searchParams.set('extra_snippets', 'true');
|
|
181
|
-
url.searchParams.set('text_decorations', 'false');
|
|
182
|
-
url.searchParams.set('safesearch', 'moderate');
|
|
183
|
-
const d = await getJson<{
|
|
184
|
-
web?: {
|
|
185
|
-
results?: { title?: string; url?: string; description?: string; extra_snippets?: string[]; age?: string }[];
|
|
186
|
-
};
|
|
187
|
-
}>(url.toString(), { Accept: 'application/json', 'X-Subscription-Token': key }, signal);
|
|
188
|
-
const hits: SearchHit[] = (d.web?.results ?? [])
|
|
189
|
-
.filter((r) => r?.url)
|
|
190
|
-
.map((r) => ({
|
|
191
|
-
title: r.title ?? r.url!,
|
|
192
|
-
url: r.url!,
|
|
193
|
-
snippet: [r.description, ...(r.extra_snippets ?? [])].filter(Boolean).join('\n'),
|
|
194
|
-
engine: 'brave',
|
|
195
|
-
date: r.age,
|
|
196
|
-
}));
|
|
197
|
-
return { engine: 'brave', hits, latencyMs: Date.now() - t0 };
|
|
198
|
-
} catch (e) {
|
|
199
|
-
return { engine: 'brave', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
async function jina(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
204
|
-
const t0 = Date.now();
|
|
205
|
-
try {
|
|
206
|
-
// documented form is s.jina.ai/?q=; search requires a key (no key = blocked)
|
|
207
|
-
const url = new URL('https://s.jina.ai/');
|
|
208
|
-
url.searchParams.set('q', opts.query);
|
|
209
|
-
url.searchParams.set('count', String(opts.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
210
|
-
const headers: Record<string, string> = {
|
|
211
|
-
Accept: 'application/json',
|
|
212
|
-
'X-Respond-With': 'no-content',
|
|
213
|
-
'X-Retain-Images': 'none',
|
|
214
|
-
};
|
|
215
|
-
headers.Authorization = `Bearer ${key}`;
|
|
216
|
-
const d = await getJson<{
|
|
217
|
-
data?: { title?: string | null; url?: string | null; description?: string | null; content?: string | null }[];
|
|
218
|
-
}>(url.toString(), headers, signal);
|
|
219
|
-
const hits: SearchHit[] = (d.data ?? [])
|
|
220
|
-
.filter((r) => r?.url)
|
|
221
|
-
.map((r) => ({
|
|
222
|
-
title: r.title ?? r.url!,
|
|
223
|
-
url: r.url!,
|
|
224
|
-
snippet: r.description?.trim() || r.content?.trim() || '',
|
|
225
|
-
engine: 'jina',
|
|
226
|
-
}));
|
|
227
|
-
return { engine: 'jina', hits, latencyMs: Date.now() - t0 };
|
|
228
|
-
} catch (e) {
|
|
229
|
-
return { engine: 'jina', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
async function kagi(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
234
|
-
const t0 = Date.now();
|
|
235
|
-
try {
|
|
236
|
-
const url = new URL('https://kagi.com/api/v1/search');
|
|
237
|
-
url.searchParams.set('q', opts.query);
|
|
238
|
-
url.searchParams.set('limit', String(opts.maxResults ?? DEFAULT_MAX_RESULTS));
|
|
239
|
-
const d = await getJson<{
|
|
240
|
-
data?: {
|
|
241
|
-
search?: {
|
|
242
|
-
url?: string;
|
|
243
|
-
link?: string;
|
|
244
|
-
title?: string;
|
|
245
|
-
t?: string;
|
|
246
|
-
snippet?: string;
|
|
247
|
-
s?: string;
|
|
248
|
-
published?: string;
|
|
249
|
-
}[];
|
|
250
|
-
};
|
|
251
|
-
}>(url.toString(), { Accept: 'application/json', Authorization: `Bot ${key}` }, signal);
|
|
252
|
-
const hits: SearchHit[] = (d.data?.search ?? [])
|
|
253
|
-
.filter((r) => r?.url || r?.link)
|
|
254
|
-
.map((r) => ({
|
|
255
|
-
title: r.title ?? r.t ?? r.url ?? r.link!,
|
|
256
|
-
url: r.url ?? r.link!,
|
|
257
|
-
snippet: r.snippet ?? r.s ?? '',
|
|
258
|
-
engine: 'kagi',
|
|
259
|
-
date: r.published,
|
|
260
|
-
}));
|
|
261
|
-
return { engine: 'kagi', hits, latencyMs: Date.now() - t0 };
|
|
262
|
-
} catch (e) {
|
|
263
|
-
return { engine: 'kagi', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
async function you(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
268
|
-
const t0 = Date.now();
|
|
269
|
-
try {
|
|
270
|
-
const body: Record<string, unknown> = {
|
|
271
|
-
query: opts.query,
|
|
272
|
-
count: opts.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
273
|
-
safesearch: 'moderate',
|
|
274
|
-
};
|
|
275
|
-
const d = await postJson<{
|
|
276
|
-
results?: {
|
|
277
|
-
web?: { url?: string; title?: string; description?: string; snippets?: string[]; page_age?: string }[];
|
|
278
|
-
news?: { url?: string; title?: string; description?: string; page_age?: string }[];
|
|
279
|
-
};
|
|
280
|
-
}>('https://ydc-index.io/v1/search', body, jsonHeaders({ 'X-API-Key': key }), signal);
|
|
281
|
-
const rows = [
|
|
282
|
-
...(d.results?.web ?? []).map((w) => ({
|
|
283
|
-
title: w.title ?? w.url ?? '',
|
|
284
|
-
url: w.url ?? '',
|
|
285
|
-
snippet: w.snippets?.join('\n') || w.description || '',
|
|
286
|
-
date: w.page_age,
|
|
287
|
-
})),
|
|
288
|
-
...(d.results?.news ?? []).map((n) => ({
|
|
289
|
-
title: n.title ?? n.url ?? '',
|
|
290
|
-
url: n.url ?? '',
|
|
291
|
-
snippet: n.description || '',
|
|
292
|
-
date: n.page_age,
|
|
293
|
-
})),
|
|
294
|
-
].filter((r) => r.url);
|
|
295
|
-
const hits: SearchHit[] = rows.map((r) => ({ ...r, engine: 'you' }));
|
|
296
|
-
return { engine: 'you', hits, latencyMs: Date.now() - t0 };
|
|
297
|
-
} catch (e) {
|
|
298
|
-
return { engine: 'you', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
async function firecrawl(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
303
|
-
const t0 = Date.now();
|
|
304
|
-
try {
|
|
305
|
-
const body: Record<string, unknown> = {
|
|
306
|
-
query: opts.query,
|
|
307
|
-
limit: opts.maxResults ?? DEFAULT_MAX_RESULTS,
|
|
308
|
-
sources: ['web'],
|
|
309
|
-
};
|
|
310
|
-
const headers: Record<string, string> = {
|
|
311
|
-
Accept: 'application/json',
|
|
312
|
-
'Content-Type': 'application/json',
|
|
313
|
-
Authorization: `Bearer ${key}`,
|
|
314
|
-
};
|
|
315
|
-
type FcHit = { url?: string; title?: string; description?: string; snippet?: string; markdown?: string };
|
|
316
|
-
const d = await request<{
|
|
317
|
-
success?: boolean;
|
|
318
|
-
data?: { web?: FcHit[] } | FcHit[];
|
|
319
|
-
results?: FcHit[];
|
|
320
|
-
}>('https://api.firecrawl.dev/v2/search', { method: 'POST', headers, body: JSON.stringify(body) }, signal);
|
|
321
|
-
const web: FcHit[] = Array.isArray(d.data) ? d.data : (d.data?.web ?? d.results ?? []);
|
|
322
|
-
const hits: SearchHit[] = web
|
|
323
|
-
.filter((r) => r?.url)
|
|
324
|
-
.map((r) => ({
|
|
325
|
-
title: r.title ?? r.url!,
|
|
326
|
-
url: r.url!,
|
|
327
|
-
snippet: r.description ?? r.snippet ?? r.markdown ?? '',
|
|
328
|
-
engine: 'firecrawl',
|
|
329
|
-
}));
|
|
330
|
-
return { engine: 'firecrawl', hits, latencyMs: Date.now() - t0 };
|
|
331
|
-
} catch (e) {
|
|
332
|
-
return { engine: 'firecrawl', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
|
|
336
|
-
async function tinyfish(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
|
|
337
|
-
const t0 = Date.now();
|
|
338
|
-
try {
|
|
339
|
-
const url = new URL('https://api.search.tinyfish.ai');
|
|
340
|
-
url.searchParams.set('query', opts.query);
|
|
341
|
-
if (opts.maxResults) url.searchParams.set('num_results', String(opts.maxResults));
|
|
342
|
-
const d = await getJson<{ results?: { url?: string; title?: string; snippet?: string; position?: number }[] }>(
|
|
343
|
-
url.toString(),
|
|
344
|
-
{ Accept: 'application/json', 'X-API-Key': key },
|
|
345
|
-
signal,
|
|
346
|
-
);
|
|
347
|
-
const hits: SearchHit[] = (d.results ?? [])
|
|
348
|
-
.filter((r) => r?.url)
|
|
349
|
-
.map((r) => ({ title: r.title ?? r.url!, url: r.url!, snippet: r.snippet ?? '', engine: 'tinyfish' }));
|
|
350
|
-
return { engine: 'tinyfish', hits, latencyMs: Date.now() - t0 };
|
|
351
|
-
} catch (e) {
|
|
352
|
-
return { engine: 'tinyfish', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
const ENGINE_REGISTRY: EngineDef[] = [
|
|
357
|
-
{ id: 'serper', key: 'serperKey', available: (c) => !!c.serperKey, search: serper },
|
|
358
|
-
{ id: 'tavily', key: 'tavilyKey', available: (c) => !!c.tavilyKey, search: tavily },
|
|
359
|
-
{ id: 'exa', key: 'exaKey', available: (c) => !!c.exaKey, search: exa },
|
|
360
|
-
{ id: 'brave', key: 'braveKey', available: (c) => !!c.braveKey, search: brave },
|
|
361
|
-
{ id: 'jina', key: 'jinaKey', available: (c) => !!c.jinaKey, search: jina },
|
|
362
|
-
{ id: 'kagi', key: 'kagiKey', available: (c) => !!c.kagiKey, search: kagi },
|
|
363
|
-
{ id: 'you', key: 'youKey', available: (c) => !!c.youKey, search: you },
|
|
364
|
-
{ id: 'firecrawl', key: 'firecrawlKey', available: (c) => !!c.firecrawlKey, search: firecrawl },
|
|
365
|
-
{ id: 'tinyfish', key: 'tinyfishKey', available: (c) => !!c.tinyfishKey, search: tinyfish },
|
|
366
|
-
];
|
|
358
|
+
const maxResults = (opts: SearchOptions): number => opts.maxResults ?? DEFAULT_MAX_RESULTS;
|
|
367
359
|
|
|
368
360
|
function parseEngineList(raw: string | undefined): EngineName[] | undefined {
|
|
369
361
|
if (!raw) return undefined;
|
|
@@ -371,7 +363,7 @@ function parseEngineList(raw: string | undefined): EngineName[] | undefined {
|
|
|
371
363
|
for (const s of raw.split(',')) {
|
|
372
364
|
const t = s.trim().toLowerCase();
|
|
373
365
|
if (!t) continue;
|
|
374
|
-
if (
|
|
366
|
+
if (Object.hasOwn(ENGINES, t)) out.push(t as EngineName);
|
|
375
367
|
}
|
|
376
368
|
return out.length ? out : undefined;
|
|
377
369
|
}
|
|
@@ -383,22 +375,14 @@ export function clearDeadEngines(): void {
|
|
|
383
375
|
deadEngines.clear();
|
|
384
376
|
}
|
|
385
377
|
|
|
386
|
-
function selectEngines(opts: SearchOptions): EngineDef[] {
|
|
387
|
-
const cfg = getConfig();
|
|
388
|
-
const wanted = opts.engines ?? parseEngineList(cfg.searchEngines);
|
|
389
|
-
const selected = ENGINE_REGISTRY.filter((def) => {
|
|
390
|
-
if (!def.available(cfg)) return false;
|
|
391
|
-
return wanted ? wanted.includes(def.id) : true;
|
|
392
|
-
});
|
|
393
|
-
return selected;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
378
|
export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]> {
|
|
397
379
|
const cfg = getConfig();
|
|
398
|
-
const
|
|
399
|
-
const
|
|
380
|
+
const wanted = opts.engines ?? parseEngineList(cfg.searchEngines);
|
|
381
|
+
const selected = Object.entries(ENGINES).filter(
|
|
382
|
+
([id, def]) => !!cfg[def.key] && (!wanted || wanted.includes(id as EngineName)),
|
|
383
|
+
);
|
|
384
|
+
const chosen = selected.filter(([id]) => !deadEngines.has(id));
|
|
400
385
|
if (chosen.length === 0) {
|
|
401
|
-
const wanted = opts.engines ?? parseEngineList(getConfig().searchEngines);
|
|
402
386
|
// selected-but-dead means KEYS EXIST and the engines were temp-banned
|
|
403
387
|
// (stable HTTP 401/402/403): the old wording lied about the cause.
|
|
404
388
|
const error =
|
|
@@ -407,13 +391,19 @@ export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]>
|
|
|
407
391
|
: wanted?.length
|
|
408
392
|
? `no configured engines match the requested set (${wanted.join(', ')})`
|
|
409
393
|
: 'no search engine keys configured; run /websearch to add one';
|
|
410
|
-
return [{ engine: 'none', hits: [], error
|
|
394
|
+
return [{ engine: 'none', hits: [], error }];
|
|
411
395
|
}
|
|
412
396
|
return Promise.all(
|
|
413
|
-
chosen.map(async (def) => {
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
397
|
+
chosen.map(async ([id, def]) => {
|
|
398
|
+
let hits: SearchHit[] = [];
|
|
399
|
+
let error: string | undefined;
|
|
400
|
+
try {
|
|
401
|
+
hits = await def.search(cfg[def.key] as string, opts, opts.signal);
|
|
402
|
+
} catch (e) {
|
|
403
|
+
error = sanitizeError(e);
|
|
404
|
+
}
|
|
405
|
+
if (error && STABLE_ENGINE_ERROR.test(error)) deadEngines.add(id);
|
|
406
|
+
return { engine: id, hits, error };
|
|
417
407
|
}),
|
|
418
408
|
);
|
|
419
409
|
}
|
package/src/fetch.ts
CHANGED
|
@@ -36,7 +36,7 @@ function fetchErrorResult(details: FetchDetails): FetchResult {
|
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
function
|
|
39
|
+
export function makeFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
|
|
40
40
|
const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
|
|
41
41
|
const body = res.outline.length
|
|
42
42
|
? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
|
|
@@ -116,7 +116,3 @@ function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: num
|
|
|
116
116
|
if (res.error) return fetchErrorResult(details);
|
|
117
117
|
return { content: [{ type: 'text', text: clipped }], details, structuredContent: { ...details } };
|
|
118
118
|
}
|
|
119
|
-
|
|
120
|
-
export function makeFetchResult(res: Awaited<ReturnType<typeof scrape>>, params: { sections: number[] }): FetchResult {
|
|
121
|
-
return buildFetchResult(res, params.sections);
|
|
122
|
-
}
|