pi-wsearch 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/engines.ts CHANGED
@@ -1,11 +1,10 @@
1
- import { getConfig, sanitizeError, type ProviderKey, type WebConfig } from './config';
2
- import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './constants';
1
+ import { getConfig, sanitizeError, type ProviderKey } from './config';
2
+ import { ABORT_ERROR, DEFAULT_MAX_RESULTS } from './config';
3
3
 
4
4
  export interface SearchHit {
5
5
  title: string;
6
6
  url: string;
7
7
  snippet: string;
8
- engine: string;
9
8
  date?: string;
10
9
  }
11
10
 
@@ -13,11 +12,8 @@ export interface EngineOutcome {
13
12
  engine: string;
14
13
  hits: SearchHit[];
15
14
  error?: string;
16
- latencyMs: number;
17
15
  }
18
16
 
19
- type EngineName = 'serper' | 'tavily' | 'exa' | 'brave' | 'jina' | 'kagi' | 'you' | 'firecrawl' | 'tinyfish';
20
-
21
17
  export interface SearchOptions {
22
18
  query: string;
23
19
  maxResults?: number;
@@ -25,12 +21,284 @@ export interface SearchOptions {
25
21
  signal?: AbortSignal;
26
22
  }
27
23
 
28
- interface EngineDef {
29
- id: EngineName;
30
- key: ProviderKey;
31
- available(cfg: WebConfig): boolean;
32
- search(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome>;
33
- }
24
+ type EngineSearch = (key: string, opts: SearchOptions, signal?: AbortSignal) => Promise<SearchHit[]>;
25
+
26
+ const ENGINES: Record<string, { key: ProviderKey; search: EngineSearch }> = {
27
+ serper: {
28
+ key: 'serperKey',
29
+ search: async (key, opts, signal) => {
30
+ const d = await postJson<{ organic?: { title?: string; link?: string; snippet?: string; date?: string }[] }>(
31
+ 'https://google.serper.dev/search',
32
+ { q: opts.query, num: maxResults(opts), gl: 'us' },
33
+ jsonHeaders({ 'X-API-KEY': key }),
34
+ signal,
35
+ );
36
+ return (d.organic ?? [])
37
+ .filter((r) => r?.link)
38
+ .map((r) => ({
39
+ title: r.title ?? r.link!,
40
+ url: r.link!,
41
+ snippet: r.snippet ?? '',
42
+ date: r.date,
43
+ }));
44
+ },
45
+ },
46
+ tavily: {
47
+ key: 'tavilyKey',
48
+ search: async (key, opts, signal) => {
49
+ const { query, domain } = extractSiteOperator(opts.query);
50
+ const body: Record<string, unknown> = {
51
+ query,
52
+ max_results: maxResults(opts),
53
+ search_depth: 'basic',
54
+ topic: 'general',
55
+ };
56
+ if (domain) body.include_domains = [domain];
57
+ const d = await postJson<{
58
+ results?: { title?: string; url?: string; content?: string; published_date?: string }[];
59
+ }>('https://api.tavily.com/search', body, jsonHeaders({ Authorization: `Bearer ${key}` }), signal);
60
+ return (d.results ?? [])
61
+ .filter((r) => r?.url)
62
+ .map((r) => ({
63
+ title: r.title ?? r.url!,
64
+ url: r.url!,
65
+ snippet: r.content ?? '',
66
+ date: r.published_date,
67
+ }));
68
+ },
69
+ },
70
+ exa: {
71
+ key: 'exaKey',
72
+ search: async (key, opts, signal) => {
73
+ const { query, domain } = extractSiteOperator(opts.query);
74
+ const body: Record<string, unknown> = {
75
+ query,
76
+ numResults: maxResults(opts),
77
+ contents: { highlights: true },
78
+ };
79
+ if (domain) body.includeDomains = [domain];
80
+ const d = await postJson<{
81
+ results?: { title?: string; url?: string; highlights?: string[]; publishedDate?: string }[];
82
+ }>('https://api.exa.ai/search', body, jsonHeaders({ 'x-api-key': key }), signal);
83
+ return (d.results ?? [])
84
+ .filter((r) => r?.url)
85
+ .map((r) => ({
86
+ title: r.title ?? r.url!,
87
+ url: r.url!,
88
+ snippet: (r.highlights ?? []).join(' '),
89
+ date: r.publishedDate,
90
+ }));
91
+ },
92
+ },
93
+ brave: {
94
+ key: 'braveKey',
95
+ search: async (key, opts, signal) => {
96
+ const url = new URL('https://api.search.brave.com/res/v1/web/search');
97
+ url.searchParams.set('q', opts.query);
98
+ url.searchParams.set('count', String(maxResults(opts)));
99
+ url.searchParams.set('extra_snippets', 'true');
100
+ url.searchParams.set('text_decorations', 'false');
101
+ url.searchParams.set('safesearch', 'moderate');
102
+ const d = await getJson<{
103
+ web?: {
104
+ results?: { title?: string; url?: string; description?: string; extra_snippets?: string[]; age?: string }[];
105
+ };
106
+ }>(url.toString(), { Accept: 'application/json', 'X-Subscription-Token': key }, signal);
107
+ return (d.web?.results ?? [])
108
+ .filter((r) => r?.url)
109
+ .map((r) => ({
110
+ title: r.title ?? r.url!,
111
+ url: r.url!,
112
+ snippet: [r.description, ...(r.extra_snippets ?? [])].filter(Boolean).join('\n'),
113
+ date: r.age,
114
+ }));
115
+ },
116
+ },
117
+ jina: {
118
+ key: 'jinaKey',
119
+ search: async (key, opts, signal) => {
120
+ const url = new URL('https://s.jina.ai/');
121
+ url.searchParams.set('q', opts.query);
122
+ url.searchParams.set('count', String(maxResults(opts)));
123
+ const d = await getJson<{
124
+ data?: { title?: string | null; url?: string | null; description?: string | null; content?: string | null }[];
125
+ }>(
126
+ url.toString(),
127
+ {
128
+ Accept: 'application/json',
129
+ 'X-Respond-With': 'no-content',
130
+ 'X-Retain-Images': 'none',
131
+ Authorization: `Bearer ${key}`,
132
+ },
133
+ signal,
134
+ );
135
+ return (d.data ?? [])
136
+ .filter((r) => r?.url)
137
+ .map((r) => ({
138
+ title: r.title ?? r.url!,
139
+ url: r.url!,
140
+ snippet: r.description?.trim() || r.content?.trim() || '',
141
+ }));
142
+ },
143
+ },
144
+ kagi: {
145
+ key: 'kagiKey',
146
+ search: async (key, opts, signal) => {
147
+ const url = new URL('https://kagi.com/api/v1/search');
148
+ url.searchParams.set('q', opts.query);
149
+ url.searchParams.set('limit', String(maxResults(opts)));
150
+ const d = await getJson<{
151
+ data?: {
152
+ search?: {
153
+ url?: string;
154
+ link?: string;
155
+ title?: string;
156
+ t?: string;
157
+ snippet?: string;
158
+ s?: string;
159
+ published?: string;
160
+ }[];
161
+ };
162
+ }>(url.toString(), { Accept: 'application/json', Authorization: `Bot ${key}` }, signal);
163
+ return (d.data?.search ?? [])
164
+ .filter((r) => r?.url || r?.link)
165
+ .map((r) => ({
166
+ title: r.title ?? r.t ?? r.url ?? r.link!,
167
+ url: r.url ?? r.link!,
168
+ snippet: r.snippet ?? r.s ?? '',
169
+ date: r.published,
170
+ }));
171
+ },
172
+ },
173
+ you: {
174
+ key: 'youKey',
175
+ search: async (key, opts, signal) => {
176
+ const d = await postJson<{
177
+ results?: {
178
+ web?: { url?: string; title?: string; description?: string; snippets?: string[]; page_age?: string }[];
179
+ news?: { url?: string; title?: string; description?: string; page_age?: string }[];
180
+ };
181
+ }>(
182
+ 'https://ydc-index.io/v1/search',
183
+ { query: opts.query, count: maxResults(opts), safesearch: 'moderate' },
184
+ jsonHeaders({ 'X-API-Key': key }),
185
+ signal,
186
+ );
187
+ return [
188
+ ...(d.results?.web ?? []).map((w) => ({
189
+ title: w.title ?? w.url ?? '',
190
+ url: w.url ?? '',
191
+ snippet: w.snippets?.join('\n') || w.description || '',
192
+ date: w.page_age,
193
+ })),
194
+ ...(d.results?.news ?? []).map((n) => ({
195
+ title: n.title ?? n.url ?? '',
196
+ url: n.url ?? '',
197
+ snippet: n.description || '',
198
+ date: n.page_age,
199
+ })),
200
+ ].filter((r) => r.url);
201
+ },
202
+ },
203
+ firecrawl: {
204
+ key: 'firecrawlKey',
205
+ search: async (key, opts, signal) => {
206
+ type FcHit = { url?: string; title?: string; description?: string; snippet?: string; markdown?: string };
207
+ const d = await postJson<{ success?: boolean; data?: { web?: FcHit[] } | FcHit[]; results?: FcHit[] }>(
208
+ 'https://api.firecrawl.dev/v2/search',
209
+ { query: opts.query, limit: maxResults(opts), sources: ['web'] },
210
+ jsonHeaders({ Accept: 'application/json', Authorization: `Bearer ${key}` }),
211
+ signal,
212
+ );
213
+ const web: FcHit[] = Array.isArray(d.data) ? d.data : (d.data?.web ?? d.results ?? []);
214
+ return web
215
+ .filter((r) => r?.url)
216
+ .map((r) => ({
217
+ title: r.title ?? r.url!,
218
+ url: r.url!,
219
+ snippet: r.description ?? r.snippet ?? r.markdown ?? '',
220
+ }));
221
+ },
222
+ },
223
+ tinyfish: {
224
+ key: 'tinyfishKey',
225
+ search: async (key, opts, signal) => {
226
+ const url = new URL('https://api.search.tinyfish.ai');
227
+ url.searchParams.set('query', opts.query);
228
+ if (opts.maxResults) url.searchParams.set('num_results', String(opts.maxResults));
229
+ const d = await getJson<{ results?: { url?: string; title?: string; snippet?: string; position?: number }[] }>(
230
+ url.toString(),
231
+ { Accept: 'application/json', 'X-API-Key': key },
232
+ signal,
233
+ );
234
+ return (d.results ?? [])
235
+ .filter((r) => r?.url)
236
+ .map((r) => ({ title: r.title ?? r.url!, url: r.url!, snippet: r.snippet ?? '' }));
237
+ },
238
+ },
239
+ parallel: {
240
+ key: 'parallelKey',
241
+ search: async (key, opts, signal) => {
242
+ const { query, domain } = extractSiteOperator(opts.query);
243
+ const advanced_settings: Record<string, unknown> = {
244
+ max_results: maxResults(opts),
245
+ excerpt_settings: { max_chars_per_result: 2_000 },
246
+ };
247
+ if (domain) advanced_settings.source_policy = { include_domains: [domain] };
248
+ const d = await postJson<{
249
+ results?: { url?: string; title?: string | null; publish_date?: string | null; excerpts?: string[] }[];
250
+ }>(
251
+ 'https://api.parallel.ai/v1/search',
252
+ { search_queries: [query], mode: 'fast', advanced_settings },
253
+ jsonHeaders({ 'x-api-key': key }),
254
+ signal,
255
+ );
256
+ return (d.results ?? [])
257
+ .filter((r) => r?.url)
258
+ .slice(0, maxResults(opts))
259
+ .map((r) => {
260
+ const hit: SearchHit = {
261
+ title: r.title ?? r.url!,
262
+ url: r.url!,
263
+ snippet: (r.excerpts ?? []).join(' '),
264
+ };
265
+ if (r.publish_date) hit.date = r.publish_date;
266
+ return hit;
267
+ });
268
+ },
269
+ },
270
+ keenable: {
271
+ key: 'keenableKey',
272
+ search: async (key, opts, signal) => {
273
+ const { query, domain } = extractSiteOperator(opts.query);
274
+ const body: Record<string, unknown> = {
275
+ query,
276
+ mode: 'pro',
277
+ max_results: maxResults(opts),
278
+ snippet_max_length: 2_000,
279
+ disable_cards: true,
280
+ };
281
+ if (domain) body.site = domain;
282
+ const d = await postJson<{
283
+ results?: { title?: string | null; url?: string; snippet?: string; published_at?: string | null }[];
284
+ }>('https://api.keenable.ai/v1/search', body, jsonHeaders({ 'X-API-Key': key }), signal);
285
+ return (d.results ?? [])
286
+ .filter((r) => r?.url)
287
+ .slice(0, maxResults(opts))
288
+ .map((r) => {
289
+ const hit: SearchHit = {
290
+ title: r.title ?? r.url!,
291
+ url: r.url!,
292
+ snippet: r.snippet ?? '',
293
+ };
294
+ if (r.published_at) hit.date = r.published_at;
295
+ return hit;
296
+ });
297
+ },
298
+ },
299
+ };
300
+
301
+ type EngineName = keyof typeof ENGINES;
34
302
 
35
303
  const jsonHeaders = (extra: Record<string, string> = {}) => ({
36
304
  'Content-Type': 'application/json',
@@ -87,283 +355,7 @@ function extractSiteOperator(query: string): { query: string; domain?: string }
87
355
  };
88
356
  }
89
357
 
90
- async function serper(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
91
- const t0 = Date.now();
92
- try {
93
- const body: Record<string, unknown> = {
94
- q: opts.query,
95
- num: opts.maxResults ?? DEFAULT_MAX_RESULTS,
96
- gl: 'us',
97
- };
98
- const d = await postJson<{
99
- organic?: { title?: string; link?: string; snippet?: string; date?: string }[];
100
- }>('https://google.serper.dev/search', body, jsonHeaders({ 'X-API-KEY': key }), signal);
101
- const hits: SearchHit[] = (d.organic ?? [])
102
- .filter((r) => r?.link)
103
- .map((r) => ({
104
- title: r.title ?? r.link!,
105
- url: r.link!,
106
- snippet: r.snippet ?? '',
107
- engine: 'serper',
108
- date: r.date,
109
- }));
110
- return { engine: 'serper', hits, latencyMs: Date.now() - t0 };
111
- } catch (e) {
112
- return { engine: 'serper', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
113
- }
114
- }
115
-
116
- async function tavily(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
117
- const t0 = Date.now();
118
- try {
119
- const { query, domain } = extractSiteOperator(opts.query);
120
- const body: Record<string, unknown> = {
121
- query,
122
- max_results: opts.maxResults ?? DEFAULT_MAX_RESULTS,
123
- search_depth: 'basic',
124
- topic: 'general',
125
- };
126
- if (domain) body.include_domains = [domain];
127
- // Docs: auth = Authorization: Bearer header
128
- const d = await postJson<{
129
- results?: { title?: string; url?: string; content?: string; published_date?: string }[];
130
- }>('https://api.tavily.com/search', body, jsonHeaders({ Authorization: `Bearer ${key}` }), signal);
131
- const hits: SearchHit[] = (d.results ?? [])
132
- .filter((r) => r?.url)
133
- .map((r) => ({
134
- title: r.title ?? r.url!,
135
- url: r.url!,
136
- snippet: r.content ?? '',
137
- engine: 'tavily',
138
- date: r.published_date,
139
- }));
140
- return { engine: 'tavily', hits, latencyMs: Date.now() - t0 };
141
- } catch (e) {
142
- return { engine: 'tavily', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
143
- }
144
- }
145
-
146
- async function exa(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
147
- const t0 = Date.now();
148
- try {
149
- const { query, domain } = extractSiteOperator(opts.query);
150
- const body: Record<string, unknown> = {
151
- query,
152
- numResults: opts.maxResults ?? DEFAULT_MAX_RESULTS,
153
- contents: { highlights: true },
154
- };
155
- if (domain) body.includeDomains = [domain];
156
- const d = await postJson<{
157
- results?: { title?: string; url?: string; highlights?: string[]; publishedDate?: string }[];
158
- }>('https://api.exa.ai/search', body, jsonHeaders({ 'x-api-key': key }), signal);
159
- const hits: SearchHit[] = (d.results ?? [])
160
- .filter((r) => r?.url)
161
- .map((r) => ({
162
- title: r.title ?? r.url!,
163
- url: r.url!,
164
- snippet: (r.highlights ?? []).join(' '),
165
- engine: 'exa',
166
- date: r.publishedDate,
167
- }));
168
- return { engine: 'exa', hits, latencyMs: Date.now() - t0 };
169
- } catch (e) {
170
- return { engine: 'exa', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
171
- }
172
- }
173
-
174
- async function brave(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
175
- const t0 = Date.now();
176
- try {
177
- const url = new URL('https://api.search.brave.com/res/v1/web/search');
178
- url.searchParams.set('q', opts.query);
179
- url.searchParams.set('count', String(opts.maxResults ?? DEFAULT_MAX_RESULTS));
180
- url.searchParams.set('extra_snippets', 'true');
181
- url.searchParams.set('text_decorations', 'false');
182
- url.searchParams.set('safesearch', 'moderate');
183
- const d = await getJson<{
184
- web?: {
185
- results?: { title?: string; url?: string; description?: string; extra_snippets?: string[]; age?: string }[];
186
- };
187
- }>(url.toString(), { Accept: 'application/json', 'X-Subscription-Token': key }, signal);
188
- const hits: SearchHit[] = (d.web?.results ?? [])
189
- .filter((r) => r?.url)
190
- .map((r) => ({
191
- title: r.title ?? r.url!,
192
- url: r.url!,
193
- snippet: [r.description, ...(r.extra_snippets ?? [])].filter(Boolean).join('\n'),
194
- engine: 'brave',
195
- date: r.age,
196
- }));
197
- return { engine: 'brave', hits, latencyMs: Date.now() - t0 };
198
- } catch (e) {
199
- return { engine: 'brave', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
200
- }
201
- }
202
-
203
- async function jina(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
204
- const t0 = Date.now();
205
- try {
206
- // documented form is s.jina.ai/?q=; search requires a key (no key = blocked)
207
- const url = new URL('https://s.jina.ai/');
208
- url.searchParams.set('q', opts.query);
209
- url.searchParams.set('count', String(opts.maxResults ?? DEFAULT_MAX_RESULTS));
210
- const headers: Record<string, string> = {
211
- Accept: 'application/json',
212
- 'X-Respond-With': 'no-content',
213
- 'X-Retain-Images': 'none',
214
- };
215
- headers.Authorization = `Bearer ${key}`;
216
- const d = await getJson<{
217
- data?: { title?: string | null; url?: string | null; description?: string | null; content?: string | null }[];
218
- }>(url.toString(), headers, signal);
219
- const hits: SearchHit[] = (d.data ?? [])
220
- .filter((r) => r?.url)
221
- .map((r) => ({
222
- title: r.title ?? r.url!,
223
- url: r.url!,
224
- snippet: r.description?.trim() || r.content?.trim() || '',
225
- engine: 'jina',
226
- }));
227
- return { engine: 'jina', hits, latencyMs: Date.now() - t0 };
228
- } catch (e) {
229
- return { engine: 'jina', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
230
- }
231
- }
232
-
233
- async function kagi(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
234
- const t0 = Date.now();
235
- try {
236
- const url = new URL('https://kagi.com/api/v1/search');
237
- url.searchParams.set('q', opts.query);
238
- url.searchParams.set('limit', String(opts.maxResults ?? DEFAULT_MAX_RESULTS));
239
- const d = await getJson<{
240
- data?: {
241
- search?: {
242
- url?: string;
243
- link?: string;
244
- title?: string;
245
- t?: string;
246
- snippet?: string;
247
- s?: string;
248
- published?: string;
249
- }[];
250
- };
251
- }>(url.toString(), { Accept: 'application/json', Authorization: `Bot ${key}` }, signal);
252
- const hits: SearchHit[] = (d.data?.search ?? [])
253
- .filter((r) => r?.url || r?.link)
254
- .map((r) => ({
255
- title: r.title ?? r.t ?? r.url ?? r.link!,
256
- url: r.url ?? r.link!,
257
- snippet: r.snippet ?? r.s ?? '',
258
- engine: 'kagi',
259
- date: r.published,
260
- }));
261
- return { engine: 'kagi', hits, latencyMs: Date.now() - t0 };
262
- } catch (e) {
263
- return { engine: 'kagi', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
264
- }
265
- }
266
-
267
- async function you(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
268
- const t0 = Date.now();
269
- try {
270
- const body: Record<string, unknown> = {
271
- query: opts.query,
272
- count: opts.maxResults ?? DEFAULT_MAX_RESULTS,
273
- safesearch: 'moderate',
274
- };
275
- const d = await postJson<{
276
- results?: {
277
- web?: { url?: string; title?: string; description?: string; snippets?: string[]; page_age?: string }[];
278
- news?: { url?: string; title?: string; description?: string; page_age?: string }[];
279
- };
280
- }>('https://ydc-index.io/v1/search', body, jsonHeaders({ 'X-API-Key': key }), signal);
281
- const rows = [
282
- ...(d.results?.web ?? []).map((w) => ({
283
- title: w.title ?? w.url ?? '',
284
- url: w.url ?? '',
285
- snippet: w.snippets?.join('\n') || w.description || '',
286
- date: w.page_age,
287
- })),
288
- ...(d.results?.news ?? []).map((n) => ({
289
- title: n.title ?? n.url ?? '',
290
- url: n.url ?? '',
291
- snippet: n.description || '',
292
- date: n.page_age,
293
- })),
294
- ].filter((r) => r.url);
295
- const hits: SearchHit[] = rows.map((r) => ({ ...r, engine: 'you' }));
296
- return { engine: 'you', hits, latencyMs: Date.now() - t0 };
297
- } catch (e) {
298
- return { engine: 'you', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
299
- }
300
- }
301
-
302
- async function firecrawl(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
303
- const t0 = Date.now();
304
- try {
305
- const body: Record<string, unknown> = {
306
- query: opts.query,
307
- limit: opts.maxResults ?? DEFAULT_MAX_RESULTS,
308
- sources: ['web'],
309
- };
310
- const headers: Record<string, string> = {
311
- Accept: 'application/json',
312
- 'Content-Type': 'application/json',
313
- Authorization: `Bearer ${key}`,
314
- };
315
- type FcHit = { url?: string; title?: string; description?: string; snippet?: string; markdown?: string };
316
- const d = await request<{
317
- success?: boolean;
318
- data?: { web?: FcHit[] } | FcHit[];
319
- results?: FcHit[];
320
- }>('https://api.firecrawl.dev/v2/search', { method: 'POST', headers, body: JSON.stringify(body) }, signal);
321
- const web: FcHit[] = Array.isArray(d.data) ? d.data : (d.data?.web ?? d.results ?? []);
322
- const hits: SearchHit[] = web
323
- .filter((r) => r?.url)
324
- .map((r) => ({
325
- title: r.title ?? r.url!,
326
- url: r.url!,
327
- snippet: r.description ?? r.snippet ?? r.markdown ?? '',
328
- engine: 'firecrawl',
329
- }));
330
- return { engine: 'firecrawl', hits, latencyMs: Date.now() - t0 };
331
- } catch (e) {
332
- return { engine: 'firecrawl', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
333
- }
334
- }
335
-
336
- async function tinyfish(key: string, opts: SearchOptions, signal?: AbortSignal): Promise<EngineOutcome> {
337
- const t0 = Date.now();
338
- try {
339
- const url = new URL('https://api.search.tinyfish.ai');
340
- url.searchParams.set('query', opts.query);
341
- if (opts.maxResults) url.searchParams.set('num_results', String(opts.maxResults));
342
- const d = await getJson<{ results?: { url?: string; title?: string; snippet?: string; position?: number }[] }>(
343
- url.toString(),
344
- { Accept: 'application/json', 'X-API-Key': key },
345
- signal,
346
- );
347
- const hits: SearchHit[] = (d.results ?? [])
348
- .filter((r) => r?.url)
349
- .map((r) => ({ title: r.title ?? r.url!, url: r.url!, snippet: r.snippet ?? '', engine: 'tinyfish' }));
350
- return { engine: 'tinyfish', hits, latencyMs: Date.now() - t0 };
351
- } catch (e) {
352
- return { engine: 'tinyfish', hits: [], error: sanitizeError(e), latencyMs: Date.now() - t0 };
353
- }
354
- }
355
-
356
- const ENGINE_REGISTRY: EngineDef[] = [
357
- { id: 'serper', key: 'serperKey', available: (c) => !!c.serperKey, search: serper },
358
- { id: 'tavily', key: 'tavilyKey', available: (c) => !!c.tavilyKey, search: tavily },
359
- { id: 'exa', key: 'exaKey', available: (c) => !!c.exaKey, search: exa },
360
- { id: 'brave', key: 'braveKey', available: (c) => !!c.braveKey, search: brave },
361
- { id: 'jina', key: 'jinaKey', available: (c) => !!c.jinaKey, search: jina },
362
- { id: 'kagi', key: 'kagiKey', available: (c) => !!c.kagiKey, search: kagi },
363
- { id: 'you', key: 'youKey', available: (c) => !!c.youKey, search: you },
364
- { id: 'firecrawl', key: 'firecrawlKey', available: (c) => !!c.firecrawlKey, search: firecrawl },
365
- { id: 'tinyfish', key: 'tinyfishKey', available: (c) => !!c.tinyfishKey, search: tinyfish },
366
- ];
358
+ const maxResults = (opts: SearchOptions): number => opts.maxResults ?? DEFAULT_MAX_RESULTS;
367
359
 
368
360
  function parseEngineList(raw: string | undefined): EngineName[] | undefined {
369
361
  if (!raw) return undefined;
@@ -371,7 +363,7 @@ function parseEngineList(raw: string | undefined): EngineName[] | undefined {
371
363
  for (const s of raw.split(',')) {
372
364
  const t = s.trim().toLowerCase();
373
365
  if (!t) continue;
374
- if (ENGINE_REGISTRY.some((e) => e.id === t)) out.push(t as EngineName);
366
+ if (Object.hasOwn(ENGINES, t)) out.push(t as EngineName);
375
367
  }
376
368
  return out.length ? out : undefined;
377
369
  }
@@ -383,22 +375,14 @@ export function clearDeadEngines(): void {
383
375
  deadEngines.clear();
384
376
  }
385
377
 
386
- function selectEngines(opts: SearchOptions): EngineDef[] {
387
- const cfg = getConfig();
388
- const wanted = opts.engines ?? parseEngineList(cfg.searchEngines);
389
- const selected = ENGINE_REGISTRY.filter((def) => {
390
- if (!def.available(cfg)) return false;
391
- return wanted ? wanted.includes(def.id) : true;
392
- });
393
- return selected;
394
- }
395
-
396
378
  export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]> {
397
379
  const cfg = getConfig();
398
- const selected = selectEngines(opts);
399
- const chosen = selected.filter((def) => !deadEngines.has(def.id));
380
+ const wanted = opts.engines ?? parseEngineList(cfg.searchEngines);
381
+ const selected = Object.entries(ENGINES).filter(
382
+ ([id, def]) => !!cfg[def.key] && (!wanted || wanted.includes(id as EngineName)),
383
+ );
384
+ const chosen = selected.filter(([id]) => !deadEngines.has(id));
400
385
  if (chosen.length === 0) {
401
- const wanted = opts.engines ?? parseEngineList(getConfig().searchEngines);
402
386
  // selected-but-dead means KEYS EXIST and the engines were temp-banned
403
387
  // (stable HTTP 401/402/403): the old wording lied about the cause.
404
388
  const error =
@@ -407,13 +391,19 @@ export async function runEngines(opts: SearchOptions): Promise<EngineOutcome[]>
407
391
  : wanted?.length
408
392
  ? `no configured engines match the requested set (${wanted.join(', ')})`
409
393
  : 'no search engine keys configured; run /websearch to add one';
410
- return [{ engine: 'none', hits: [], error, latencyMs: 0 }];
394
+ return [{ engine: 'none', hits: [], error }];
411
395
  }
412
396
  return Promise.all(
413
- chosen.map(async (def) => {
414
- const outcome = await def.search(cfg[def.key] as string, opts, opts.signal);
415
- if (outcome.error && STABLE_ENGINE_ERROR.test(outcome.error)) deadEngines.add(def.id);
416
- return outcome;
397
+ chosen.map(async ([id, def]) => {
398
+ let hits: SearchHit[] = [];
399
+ let error: string | undefined;
400
+ try {
401
+ hits = await def.search(cfg[def.key] as string, opts, opts.signal);
402
+ } catch (e) {
403
+ error = sanitizeError(e);
404
+ }
405
+ if (error && STABLE_ENGINE_ERROR.test(error)) deadEngines.add(id);
406
+ return { engine: id, hits, error };
417
407
  }),
418
408
  );
419
409
  }
package/src/fetch.ts CHANGED
@@ -36,7 +36,7 @@ function fetchErrorResult(details: FetchDetails): FetchResult {
36
36
  };
37
37
  }
38
38
 
39
- function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
39
+ export function makeFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: number[]): FetchResult {
40
40
  const want = [...new Set(sections.map((i) => Math.max(0, Math.min(res.sections.length - 1, i))))];
41
41
  const body = res.outline.length
42
42
  ? `indices - 0 - ${res.outline.length - 1} section${res.outline.length === 1 ? '' : 's'}\n${res.outline.map((h, i) => `- *${i}* ${h}`).join('\n')}\n\n`
@@ -116,7 +116,3 @@ function buildFetchResult(res: Awaited<ReturnType<typeof scrape>>, sections: num
116
116
  if (res.error) return fetchErrorResult(details);
117
117
  return { content: [{ type: 'text', text: clipped }], details, structuredContent: { ...details } };
118
118
  }
119
-
120
- export function makeFetchResult(res: Awaited<ReturnType<typeof scrape>>, params: { sections: number[] }): FetchResult {
121
- return buildFetchResult(res, params.sections);
122
- }