research-powerpack-mcp 3.0.14 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/reddit.d.ts +9 -1
- package/dist/clients/reddit.d.ts.map +1 -1
- package/dist/clients/reddit.js +121 -41
- package/dist/clients/reddit.js.map +1 -1
- package/dist/clients/research.d.ts +16 -0
- package/dist/clients/research.d.ts.map +1 -1
- package/dist/clients/research.js +135 -31
- package/dist/clients/research.js.map +1 -1
- package/dist/clients/scraper.d.ts +27 -1
- package/dist/clients/scraper.d.ts.map +1 -1
- package/dist/clients/scraper.js +206 -101
- package/dist/clients/scraper.js.map +1 -1
- package/dist/clients/search.d.ts +15 -0
- package/dist/clients/search.d.ts.map +1 -1
- package/dist/clients/search.js +174 -52
- package/dist/clients/search.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +51 -6
- package/dist/index.js.map +1 -1
- package/dist/services/llm-processor.d.ts +18 -0
- package/dist/services/llm-processor.d.ts.map +1 -1
- package/dist/services/llm-processor.js +174 -24
- package/dist/services/llm-processor.js.map +1 -1
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -1
- package/dist/tools/research.js +63 -18
- package/dist/tools/research.js.map +1 -1
- package/dist/tools/scrape.d.ts +5 -0
- package/dist/tools/scrape.d.ts.map +1 -1
- package/dist/tools/scrape.js +144 -81
- package/dist/tools/scrape.js.map +1 -1
- package/dist/utils/errors.d.ts +79 -1
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +253 -22
- package/dist/utils/errors.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,OAAO,GAAG,YAAY,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AAOD,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA2B;gBAE9B,MAAM,CAAC,EAAE,MAAM;IAS3B;;;OAGG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,UAAU,IAAsB,GAAG,OAAO,CAAC,cAAc,CAAC;IA0J/F;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACG,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,cAAc,CAAC;IA2DlG;;;OAGG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAa1H;;;OAGG;IACG,WAAW,CACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,EAClC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,GACpF,OAAO,CAAC,iBAAiB,CAAC;IAiE7B;;;OAGG;YACW,YAAY;CAoB3B"}
|
package/dist/clients/scraper.js
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Web Scraper Client
|
|
3
3
|
* Generic interface for URL scraping with automatic fallback modes
|
|
4
|
+
* Implements robust error handling that NEVER crashes
|
|
4
5
|
*/
|
|
5
6
|
import { parseEnv, SCRAPER } from '../config/index.js';
|
|
7
|
+
import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
|
|
8
|
+
// Status codes that indicate we should retry (no credit consumed)
|
|
9
|
+
const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504, 510]);
|
|
10
|
+
// Status codes that are permanent failures (don't retry)
|
|
11
|
+
const PERMANENT_FAILURE_CODES = new Set([400, 401, 403]);
|
|
6
12
|
export class ScraperClient {
|
|
7
13
|
apiKey;
|
|
8
14
|
baseURL = 'https://api.scrape.do';
|
|
@@ -13,8 +19,25 @@ export class ScraperClient {
|
|
|
13
19
|
throw new Error('SCRAPEDO_API_KEY is required');
|
|
14
20
|
}
|
|
15
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Scrape a single URL with retry logic
|
|
24
|
+
* NEVER throws - always returns a ScrapeResponse (possibly with error)
|
|
25
|
+
*/
|
|
16
26
|
async scrape(request, maxRetries = SCRAPER.RETRY_COUNT) {
|
|
17
27
|
const { url, mode = 'basic', timeout = 30, country } = request;
|
|
28
|
+
const credits = mode === 'javascript' ? 5 : 1;
|
|
29
|
+
// Validate URL first
|
|
30
|
+
try {
|
|
31
|
+
new URL(url);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return {
|
|
35
|
+
content: `Invalid URL: ${url}`,
|
|
36
|
+
statusCode: 400,
|
|
37
|
+
credits: 0,
|
|
38
|
+
error: { code: ErrorCode.INVALID_INPUT, message: `Invalid URL: ${url}`, retryable: false },
|
|
39
|
+
};
|
|
40
|
+
}
|
|
18
41
|
const params = new URLSearchParams({
|
|
19
42
|
url: url,
|
|
20
43
|
token: this.apiKey,
|
|
@@ -27,22 +50,25 @@ export class ScraperClient {
|
|
|
27
50
|
params.append('geoCode', country.toUpperCase());
|
|
28
51
|
}
|
|
29
52
|
const apiUrl = `${this.baseURL}?${params.toString()}`;
|
|
30
|
-
|
|
31
|
-
// Scrape.do Status Code Handling (based on official docs)
|
|
32
|
-
// ============================================================
|
|
33
|
-
// RETRY (no credit consumed): 429, 502, 510
|
|
34
|
-
// NO RETRY: 404 (permanent), 401 (no credits), 400 (bad request)
|
|
35
|
-
// SUCCESS: 2xx (consumes credit)
|
|
36
|
-
// ============================================================
|
|
53
|
+
let lastError;
|
|
37
54
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
38
55
|
try {
|
|
39
|
-
|
|
56
|
+
// Use AbortController for timeout
|
|
57
|
+
const timeoutMs = (timeout + 10) * 1000; // Add 10s buffer over scrape timeout
|
|
58
|
+
const response = await fetchWithTimeout(apiUrl, {
|
|
40
59
|
method: 'GET',
|
|
41
60
|
headers: { Accept: 'text/html,application/json' },
|
|
61
|
+
timeoutMs,
|
|
42
62
|
});
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
63
|
+
// Safely read response body
|
|
64
|
+
let content;
|
|
65
|
+
try {
|
|
66
|
+
content = await response.text();
|
|
67
|
+
}
|
|
68
|
+
catch (readError) {
|
|
69
|
+
content = `Failed to read response: ${readError instanceof Error ? readError.message : String(readError)}`;
|
|
70
|
+
}
|
|
71
|
+
// SUCCESS: 2xx - Successful API call
|
|
46
72
|
if (response.ok) {
|
|
47
73
|
return {
|
|
48
74
|
content,
|
|
@@ -51,77 +77,102 @@ export class ScraperClient {
|
|
|
51
77
|
headers: Object.fromEntries(response.headers.entries()),
|
|
52
78
|
};
|
|
53
79
|
}
|
|
54
|
-
//
|
|
55
|
-
if (response.status ===
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}
|
|
62
|
-
throw new Error('Rate limited (429). Try fewer concurrent URLs or retry later.');
|
|
80
|
+
// 404 - Target not found (permanent, but not an error for our purposes)
|
|
81
|
+
if (response.status === 404) {
|
|
82
|
+
return {
|
|
83
|
+
content: '404 - Page not found',
|
|
84
|
+
statusCode: 404,
|
|
85
|
+
credits,
|
|
86
|
+
};
|
|
63
87
|
}
|
|
64
|
-
//
|
|
65
|
-
if (response.status
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
88
|
+
// Permanent failures - don't retry
|
|
89
|
+
if (PERMANENT_FAILURE_CODES.has(response.status)) {
|
|
90
|
+
const errorMsg = response.status === 401
|
|
91
|
+
? 'No credits remaining or subscription suspended'
|
|
92
|
+
: `Request failed with status ${response.status}`;
|
|
93
|
+
return {
|
|
94
|
+
content: `Error: ${errorMsg}`,
|
|
95
|
+
statusCode: response.status,
|
|
96
|
+
credits: 0,
|
|
97
|
+
error: {
|
|
98
|
+
code: response.status === 401 ? ErrorCode.AUTH_ERROR : ErrorCode.INVALID_INPUT,
|
|
99
|
+
message: errorMsg,
|
|
100
|
+
retryable: false,
|
|
101
|
+
statusCode: response.status,
|
|
102
|
+
},
|
|
103
|
+
};
|
|
73
104
|
}
|
|
74
|
-
//
|
|
75
|
-
if (response.status
|
|
105
|
+
// Retryable status codes
|
|
106
|
+
if (RETRYABLE_STATUS_CODES.has(response.status)) {
|
|
107
|
+
lastError = {
|
|
108
|
+
code: response.status === 429 ? ErrorCode.RATE_LIMITED : ErrorCode.SERVICE_UNAVAILABLE,
|
|
109
|
+
message: `Server returned ${response.status}`,
|
|
110
|
+
retryable: true,
|
|
111
|
+
statusCode: response.status,
|
|
112
|
+
};
|
|
76
113
|
if (attempt < maxRetries - 1) {
|
|
77
|
-
const delayMs =
|
|
78
|
-
console.error(`[Scraper]
|
|
79
|
-
await
|
|
114
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
115
|
+
console.error(`[Scraper] ${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`);
|
|
116
|
+
await sleep(delayMs);
|
|
80
117
|
continue;
|
|
81
118
|
}
|
|
82
|
-
throw new Error('Request canceled (510). Please try again.');
|
|
83
|
-
}
|
|
84
|
-
// NO RETRY: 404 - Target not found (consumes credit, permanent error)
|
|
85
|
-
if (response.status === 404) {
|
|
86
|
-
return { content: '404 - Target not found', statusCode: 404, credits };
|
|
87
|
-
}
|
|
88
|
-
// NO RETRY: 401 - No credits or subscription suspended (no credit consumed)
|
|
89
|
-
if (response.status === 401) {
|
|
90
|
-
throw new Error('No credits remaining or subscription suspended (401). Check your Scrape.do account.');
|
|
91
|
-
}
|
|
92
|
-
// NO RETRY: 400 - Bad request (may or may not consume credit, permanent error)
|
|
93
|
-
if (response.status === 400) {
|
|
94
|
-
throw new Error(`Bad request (400): ${content.substring(0, 200)}`);
|
|
95
119
|
}
|
|
96
|
-
// Other
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
120
|
+
// Other non-success status - treat as retryable
|
|
121
|
+
lastError = classifyError({ status: response.status, message: content });
|
|
122
|
+
if (attempt < maxRetries - 1 && lastError.retryable) {
|
|
123
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
124
|
+
console.error(`[Scraper] Status ${response.status}. Retrying in ${delayMs}ms`);
|
|
125
|
+
await sleep(delayMs);
|
|
101
126
|
continue;
|
|
102
127
|
}
|
|
103
|
-
|
|
128
|
+
// Final attempt failed
|
|
129
|
+
return {
|
|
130
|
+
content: `Error: ${lastError.message}`,
|
|
131
|
+
statusCode: response.status,
|
|
132
|
+
credits: 0,
|
|
133
|
+
error: lastError,
|
|
134
|
+
};
|
|
104
135
|
}
|
|
105
136
|
catch (error) {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
console.error(`[Scraper] Error: ${error.message}. Retry ${attempt + 1}/${maxRetries} after ${delayMs}ms`);
|
|
116
|
-
await this.delay(delayMs);
|
|
137
|
+
lastError = classifyError(error);
|
|
138
|
+
// Non-retryable errors - return immediately
|
|
139
|
+
if (!lastError.retryable) {
|
|
140
|
+
return {
|
|
141
|
+
content: `Error: ${lastError.message}`,
|
|
142
|
+
statusCode: lastError.statusCode || 500,
|
|
143
|
+
credits: 0,
|
|
144
|
+
error: lastError,
|
|
145
|
+
};
|
|
117
146
|
}
|
|
118
|
-
|
|
119
|
-
|
|
147
|
+
// Retryable error - continue if attempts remaining
|
|
148
|
+
if (attempt < maxRetries - 1) {
|
|
149
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
150
|
+
console.error(`[Scraper] ${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`);
|
|
151
|
+
await sleep(delayMs);
|
|
152
|
+
continue;
|
|
120
153
|
}
|
|
121
154
|
}
|
|
122
155
|
}
|
|
123
|
-
|
|
156
|
+
// All retries exhausted
|
|
157
|
+
return {
|
|
158
|
+
content: `Error: Failed after ${maxRetries} attempts. ${lastError?.message || 'Unknown error'}`,
|
|
159
|
+
statusCode: lastError?.statusCode || 500,
|
|
160
|
+
credits: 0,
|
|
161
|
+
error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'All retries exhausted', retryable: false },
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Calculate exponential backoff with jitter
|
|
166
|
+
*/
|
|
167
|
+
calculateBackoff(attempt) {
|
|
168
|
+
const baseDelay = SCRAPER.RETRY_DELAYS[attempt] || 8000;
|
|
169
|
+
const jitter = Math.random() * 0.3 * baseDelay;
|
|
170
|
+
return Math.floor(baseDelay + jitter);
|
|
124
171
|
}
|
|
172
|
+
/**
|
|
173
|
+
* Scrape with automatic fallback through different modes
|
|
174
|
+
* NEVER throws - always returns a ScrapeResponse
|
|
175
|
+
*/
|
|
125
176
|
async scrapeWithFallback(url, options = {}) {
|
|
126
177
|
const attempts = [
|
|
127
178
|
{ mode: 'basic', description: 'basic mode' },
|
|
@@ -129,41 +180,67 @@ export class ScraperClient {
|
|
|
129
180
|
{ mode: 'javascript', country: 'us', description: 'javascript + US geo-targeting' },
|
|
130
181
|
];
|
|
131
182
|
const attemptResults = [];
|
|
183
|
+
let lastResult = null;
|
|
132
184
|
for (const attempt of attempts) {
|
|
133
|
-
try
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
}
|
|
146
|
-
if (result.statusCode === 404) {
|
|
147
|
-
return { content: '404 - Page not found', statusCode: 404, credits: result.credits };
|
|
185
|
+
// scrape() never throws, so no try-catch needed
|
|
186
|
+
const result = await this.scrape({
|
|
187
|
+
url,
|
|
188
|
+
mode: attempt.mode,
|
|
189
|
+
timeout: options.timeout,
|
|
190
|
+
country: attempt.country,
|
|
191
|
+
});
|
|
192
|
+
lastResult = result;
|
|
193
|
+
// Success
|
|
194
|
+
if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {
|
|
195
|
+
if (attemptResults.length > 0) {
|
|
196
|
+
console.error(`[Scraper] Success with ${attempt.description} after ${attemptResults.length} fallback(s)`);
|
|
148
197
|
}
|
|
149
|
-
|
|
150
|
-
console.error(`[Scraper] Failed with ${attempt.description} (status: ${result.statusCode}), trying next fallback...`);
|
|
198
|
+
return result;
|
|
151
199
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
200
|
+
// 404 is a valid response, not an error
|
|
201
|
+
if (result.statusCode === 404) {
|
|
202
|
+
return result;
|
|
203
|
+
}
|
|
204
|
+
// Non-retryable errors - don't try other modes
|
|
205
|
+
if (result.error && !result.error.retryable) {
|
|
206
|
+
console.error(`[Scraper] Non-retryable error with ${attempt.description}: ${result.error.message}`);
|
|
207
|
+
return result;
|
|
156
208
|
}
|
|
209
|
+
// Collect failure reason and try next mode
|
|
210
|
+
attemptResults.push(`${attempt.description}: ${result.error?.message || result.statusCode}`);
|
|
211
|
+
console.error(`[Scraper] Failed with ${attempt.description} (${result.statusCode}), trying next fallback...`);
|
|
157
212
|
}
|
|
158
|
-
|
|
213
|
+
// All fallbacks exhausted - return last result with aggregated error info
|
|
214
|
+
const errorMessage = `Failed after ${attempts.length} fallback modes: ${attemptResults.join('; ')}`;
|
|
215
|
+
return {
|
|
216
|
+
content: `Error: ${errorMessage}`,
|
|
217
|
+
statusCode: lastResult?.statusCode || 500,
|
|
218
|
+
credits: 0,
|
|
219
|
+
error: {
|
|
220
|
+
code: ErrorCode.SERVICE_UNAVAILABLE,
|
|
221
|
+
message: errorMessage,
|
|
222
|
+
retryable: false,
|
|
223
|
+
},
|
|
224
|
+
};
|
|
159
225
|
}
|
|
226
|
+
/**
|
|
227
|
+
* Scrape multiple URLs with batching
|
|
228
|
+
* NEVER throws - always returns results array
|
|
229
|
+
*/
|
|
160
230
|
async scrapeMultiple(urls, options = {}) {
|
|
231
|
+
if (urls.length === 0) {
|
|
232
|
+
return [];
|
|
233
|
+
}
|
|
161
234
|
if (urls.length <= SCRAPER.BATCH_SIZE) {
|
|
162
235
|
return this.processBatch(urls, options);
|
|
163
236
|
}
|
|
164
237
|
const result = await this.batchScrape(urls, options);
|
|
165
238
|
return result.results;
|
|
166
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* Batch scrape with progress callback
|
|
242
|
+
* NEVER throws - uses Promise.allSettled internally
|
|
243
|
+
*/
|
|
167
244
|
async batchScrape(urls, options = {}, onBatchComplete) {
|
|
168
245
|
const totalBatches = Math.ceil(urls.length / SCRAPER.BATCH_SIZE);
|
|
169
246
|
const allResults = [];
|
|
@@ -174,40 +251,68 @@ export class ScraperClient {
|
|
|
174
251
|
const endIdx = Math.min(startIdx + SCRAPER.BATCH_SIZE, urls.length);
|
|
175
252
|
const batchUrls = urls.slice(startIdx, endIdx);
|
|
176
253
|
console.error(`[Scraper] Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`);
|
|
254
|
+
// Promise.allSettled never throws
|
|
177
255
|
const batchResults = await Promise.allSettled(batchUrls.map(url => this.scrapeWithFallback(url, options)));
|
|
178
256
|
for (let i = 0; i < batchResults.length; i++) {
|
|
179
257
|
const result = batchResults[i];
|
|
180
258
|
const url = batchUrls[i] || '';
|
|
181
259
|
if (result.status === 'fulfilled') {
|
|
182
|
-
|
|
260
|
+
const scrapeResult = result.value;
|
|
261
|
+
allResults.push({ ...scrapeResult, url });
|
|
262
|
+
// Track rate limits
|
|
263
|
+
if (scrapeResult.error?.code === ErrorCode.RATE_LIMITED) {
|
|
264
|
+
rateLimitHits++;
|
|
265
|
+
}
|
|
183
266
|
}
|
|
184
267
|
else {
|
|
268
|
+
// This shouldn't happen since scrapeWithFallback never throws,
|
|
269
|
+
// but handle it gracefully just in case
|
|
185
270
|
const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
271
|
+
console.error(`[Scraper] Unexpected rejection for ${url}: ${errorMsg}`);
|
|
272
|
+
allResults.push({
|
|
273
|
+
url,
|
|
274
|
+
content: `Error: Unexpected failure - ${errorMsg}`,
|
|
275
|
+
statusCode: 500,
|
|
276
|
+
credits: 0,
|
|
277
|
+
error: classifyError(result.reason),
|
|
278
|
+
});
|
|
190
279
|
}
|
|
191
280
|
}
|
|
192
|
-
|
|
281
|
+
// Safe callback invocation
|
|
282
|
+
try {
|
|
283
|
+
onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);
|
|
284
|
+
}
|
|
285
|
+
catch (callbackError) {
|
|
286
|
+
console.error(`[Scraper] onBatchComplete callback error:`, callbackError);
|
|
287
|
+
}
|
|
193
288
|
console.error(`[Scraper] Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`);
|
|
289
|
+
// Small delay between batches to avoid overwhelming the API
|
|
194
290
|
if (batchNum < totalBatches - 1) {
|
|
195
|
-
await
|
|
291
|
+
await sleep(500);
|
|
196
292
|
}
|
|
197
293
|
}
|
|
198
294
|
return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };
|
|
199
295
|
}
|
|
296
|
+
/**
|
|
297
|
+
* Process a single batch of URLs
|
|
298
|
+
* NEVER throws
|
|
299
|
+
*/
|
|
200
300
|
async processBatch(urls, options) {
|
|
201
301
|
const results = await Promise.allSettled(urls.map(url => this.scrapeWithFallback(url, options)));
|
|
202
302
|
return results.map((result, index) => {
|
|
303
|
+
const url = urls[index] || '';
|
|
203
304
|
if (result.status === 'fulfilled') {
|
|
204
|
-
return { ...result.value, url
|
|
305
|
+
return { ...result.value, url };
|
|
205
306
|
}
|
|
206
|
-
|
|
307
|
+
// Shouldn't happen, but handle gracefully
|
|
308
|
+
return {
|
|
309
|
+
url,
|
|
310
|
+
content: `Error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
|
|
311
|
+
statusCode: 500,
|
|
312
|
+
credits: 0,
|
|
313
|
+
error: classifyError(result.reason),
|
|
314
|
+
};
|
|
207
315
|
});
|
|
208
316
|
}
|
|
209
|
-
delay(ms) {
|
|
210
|
-
return new Promise(resolve => setTimeout(resolve, ms));
|
|
211
|
-
}
|
|
212
317
|
}
|
|
213
318
|
//# sourceMappingURL=scraper.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAuBvD,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW;QACnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAE/D,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QAEtD,+DAA+D;QAC/D,0DAA0D;QAC1D,+DAA+D;QAC/D,4CAA4C;QAC5C,iEAAiE;QACjE,iCAAiC;QACjC,+DAA+D;QAE/D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;oBACnC,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;iBAClD,CAAC,CAAC;gBAEH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACtC,MAAM,OAAO,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAE9C,uDAAuD;gBACvD,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,iDAAiD;gBACjD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;wBACtD,OAAO,CAAC,KAAK,CAAC,wDAAwD,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,OAAO,IAAI,CAAC,CAAC;wBACtH,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC1B,SAAS;oBACX,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;gBACnF,CAAC;gBAED,mDAAmD;gBACnD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;wBACtD,OAAO,CAAC,KAAK,CAAC,0DAA0D,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,OAAO,IAAI,CAAC,CAAC;wBACxH,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC1B,SAAS;oBACX,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;gBAC7D,CAAC;gBAED,oEAAoE;gBACpE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;wBACtD,OAAO,CAAC,KAAK,CAAC,4DAA4D,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,OAAO,IAAI,CAAC,CAAC;wBAC1H,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC1B,SAAS;oBACX,CAAC;oBACD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;gBAC/D,CAAC;gBAED,sEAAsE;gBACtE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO,EAAE,OAAO,EAAE,wBAAwB,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;gBACzE,CAAC;gBAED,4EAA4E;gBAC5E,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,qFAAqF,CAAC,CAAC;gBACzG,CAAC;gBAED,+EAA+E;gBAC/E,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,MAAM,IAAI,KAAK,CAAC,sBAAsB,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;gBACrE,CAAC;gBAED,wDAAwD;gBACxD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;oBACtD,OAAO,CAAC,KAAK,CAAC,gCAAgC,QAAQ,CAAC,MAAM,YAAY,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,OAAO,IAAI,CAAC,CAAC;oBACzH,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC1B,SAAS;gBACX,CAAC;gBAED,MAAM,IAAI,KAAK,CAAC,kBAAkB,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACpF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,KAAK,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CAAC,8BAA8B,UAAU,cAAc,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAClI,CAAC;gBAED,sCAAsC;gBACtC,IAAI,KAAK,YAAY,KAAK;oBACtB,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAChC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAChC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;oBAChD,MAAM,OAAO,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;oBACtD,OAAO,CAAC,KAAK,CAAC,oBAAoB,KAAK,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,OAAO,IAAI,CAAC,CAAC;oBAC1G,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,QAAQ,GAAmF;YAC/F,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;YAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;YAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACpF,CAAC;QAEF,MAAM,cAAc,GAAa,EAAE,CAAC;QAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;oBAC/B,GAAG;oBACH,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;iBACzB,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;oBACxD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC9B,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,oBAAoB,CAAC,CAAC;oBAClH,CAAC;oBACD,OAAO,MAAM,CAAC;gBAChB,CAAC;gBAED,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC9B,OAAO,EAAE,OAAO,EAAE,sBAAsB,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;gBACvF,CAAC;gBAED,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;gBACpE,OAAO,CAAC,KAAK,CAAC,yBAAyB,OAAO,CAAC,WAAW,aAAa,MAAM,CAAC,UAAU,4BAA4B,CAAC,CAAC;YACxH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnE,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,GAAG,EAAE,CAAC,CAAC;gBACtD,OAAO,CAAC,KAAK,CAAC,wBAAwB,OAAO,CAAC,WAAW,KAAK,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,sCAAsC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,OAAO,CAAC,KAAK,CAAC,wCAAwC,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,CAAC,CAAC;QAEvG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,OAAO,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,CAAC,CAAC;YAEvG,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAC5D,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;gBAC5C,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAEhG,IAAI,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;wBAClE,aAAa,EAAE,CAAC;oBAClB,CAAC;oBAED,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,QAAQ,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;gBACvF,CAAC;YACH,CAAC;YAED,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACjE,OAAO,CAAC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAEvH,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QAEjG,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACrD,CAAC;YACD,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACrG,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,EAAU;QACtB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IACzD,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAwB5B,kEAAkE;AAClE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,yDAAyD;AACzD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW;QACnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9C,qBAAqB;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,gBAAgB,GAAG,EAAE;gBAC9B,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtD,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,kCAAkC;gBAClC,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,qCAAqC;gBAC9E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;oBACjD,SAAS;iBACV,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,GAAG,4BAA4B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7G,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,wEAAwE;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO;wBACL,OAAO,EAAE,sBAAsB;wBAC/B,UAAU,EAAE,GAAG;wBACf,OAAO;qBACR,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG;wBACtC,CAAC,CAAC,gDAAgD;wBAClD,CAAC,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpD,OAAO;wBACL,OAAO,EAAE,UAAU,QAAQ,EAAE;wBAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa;4BAC9E,OAAO,EAAE,QAAQ;4BACjB,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,SAAS,GAAG;wBACV,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB;wBACtF,OAAO,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;wBAC7C,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;qBAC5B,CAAC;oBAEF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,eAAe,OAAO,GAAG,CAAC,IAAI,UAAU,iBAAiB,OAAO,IAAI,CAAC,CAAC;wBAChH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,iBAAiB,OAAO,IAAI,CAAC,CAAC;oBAC/E,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uBAAuB;gBACvB,OAAO;oBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;oBACtC,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,4CAA4C;gBAC5C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO;wBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;wBACtC,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,GAAG;wBACvC,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,mDAAmD;gBACnD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,OAAO,CAAC,KAAK,CAAC,aAAa,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,OAAO,IAAI,CAAC,CAAC;oBACvH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,OAAO,EAAE,uBAAuB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE;YAC/F,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,GAAG;YACxC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,uBAAuB,EAAE,SAAS,EAAE,KAAK,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,SAAS,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,QAAQ,GAAmF;YAC/F,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;YAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;YAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACpF,CAAC;QAEF,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAA0B,IAAI,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,gDAAgD;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG;gBACH,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;YAEH,UAAU,GAAG,MAAM,CAAC;YAEpB,UAAU;YACV,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACzE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,cAAc,CAAC,CAAC;gBAC5G,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,wCAAwC;YACxC,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC9B,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,+CAA+C;YAC/C,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC5C,OAAO,CAAC,KAAK,CAAC,sCAAsC,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpG,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,2CAA2C;YAC3C,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC7F,OAAO,CAAC,KAAK,CAAC,yBAAyB,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,4BAA4B,CAAC,CAAC;QAChH,CAAC;QAED,0EAA0E;QAC1E,MAAM,YAAY,GAAG,gBAAgB,QAAQ,CAAC,MAAM,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpG,OAAO;YACL,OAAO,EAAE,UAAU,YAAY,EAAE;YACjC,UAAU,EAAE,UAAU,EAAE,UAAU,IAAI,GAAG;YACzC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS,CAAC,mBAAmB;gBACnC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,OAAO,CAAC,KAAK,CAAC,wCAAwC,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,CAAC,CAAC;QAEvG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,OAAO,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,CAAC,CAAC;YAEvG,kCAAkC;YAClC,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAC5D,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE1C,oBAAoB;oBACpB,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;wBACxD,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,wCAAwC;oBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChG,OAAO,CAAC,KAAK,CAAC,sCAAsC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;oBAExE,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO,EAAE,+BAA+B,QAAQ,EAAE;wBAClD,UAAU,EAAE,GAAG;wBACf,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,CAAC;gBACH,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,aAAa,CAAC,CAAC;YAC5E,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAEvH,4DAA4D;YAC5D,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QAEjG,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,CAAC;YAED,0CAA0C;YAC1C,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnG,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
package/dist/clients/search.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Web Search Client
|
|
3
3
|
* Generic interface for web search via Google (Serper implementation)
|
|
4
|
+
* Implements robust error handling that NEVER crashes
|
|
4
5
|
*/
|
|
6
|
+
import { type StructuredError } from '../utils/errors.js';
|
|
5
7
|
export interface SearchResult {
|
|
6
8
|
title: string;
|
|
7
9
|
link: string;
|
|
@@ -14,11 +16,13 @@ export interface KeywordSearchResult {
|
|
|
14
16
|
results: SearchResult[];
|
|
15
17
|
totalResults: number;
|
|
16
18
|
related: string[];
|
|
19
|
+
error?: StructuredError;
|
|
17
20
|
}
|
|
18
21
|
export interface MultipleSearchResponse {
|
|
19
22
|
searches: KeywordSearchResult[];
|
|
20
23
|
totalKeywords: number;
|
|
21
24
|
executionTime: number;
|
|
25
|
+
error?: StructuredError;
|
|
22
26
|
}
|
|
23
27
|
export interface RedditSearchResult {
|
|
24
28
|
title: string;
|
|
@@ -30,16 +34,27 @@ export declare class SearchClient {
|
|
|
30
34
|
private apiKey;
|
|
31
35
|
private baseURL;
|
|
32
36
|
constructor(apiKey?: string);
|
|
37
|
+
/**
|
|
38
|
+
* Calculate backoff delay
|
|
39
|
+
*/
|
|
40
|
+
private calculateBackoff;
|
|
41
|
+
/**
|
|
42
|
+
* Check if error is retryable
|
|
43
|
+
*/
|
|
44
|
+
private isRetryable;
|
|
33
45
|
/**
|
|
34
46
|
* Search multiple keywords in parallel
|
|
47
|
+
* NEVER throws - always returns a valid response
|
|
35
48
|
*/
|
|
36
49
|
searchMultiple(keywords: string[]): Promise<MultipleSearchResponse>;
|
|
37
50
|
/**
|
|
38
51
|
* Search Reddit via Google (adds site:reddit.com automatically)
|
|
52
|
+
* NEVER throws - returns empty array on failure
|
|
39
53
|
*/
|
|
40
54
|
searchReddit(query: string, dateAfter?: string): Promise<RedditSearchResult[]>;
|
|
41
55
|
/**
|
|
42
56
|
* Search Reddit with multiple queries in parallel
|
|
57
|
+
* NEVER throws - uses Promise.allSettled pattern
|
|
43
58
|
*/
|
|
44
59
|
searchRedditMultiple(queries: string[], dateAfter?: string): Promise<Map<string, RedditSearchResult[]>>;
|
|
45
60
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAYD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA+B;gBAElC,MAAM,CAAC,EAAE,MAAM;IAS3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;OAEG;IACH,OAAO,CAAC,WAAW;IAOnB;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAsHzE;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAuDpF;;;OAGG;IACG,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;CAY9G"}
|