research-powerpack-mcp 3.0.14 ā 3.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/reddit.d.ts +9 -1
- package/dist/clients/reddit.d.ts.map +1 -1
- package/dist/clients/reddit.js +121 -41
- package/dist/clients/reddit.js.map +1 -1
- package/dist/clients/research.d.ts +16 -0
- package/dist/clients/research.d.ts.map +1 -1
- package/dist/clients/research.js +135 -31
- package/dist/clients/research.js.map +1 -1
- package/dist/clients/scraper.d.ts +27 -1
- package/dist/clients/scraper.d.ts.map +1 -1
- package/dist/clients/scraper.js +206 -101
- package/dist/clients/scraper.js.map +1 -1
- package/dist/clients/search.d.ts +15 -0
- package/dist/clients/search.d.ts.map +1 -1
- package/dist/clients/search.js +174 -52
- package/dist/clients/search.js.map +1 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +51 -6
- package/dist/index.js.map +1 -1
- package/dist/services/llm-processor.d.ts +18 -0
- package/dist/services/llm-processor.d.ts.map +1 -1
- package/dist/services/llm-processor.js +174 -24
- package/dist/services/llm-processor.js.map +1 -1
- package/dist/tools/research.d.ts +5 -0
- package/dist/tools/research.d.ts.map +1 -1
- package/dist/tools/research.js +63 -18
- package/dist/tools/research.js.map +1 -1
- package/dist/tools/scrape.d.ts +5 -0
- package/dist/tools/scrape.d.ts.map +1 -1
- package/dist/tools/scrape.js +144 -81
- package/dist/tools/scrape.js.map +1 -1
- package/dist/utils/errors.d.ts +79 -1
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +253 -22
- package/dist/utils/errors.js.map +1 -1
- package/package.json +1 -1
package/dist/clients/search.js
CHANGED
|
@@ -1,8 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Web Search Client
|
|
3
3
|
* Generic interface for web search via Google (Serper implementation)
|
|
4
|
+
* Implements robust error handling that NEVER crashes
|
|
4
5
|
*/
|
|
5
6
|
import { parseEnv } from '../config/index.js';
|
|
7
|
+
import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
|
|
8
|
+
// Search retry configuration
|
|
9
|
+
const SEARCH_RETRY_CONFIG = {
|
|
10
|
+
maxRetries: 2,
|
|
11
|
+
baseDelayMs: 1000,
|
|
12
|
+
maxDelayMs: 10000,
|
|
13
|
+
timeoutMs: 30000,
|
|
14
|
+
};
|
|
15
|
+
const RETRYABLE_SEARCH_CODES = new Set([429, 500, 502, 503, 504]);
|
|
6
16
|
export class SearchClient {
|
|
7
17
|
apiKey;
|
|
8
18
|
baseURL = 'https://google.serper.dev';
|
|
@@ -13,79 +23,191 @@ export class SearchClient {
|
|
|
13
23
|
throw new Error('SERPER_API_KEY is required for search functionality');
|
|
14
24
|
}
|
|
15
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* Calculate backoff delay
|
|
28
|
+
*/
|
|
29
|
+
calculateBackoff(attempt) {
|
|
30
|
+
const exponentialDelay = SEARCH_RETRY_CONFIG.baseDelayMs * Math.pow(2, attempt);
|
|
31
|
+
const jitter = Math.random() * 0.3 * exponentialDelay;
|
|
32
|
+
return Math.min(exponentialDelay + jitter, SEARCH_RETRY_CONFIG.maxDelayMs);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Check if error is retryable
|
|
36
|
+
*/
|
|
37
|
+
isRetryable(status, error) {
|
|
38
|
+
if (status && RETRYABLE_SEARCH_CODES.has(status))
|
|
39
|
+
return true;
|
|
40
|
+
const message = error?.message?.toLowerCase() || '';
|
|
41
|
+
return message.includes('timeout') || message.includes('rate limit') || message.includes('connection');
|
|
42
|
+
}
|
|
16
43
|
/**
|
|
17
44
|
* Search multiple keywords in parallel
|
|
45
|
+
* NEVER throws - always returns a valid response
|
|
18
46
|
*/
|
|
19
47
|
async searchMultiple(keywords) {
|
|
20
48
|
const startTime = Date.now();
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
},
|
|
29
|
-
body: JSON.stringify(searchQueries),
|
|
30
|
-
});
|
|
31
|
-
if (!response.ok) {
|
|
32
|
-
const errorText = await response.text().catch(() => '');
|
|
33
|
-
throw new Error(`Search API error: ${response.status} ${errorText}`);
|
|
34
|
-
}
|
|
35
|
-
const data = await response.json();
|
|
36
|
-
const responses = Array.isArray(data) ? data : [data];
|
|
37
|
-
const searches = responses.map((resp, index) => {
|
|
38
|
-
const organic = (resp.organic || []);
|
|
39
|
-
const results = organic.map((item, idx) => ({
|
|
40
|
-
title: item.title || 'No title',
|
|
41
|
-
link: item.link || '#',
|
|
42
|
-
snippet: item.snippet || '',
|
|
43
|
-
date: item.date,
|
|
44
|
-
position: item.position || idx + 1,
|
|
45
|
-
}));
|
|
46
|
-
const searchInfo = resp.searchInformation;
|
|
47
|
-
const totalResults = searchInfo?.totalResults
|
|
48
|
-
? parseInt(String(searchInfo.totalResults).replace(/,/g, ''), 10)
|
|
49
|
-
: results.length;
|
|
50
|
-
const relatedSearches = (resp.relatedSearches || []);
|
|
51
|
-
const related = relatedSearches.map((r) => r.query || '');
|
|
52
|
-
return { keyword: keywords[index] || '', results, totalResults, related };
|
|
53
|
-
});
|
|
54
|
-
return { searches, totalKeywords: keywords.length, executionTime: Date.now() - startTime };
|
|
49
|
+
if (keywords.length === 0) {
|
|
50
|
+
return {
|
|
51
|
+
searches: [],
|
|
52
|
+
totalKeywords: 0,
|
|
53
|
+
executionTime: 0,
|
|
54
|
+
error: { code: ErrorCode.INVALID_INPUT, message: 'No keywords provided', retryable: false },
|
|
55
|
+
};
|
|
55
56
|
}
|
|
56
|
-
|
|
57
|
-
|
|
57
|
+
let lastError;
|
|
58
|
+
for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
|
|
59
|
+
try {
|
|
60
|
+
if (attempt > 0) {
|
|
61
|
+
console.error(`[Search] Retry attempt ${attempt}/${SEARCH_RETRY_CONFIG.maxRetries}`);
|
|
62
|
+
}
|
|
63
|
+
const searchQueries = keywords.map(keyword => ({ q: keyword }));
|
|
64
|
+
const response = await fetchWithTimeout(`${this.baseURL}/search`, {
|
|
65
|
+
method: 'POST',
|
|
66
|
+
headers: {
|
|
67
|
+
'X-API-KEY': this.apiKey,
|
|
68
|
+
'Content-Type': 'application/json',
|
|
69
|
+
},
|
|
70
|
+
body: JSON.stringify(searchQueries),
|
|
71
|
+
timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
|
|
72
|
+
});
|
|
73
|
+
if (!response.ok) {
|
|
74
|
+
const errorText = await response.text().catch(() => '');
|
|
75
|
+
lastError = classifyError({ status: response.status, message: errorText });
|
|
76
|
+
if (this.isRetryable(response.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
77
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
78
|
+
console.error(`[Search] API returned ${response.status}, retrying in ${delayMs}ms...`);
|
|
79
|
+
await sleep(delayMs);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
// Return partial result with error
|
|
83
|
+
return {
|
|
84
|
+
searches: [],
|
|
85
|
+
totalKeywords: keywords.length,
|
|
86
|
+
executionTime: Date.now() - startTime,
|
|
87
|
+
error: lastError,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
// Parse response safely
|
|
91
|
+
let data;
|
|
92
|
+
try {
|
|
93
|
+
data = await response.json();
|
|
94
|
+
}
|
|
95
|
+
catch (parseError) {
|
|
96
|
+
return {
|
|
97
|
+
searches: [],
|
|
98
|
+
totalKeywords: keywords.length,
|
|
99
|
+
executionTime: Date.now() - startTime,
|
|
100
|
+
error: { code: ErrorCode.PARSE_ERROR, message: 'Failed to parse search response', retryable: false },
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
const responses = Array.isArray(data) ? data : [data];
|
|
104
|
+
const searches = responses.map((resp, index) => {
|
|
105
|
+
try {
|
|
106
|
+
const organic = (resp.organic || []);
|
|
107
|
+
const results = organic.map((item, idx) => ({
|
|
108
|
+
title: item.title || 'No title',
|
|
109
|
+
link: item.link || '#',
|
|
110
|
+
snippet: item.snippet || '',
|
|
111
|
+
date: item.date,
|
|
112
|
+
position: item.position || idx + 1,
|
|
113
|
+
}));
|
|
114
|
+
const searchInfo = resp.searchInformation;
|
|
115
|
+
const totalResults = searchInfo?.totalResults
|
|
116
|
+
? parseInt(String(searchInfo.totalResults).replace(/,/g, ''), 10)
|
|
117
|
+
: results.length;
|
|
118
|
+
const relatedSearches = (resp.relatedSearches || []);
|
|
119
|
+
const related = relatedSearches.map((r) => r.query || '');
|
|
120
|
+
return { keyword: keywords[index] || '', results, totalResults, related };
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
// Return empty result for this keyword on parse error
|
|
124
|
+
return { keyword: keywords[index] || '', results: [], totalResults: 0, related: [] };
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
return { searches, totalKeywords: keywords.length, executionTime: Date.now() - startTime };
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
lastError = classifyError(error);
|
|
131
|
+
if (this.isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
132
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
133
|
+
console.error(`[Search] ${lastError.code}: ${lastError.message}, retrying in ${delayMs}ms...`);
|
|
134
|
+
await sleep(delayMs);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
58
139
|
}
|
|
140
|
+
// All retries failed
|
|
141
|
+
return {
|
|
142
|
+
searches: [],
|
|
143
|
+
totalKeywords: keywords.length,
|
|
144
|
+
executionTime: Date.now() - startTime,
|
|
145
|
+
error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'Search failed', retryable: false },
|
|
146
|
+
};
|
|
59
147
|
}
|
|
60
148
|
/**
|
|
61
149
|
* Search Reddit via Google (adds site:reddit.com automatically)
|
|
150
|
+
* NEVER throws - returns empty array on failure
|
|
62
151
|
*/
|
|
63
152
|
async searchReddit(query, dateAfter) {
|
|
153
|
+
if (!query?.trim()) {
|
|
154
|
+
return [];
|
|
155
|
+
}
|
|
64
156
|
let q = /site:\s*reddit\.com/i.test(query) ? query : `${query} site:reddit.com`;
|
|
65
157
|
if (dateAfter) {
|
|
66
158
|
q += ` after:${dateAfter}`;
|
|
67
159
|
}
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
160
|
+
for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
|
|
161
|
+
try {
|
|
162
|
+
const res = await fetchWithTimeout(`${this.baseURL}/search`, {
|
|
163
|
+
method: 'POST',
|
|
164
|
+
headers: { 'X-API-KEY': this.apiKey, 'Content-Type': 'application/json' },
|
|
165
|
+
body: JSON.stringify({ q, num: 10 }),
|
|
166
|
+
timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
|
|
167
|
+
});
|
|
168
|
+
if (!res.ok) {
|
|
169
|
+
if (this.isRetryable(res.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
170
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
171
|
+
console.error(`[Search Reddit] ${res.status}, retrying in ${delayMs}ms...`);
|
|
172
|
+
await sleep(delayMs);
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
console.error(`[Search Reddit] Failed with status ${res.status}`);
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
const data = await res.json();
|
|
179
|
+
return (data.organic || []).map((r) => ({
|
|
180
|
+
title: (r.title || '').replace(/ : r\/\w+$/, '').replace(/ - Reddit$/, ''),
|
|
181
|
+
url: r.link || '',
|
|
182
|
+
snippet: r.snippet || '',
|
|
183
|
+
date: r.date,
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
catch (error) {
|
|
187
|
+
const err = classifyError(error);
|
|
188
|
+
if (this.isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
189
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
190
|
+
console.error(`[Search Reddit] ${err.code}, retrying in ${delayMs}ms...`);
|
|
191
|
+
await sleep(delayMs);
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
console.error(`[Search Reddit] Failed: ${err.message}`);
|
|
195
|
+
return [];
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return [];
|
|
82
199
|
}
|
|
83
200
|
/**
|
|
84
201
|
* Search Reddit with multiple queries in parallel
|
|
202
|
+
* NEVER throws - uses Promise.allSettled pattern
|
|
85
203
|
*/
|
|
86
204
|
async searchRedditMultiple(queries, dateAfter) {
|
|
87
|
-
|
|
88
|
-
|
|
205
|
+
if (queries.length === 0) {
|
|
206
|
+
return new Map();
|
|
207
|
+
}
|
|
208
|
+
// All searchReddit calls never throw, so we can use Promise.all safely
|
|
209
|
+
const results = await Promise.all(queries.map(q => this.searchReddit(q, dateAfter)));
|
|
210
|
+
return new Map(queries.map((q, i) => [q, results[i] || []]));
|
|
89
211
|
}
|
|
90
212
|
}
|
|
91
213
|
//# sourceMappingURL=search.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAgC5B,6BAA6B;AAC7B,MAAM,mBAAmB,GAAG;IAC1B,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;CACR,CAAC;AAEX,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAElE,MAAM,OAAO,YAAY;IACf,MAAM,CAAS;IACf,OAAO,GAAG,2BAA2B,CAAC;IAE9C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,gBAAgB,CAAC;QACtD,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,MAAM,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,MAAe,EAAE,KAAe;QAClD,IAAI,MAAM,IAAI,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAE9D,MAAM,OAAO,GAAI,KAA8B,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC9E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACzG,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAAE,KAAK,EAAE;aAC5F,CAAC;QACJ,CAAC;QAED,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3E,IAAI,CAAC;gBACH,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,CAAC,CAAC;gBACvF,CAAC;gBAED,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;gBAEhE,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,SAAS,EAAE;oBAChE,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI,CAAC,MAAM;wBACxB,cAAc,EAAE,kBAAkB;qBACnC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;oBACnC,SAAS,EAAE,mBAAmB,CAAC,SAAS;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;oBACxD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;oBAE3E,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;wBAClF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,OAAO,CAAC,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,iBAAiB,OAAO,OAAO,CAAC,CAAC;wBACvF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;oBAED,mCAAmC;oBACnC,OAAO;wBACL,QAAQ,EAAE,EAAE;wBACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;wBAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACrC,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,wBAAwB;gBACxB,IAAI,IAAa,CAAC;gBAClB,IAAI,CAAC;oBACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC/B,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,OAAO;wBACL,QAAQ,EAAE,EAAE;wBACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;wBAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACrC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,KAAK,EAAE;qBACrG,CAAC;gBACJ,CAAC;gBAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEtD,MAAM,QAAQ,GAA0B,SAAS,CAAC,GAAG,CAAC,CAAC,IAA6B,EAAE,KAAa,EAAE,EAAE;oBACrG,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAmC,CAAC;wBACvE,MAAM,OAAO,GAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,IAA6B,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC;4BAC3F,KAAK,EAAG,IAAI,CAAC,KAAgB,IAAI,UAAU;4BAC3C,IAAI,EAAG,IAAI,CAAC,IAAe,IAAI,GAAG;4BAClC,OAAO,EAAG,IAAI,CAAC,OAAkB,IAAI,EAAE;4BACvC,IAAI,EAAE,IAAI,CAAC,IAA0B;4BACrC,QAAQ,EAAG,IAAI,CAAC,QAAmB,IAAI,GAAG,GAAG,CAAC;yBAC/C,CAAC,CAAC,CAAC;wBAEJ,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAwD,CAAC;wBACjF,MAAM,YAAY,GAAG,UAAU,EAAE,YAAY;4BAC3C,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;4BACjE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;wBAEnB,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAmC,CAAC;wBACvF,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAE,CAAC,CAAC,KAAgB,IAAI,EAAE,CAAC,CAAC;wBAE/F,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;oBAC5E,CAAC;oBAAC,MAAM,CAAC;wBACP,sDAAsD;wBACtD,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;oBACvF,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;YAE7F,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBACnF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,OAAO,CAAC,KAAK,CAAC,YAAY,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,iBAAiB,OAAO,OAAO,CAAC,CAAC;oBAC/F,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,MAAM;YACR,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACrC,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE;SAClG,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,SAAkB;QAClD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,GAAG,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,kBAAkB,CAAC;QAEhF,IAAI,SAAS,EAAE,CAAC;YACd,CAAC,IAAI,UAAU,SAAS,EAAE,CAAC;QAC7B,CAAC;QAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3E,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,SAAS,EAAE;oBAC3D,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBACzE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;oBACpC,SAAS,EAAE,mBAAmB,CAAC,SAAS;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,MAAM,iBAAiB,OAAO,OAAO,CAAC,CAAC;wBAC5E,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;oBACD,OAAO,CAAC,KAAK,CAAC,sCAAsC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;oBAClE,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA0F,CAAC;gBACtH,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC;oBAC1E,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE;oBACjB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;oBACxB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC,CAAC;YAEN,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBACnF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,CAAC,IAAI,iBAAiB,OAAO,OAAO,CAAC,CAAC;oBAC1E,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBACD,OAAO,CAAC,KAAK,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACxD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB,CAAC,OAAiB,EAAE,SAAkB;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,uEAAuE;QACvE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAClD,CAAC;QAEF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;GAGG"}
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Research Powerpack MCP Server
|
|
4
|
+
* Implements robust error handling - server NEVER crashes on tool failures
|
|
5
|
+
*/
|
|
2
6
|
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
3
7
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
4
8
|
import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
|
@@ -10,7 +14,7 @@ import { handleWebSearch } from './tools/search.js';
|
|
|
10
14
|
import { deepResearchParamsSchema } from './schemas/deep-research.js';
|
|
11
15
|
import { scrapeLinksParamsSchema } from './schemas/scrape-links.js';
|
|
12
16
|
import { webSearchParamsSchema } from './schemas/web-search.js';
|
|
13
|
-
import {
|
|
17
|
+
import { classifyError } from './utils/errors.js';
|
|
14
18
|
import { parseEnv, SERVER, getCapabilities, getMissingEnvMessage } from './config/index.js';
|
|
15
19
|
// ============================================================================
|
|
16
20
|
// Capability Detection (no ENV required - tools fail gracefully when called)
|
|
@@ -137,16 +141,57 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
137
141
|
return { content: [{ type: 'text', text: `ā ļø **Unknown tool:** \`${name}\`\n\nAvailable tools: search_reddit, get_reddit_post, deep_research, scrape_links, web_search` }], isError: true };
|
|
138
142
|
}
|
|
139
143
|
catch (error) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
144
|
+
// Classify the error for helpful messaging
|
|
145
|
+
const structuredError = classifyError(error);
|
|
146
|
+
// Log for debugging
|
|
147
|
+
console.error(`[MCP Server] Tool "${name}" error:`, {
|
|
148
|
+
code: structuredError.code,
|
|
149
|
+
message: structuredError.message,
|
|
150
|
+
retryable: structuredError.retryable,
|
|
151
|
+
});
|
|
152
|
+
// Format user-friendly error message
|
|
153
|
+
const retryHint = structuredError.retryable ? '\n\nš” This error may be temporary. Try again in a moment.' : '';
|
|
154
|
+
const errorText = `## ā Error\n\n**${structuredError.code}:** ${structuredError.message}${retryHint}\n\nPlease check your input parameters and try again.`;
|
|
155
|
+
// ALWAYS return a valid response - never let the server crash
|
|
143
156
|
return { content: [{ type: 'text', text: errorText }], isError: true };
|
|
144
157
|
}
|
|
145
158
|
});
|
|
146
159
|
// ============================================================================
|
|
160
|
+
// Global Error Handlers - Prevent server crashes
|
|
161
|
+
// ============================================================================
|
|
162
|
+
// Handle uncaught exceptions
|
|
163
|
+
process.on('uncaughtException', (error) => {
|
|
164
|
+
console.error('[MCP Server] Uncaught exception (server continues):', error.message);
|
|
165
|
+
// Don't exit - the server should continue running
|
|
166
|
+
});
|
|
167
|
+
// Handle unhandled promise rejections
|
|
168
|
+
process.on('unhandledRejection', (reason) => {
|
|
169
|
+
const error = classifyError(reason);
|
|
170
|
+
console.error('[MCP Server] Unhandled rejection (server continues):', error.message);
|
|
171
|
+
// Don't exit - the server should continue running
|
|
172
|
+
});
|
|
173
|
+
// Handle SIGTERM gracefully
|
|
174
|
+
process.on('SIGTERM', () => {
|
|
175
|
+
console.error('[MCP Server] Received SIGTERM, shutting down gracefully');
|
|
176
|
+
process.exit(0);
|
|
177
|
+
});
|
|
178
|
+
// Handle SIGINT gracefully (Ctrl+C)
|
|
179
|
+
process.on('SIGINT', () => {
|
|
180
|
+
console.error('[MCP Server] Received SIGINT, shutting down gracefully');
|
|
181
|
+
process.exit(0);
|
|
182
|
+
});
|
|
183
|
+
// ============================================================================
|
|
147
184
|
// Start Server
|
|
148
185
|
// ============================================================================
|
|
149
186
|
const transport = new StdioServerTransport();
|
|
150
|
-
|
|
151
|
-
|
|
187
|
+
// Connect with error handling
|
|
188
|
+
try {
|
|
189
|
+
server.connect(transport);
|
|
190
|
+
console.error(`š ${SERVER.NAME} v${SERVER.VERSION} ready`);
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
const err = classifyError(error);
|
|
194
|
+
console.error(`[MCP Server] Failed to start: ${err.message}`);
|
|
195
|
+
process.exit(1);
|
|
196
|
+
}
|
|
152
197
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAEnG,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAEnG,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,4BAA4B,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAmC,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,QAAQ,EAAY,MAAM,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEtG,+EAA+E;AAC/E,6EAA6E;AAC7E,+EAA+E;AAE/E,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;AACvB,MAAM,YAAY,GAAG,eAAe,EAAE,CAAC;AAEvC,2CAA2C;AAC3C,MAAM,YAAY,GAAa,EAAE,CAAC;AAClC,MAAM,aAAa,GAAa,EAAE,CAAC;AAEnC,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;IACxB,YAAY,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;AACnD,CAAC;KAAM,CAAC;IACN,aAAa,CAAC,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;AACpD,CAAC;AACD,IAAI,YAAY,CAAC,MAAM,EAAE,CAAC;IACxB,YAAY,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACvC,CAAC;KAAM,CAAC;IACN,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AACxC,CAAC;AACD,IAAI,YAAY,CAAC,QAAQ,EAAE,CAAC;IAC1B,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;AACpC,CAAC;KAAM,CAAC;IACN,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;AACrC,CAAC;AACD,IAAI,YAAY,CAAC,YAAY,EAAE,CAAC;IAC9B,YAAY,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AACrC,CAAC;KAAM,CAAC;IACN,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AACtC,CAAC;AAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;IAC5B,OAAO,CAAC,KAAK,CAAC,oBAAoB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC/D,CAAC;AACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;IAC7B,OAAO,CAAC,KAAK,CAAC,oCAAoC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AACD,IAAI,YAAY,CAAC,QAAQ,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;IACzD,OAAO,CAAC,KAAK,CAAC,sFAAsF,CAAC,CAAC;AACxG,CAAC;AAED,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,EAC9C,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAChC,CAAC;AAEF,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AAEjF,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjD,IAAI,CAAC;QACH,sCAAsC;QACtC,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,mBAAmB;YACnB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBACzB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC9F,CAAC;YACD,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAkD,CAAC;YACnF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACpD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qDAAqD,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACrH,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,OAAO,EAAE,GAAG,CAAC,cAAe,EAAE,UAAU,CAAC,CAAC;YAClF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QACvD,CAAC;QAED,wCAAwC;QACxC,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC/B,mBAAmB;YACnB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBACzB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC9F,CAAC;YACD,MAAM,EAAE,IAAI,EAAE,YAAY,GAAG,GAAG,EAAE,cAAc,GAAG,IAAI,EAAE,GAAG,IAA2E,CAAC;YACxI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,2DAA2D,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC3H,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,GAAG,CAAC,gBAAiB,EAAE,GAAG,CAAC,oBAAqB,EAAE,YAAY,EAAE;gBAC9G,aAAa,EAAE,cAAc;gBAC7B,mBAAmB,EAAE,YAAY,KAAK,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;aACrE,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QACvD,CAAC;QAED,sCAAsC;QACtC,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;YAC7B,mBAAmB;YACnB,IAAI,CAAC,YAAY,CAAC,YAAY,EAAE,CAAC;gBAC/B,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,cAAc,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACpG,CAAC;YACD,MAAM,eAAe,GAAG,wBAAwB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7D,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,kBAAkB,CAAC,eAAe,CAAC,CAAC;YACjF,IAAI,iBAAiB,IAAI,OAAO,iBAAiB,KAAK,QAAQ,IAAI,OAAO,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,KAAK,EAAE,CAAC;gBAC1H,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACvE,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACxD,CAAC;QAED,qCAAqC;QACrC,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;YAC5B,mBAAmB;YACnB,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,CAAC;gBAC3B,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAChG,CAAC;YACD,MAAM,eAAe,GAAG,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAE5D,kDAAkD;YAClD,IAAI,eAAe,CAAC,OAAO,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;gBAC3D,OAAO,CAAC,KAAK,CAAC,oGAAoG,CAAC,CAAC;gBACpH,eAAe,CAAC,OAAO,GAAG,KAAK,CAAC;YAClC,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,iBAAiB,CAAC,eAAe,CAAC,CAAC;YAChF,IAAI,iBAAiB,CAAC,QAAQ,CAAC,MAAM,KAAK,iBAAiB,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;gBAChF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACvE,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACxD,CAAC;QAED,mCAAmC;QACnC,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,mBAAmB;YACnB,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBACzB,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YAC9F,CAAC;YACD,MAAM,eAAe,GAAG,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC1D,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,CAAC;YAC9E,IAAI,iBAAiB,CAAC,QAAQ,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;YACvE,CAAC;YACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACxD,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0BAA0B,IAAI,gGAAgG,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC9L,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2CAA2C;QAC3C,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAE7C,oBAAoB;QACpB,OAAO,CAAC,KAAK,CAAC,sBAAsB,IAAI,UAAU,EAAE;YAClD,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,OAAO,EAAE,eAAe,CAAC,OAAO;YAChC,SAAS,EAAE,eAAe,CAAC,SAAS;SACrC,CAAC,CAAC;QAEH,qCAAqC;QACrC,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,4DAA4D,CAAC,CAAC,CAAC,EAAE,CAAC;QAChH,MAAM,SAAS,GAAG,mBAAmB,eAAe,CAAC,IAAI,OAAO,eAAe,CAAC,OAAO,GAAG,SAAS,uDAAuD,CAAC;QAE3J,8DAA8D;QAC9D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACzE,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,+EAA+E;AAC/E,iDAAiD;AACjD,+EAA+E;AAE/E,6BAA6B;AAC7B,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAY,EAAE,EAAE;IAC/C,OAAO,CAAC,KAAK,CAAC,qDAAqD,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACpF,kDAAkD;AACpD,CAAC,CAAC,CAAC;AAEH,sCAAsC;AACtC,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAe,EAAE,EAAE;IACnD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,CAAC,KAAK,CAAC,sDAAsD,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACrF,kDAAkD;AACpD,CAAC,CAAC,CAAC;AAEH,4BAA4B;AAC5B,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;IACzB,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;IACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,oCAAoC;AACpC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;IACxB,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;IACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;AAE7C,8BAA8B;AAC9B,IAAI,CAAC;IACH,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,OAAO,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* LLM Processor for content extraction
|
|
3
3
|
* Uses OpenRouter via OPENROUTER_API_KEY for AI-powered content filtering
|
|
4
|
+
* Implements robust retry logic and NEVER throws
|
|
4
5
|
*/
|
|
5
6
|
import OpenAI from 'openai';
|
|
7
|
+
import { type StructuredError } from '../utils/errors.js';
|
|
6
8
|
interface ProcessingConfig {
|
|
7
9
|
use_llm: boolean;
|
|
8
10
|
what_to_extract: string | undefined;
|
|
@@ -12,8 +14,24 @@ interface LLMResult {
|
|
|
12
14
|
content: string;
|
|
13
15
|
processed: boolean;
|
|
14
16
|
error?: string;
|
|
17
|
+
errorDetails?: StructuredError;
|
|
15
18
|
}
|
|
16
19
|
export declare function createLLMProcessor(): OpenAI | null;
|
|
20
|
+
/**
|
|
21
|
+
* Process content with LLM extraction
|
|
22
|
+
* NEVER throws - always returns a valid LLMResult
|
|
23
|
+
* Implements retry logic with exponential backoff for transient failures
|
|
24
|
+
*/
|
|
17
25
|
export declare function processContentWithLLM(content: string, config: ProcessingConfig, processor?: OpenAI | null): Promise<LLMResult>;
|
|
26
|
+
/**
|
|
27
|
+
* Process multiple contents with LLM in parallel with rate limiting
|
|
28
|
+
* NEVER throws - always returns results array
|
|
29
|
+
*/
|
|
30
|
+
export declare function processMultipleWithLLM(contents: Array<{
|
|
31
|
+
content: string;
|
|
32
|
+
url: string;
|
|
33
|
+
}>, config: ProcessingConfig, processor?: OpenAI | null, concurrency?: number): Promise<Array<LLMResult & {
|
|
34
|
+
url: string;
|
|
35
|
+
}>>;
|
|
18
36
|
export {};
|
|
19
37
|
//# sourceMappingURL=llm-processor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llm-processor.d.ts","sourceRoot":"","sources":["../../src/services/llm-processor.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"llm-processor.d.ts","sourceRoot":"","sources":["../../src/services/llm-processor.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAE5B,UAAU,gBAAgB;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,eAAe,CAAC;CAChC;AAmBD,wBAAgB,kBAAkB,IAAI,MAAM,GAAG,IAAI,CAYlD;AAqDD;;;;GAIG;AACH,wBAAsB,qBAAqB,CACzC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,GACxB,OAAO,CAAC,SAAS,CAAC,CAkHpB;AAED;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,EACjD,MAAM,EAAE,gBAAgB,EACxB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,EACzB,WAAW,SAAI,GACd,OAAO,CAAC,KAAK,CAAC,SAAS,GAAG;IAAE,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CA2B7C"}
|