research-powerpack-mcp 3.6.0 ā 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/scraper.d.ts.map +1 -1
- package/dist/clients/scraper.js +6 -5
- package/dist/clients/scraper.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/tools/scrape.d.ts.map +1 -1
- package/dist/tools/scrape.js +25 -16
- package/dist/tools/scrape.js.map +1 -1
- package/dist/utils/logger.d.ts +12 -5
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/logger.js +22 -8
- package/dist/utils/logger.js.map +1 -1
- package/package.json +1 -1
- package/dist/config/env.d.ts +0 -75
- package/dist/config/env.d.ts.map +0 -1
- package/dist/config/env.js +0 -87
- package/dist/config/env.js.map +0 -1
- package/dist/config/yaml/tools-enhanced.yaml +0 -0
- package/dist/utils/mcp-logger.d.ts +0 -8
- package/dist/utils/mcp-logger.d.ts.map +0 -1
- package/dist/utils/mcp-logger.js +0 -14
- package/dist/utils/mcp-logger.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAI5B,UAAU,aAAa;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,OAAO,GAAG,YAAY,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,cAAc;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,eAAe,CAAC;CACzB;AAED,UAAU,iBAAiB;IACzB,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AAOD,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAA2B;gBAE9B,MAAM,CAAC,EAAE,MAAM;IAS3B;;;OAGG;IACG,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,UAAU,IAAsB,GAAG,OAAO,CAAC,cAAc,CAAC;IA0J/F;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;;OAGG;IACG,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,cAAc,CAAC;IA2DlG;;;OAGG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,KAAK,CAAC,cAAc,GAAG;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAa1H;;;OAGG;IACG,WAAW,CACf,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAO,EAClC,eAAe,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,GACpF,OAAO,CAAC,iBAAiB,CAAC;IAmE7B;;;OAGG;YACW,YAAY;CAqB3B"}
|
package/dist/clients/scraper.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { parseEnv, SCRAPER } from '../config/index.js';
|
|
7
7
|
import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
|
|
8
8
|
import { pMapSettled } from '../utils/concurrency.js';
|
|
9
|
+
import { mcpLog } from '../utils/logger.js';
|
|
9
10
|
// Status codes that indicate we should retry (no credit consumed)
|
|
10
11
|
const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504, 510]);
|
|
11
12
|
// Status codes that are permanent failures (don't retry)
|
|
@@ -113,7 +114,7 @@ export class ScraperClient {
|
|
|
113
114
|
};
|
|
114
115
|
if (attempt < maxRetries - 1) {
|
|
115
116
|
const delayMs = this.calculateBackoff(attempt);
|
|
116
|
-
|
|
117
|
+
mcpLog('warning', `${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`, 'scraper');
|
|
117
118
|
await sleep(delayMs);
|
|
118
119
|
continue;
|
|
119
120
|
}
|
|
@@ -122,7 +123,7 @@ export class ScraperClient {
|
|
|
122
123
|
lastError = classifyError({ status: response.status, message: content });
|
|
123
124
|
if (attempt < maxRetries - 1 && lastError.retryable) {
|
|
124
125
|
const delayMs = this.calculateBackoff(attempt);
|
|
125
|
-
|
|
126
|
+
mcpLog('warning', `Status ${response.status}. Retrying in ${delayMs}ms`, 'scraper');
|
|
126
127
|
await sleep(delayMs);
|
|
127
128
|
continue;
|
|
128
129
|
}
|
|
@@ -148,7 +149,7 @@ export class ScraperClient {
|
|
|
148
149
|
// Retryable error - continue if attempts remaining
|
|
149
150
|
if (attempt < maxRetries - 1) {
|
|
150
151
|
const delayMs = this.calculateBackoff(attempt);
|
|
151
|
-
|
|
152
|
+
mcpLog('warning', `${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`, 'scraper');
|
|
152
153
|
await sleep(delayMs);
|
|
153
154
|
continue;
|
|
154
155
|
}
|
|
@@ -194,7 +195,7 @@ export class ScraperClient {
|
|
|
194
195
|
// Success
|
|
195
196
|
if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {
|
|
196
197
|
if (attemptResults.length > 0) {
|
|
197
|
-
|
|
198
|
+
mcpLog('info', `Success with ${attempt.description} after ${attemptResults.length} fallback(s)`, 'scraper');
|
|
198
199
|
}
|
|
199
200
|
return result;
|
|
200
201
|
}
|
|
@@ -204,7 +205,7 @@ export class ScraperClient {
|
|
|
204
205
|
}
|
|
205
206
|
// Non-retryable errors - don't try other modes
|
|
206
207
|
if (result.error && !result.error.retryable) {
|
|
207
|
-
|
|
208
|
+
mcpLog('error', `Non-retryable error with ${attempt.description}: ${result.error.message}`, 'scraper');
|
|
208
209
|
return result;
|
|
209
210
|
}
|
|
210
211
|
// Collect failure reason and try next mode
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAwBtD,kEAAkE;AAClE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,yDAAyD;AACzD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW;QACnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9C,qBAAqB;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,gBAAgB,GAAG,EAAE;gBAC9B,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtD,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,kCAAkC;gBAClC,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,qCAAqC;gBAC9E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;oBACjD,SAAS;iBACV,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,GAAG,4BAA4B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7G,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,wEAAwE;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO;wBACL,OAAO,EAAE,sBAAsB;wBAC/B,UAAU,EAAE,GAAG;wBACf,OAAO;qBACR,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG;wBACtC,CAAC,CAAC,gDAAgD;wBAClD,CAAC,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpD,OAAO;wBACL,OAAO,EAAE,UAAU,QAAQ,EAAE;wBAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa;4BAC9E,OAAO,EAAE,QAAQ;4BACjB,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,SAAS,GAAG;wBACV,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB;wBACtF,OAAO,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;wBAC7C,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;qBAC5B,CAAC;oBAEF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,OAAO,CAAC,KAAK,CAAC,aAAa,QAAQ,CAAC,MAAM,eAAe,OAAO,GAAG,CAAC,IAAI,UAAU,iBAAiB,OAAO,IAAI,CAAC,CAAC;wBAChH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,OAAO,CAAC,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,iBAAiB,OAAO,IAAI,CAAC,CAAC;oBAC/E,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uBAAuB;gBACvB,OAAO;oBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;oBACtC,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,4CAA4C;gBAC5C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO;wBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;wBACtC,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,GAAG;wBACvC,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,mDAAmD;gBACnD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,OAAO,CAAC,KAAK,CAAC,aAAa,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,OAAO,IAAI,CAAC,CAAC;oBACvH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,OAAO,EAAE,uBAAuB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE;YAC/F,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,GAAG;YACxC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,uBAAuB,EAAE,SAAS,EAAE,KAAK,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,SAAS,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,QAAQ,GAAmF;YAC/F,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;YAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;YAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACpF,CAAC;QAEF,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAA0B,IAAI,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,gDAAgD;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG;gBACH,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;YAEH,UAAU,GAAG,MAAM,CAAC;YAEpB,UAAU;YACV,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACzE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,cAAc,CAAC,CAAC;gBAC5G,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,wCAAwC;YACxC,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC9B,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,+CAA+C;YAC/C,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC5C,OAAO,CAAC,KAAK,CAAC,sCAAsC,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpG,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,2CAA2C;YAC3C,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC7F,OAAO,CAAC,KAAK,CAAC,yBAAyB,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,4BAA4B,CAAC,CAAC;QAChH,CAAC;QAED,0EAA0E;QAC1E,MAAM,YAAY,GAAG,gBAAgB,QAAQ,CAAC,MAAM,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpG,OAAO;YACL,OAAO,EAAE,UAAU,YAAY,EAAE;YACjC,UAAU,EAAE,UAAU,EAAE,UAAU,IAAI,GAAG;YACzC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS,CAAC,mBAAmB;gBACnC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,OAAO,CAAC,KAAK,CAAC,wCAAwC,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,CAAC,CAAC;QAEvG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,OAAO,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,CAAC,CAAC;YAEvG,oFAAoF;YACpF,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,SAAS,EACT,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAC5C,EAAE,CACH,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE1C,oBAAoB;oBACpB,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;wBACxD,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,wCAAwC;oBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChG,OAAO,CAAC,KAAK,CAAC,sCAAsC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;oBAExE,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO,EAAE,+BAA+B,QAAQ,EAAE;wBAClD,UAAU,EAAE,GAAG;wBACf,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,CAAC;gBACH,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,aAAa,CAAC,CAAC;YAC5E,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAEvH,4DAA4D;YAC5D,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,kEAAkE;QAClE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAE1F,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,CAAC;YAED,0CAA0C;YAC1C,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnG,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAwB5C,kEAAkE;AAClE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,yDAAyD;AACzD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEzD,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW;QACnE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9C,qBAAqB;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,gBAAgB,GAAG,EAAE;gBAC9B,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtD,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,kCAAkC;gBAClC,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,qCAAqC;gBAC9E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;oBACjD,SAAS;iBACV,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,GAAG,4BAA4B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7G,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,wEAAwE;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO;wBACL,OAAO,EAAE,sBAAsB;wBAC/B,UAAU,EAAE,GAAG;wBACf,OAAO;qBACR,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG;wBACtC,CAAC,CAAC,gDAAgD;wBAClD,CAAC,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpD,OAAO;wBACL,OAAO,EAAE,UAAU,QAAQ,EAAE;wBAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa;4BAC9E,OAAO,EAAE,QAAQ;4BACjB,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,SAAS,GAAG;wBACV,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB;wBACtF,OAAO,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;wBAC7C,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;qBAC5B,CAAC;oBAEF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,MAAM,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,MAAM,eAAe,OAAO,GAAG,CAAC,IAAI,UAAU,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;wBACrH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,UAAU,QAAQ,CAAC,MAAM,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBACpF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uBAAuB;gBACvB,OAAO;oBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;oBACtC,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,4CAA4C;gBAC5C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO;wBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;wBACtC,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,GAAG;wBACvC,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,mDAAmD;gBACnD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC5H,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,OAAO,EAAE,uBAAuB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE;YAC/F,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,GAAG;YACxC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,uBAAuB,EAAE,SAAS,EAAE,KAAK,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,SAAS,CAAC;QAC/C,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC;IACxC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,QAAQ,GAAmF;YAC/F,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;YAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;YAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;SACpF,CAAC;QAEF,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAA0B,IAAI,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,gDAAgD;YAChD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG;gBACH,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,OAAO,EAAE,OAAO,CAAC,OAAO;aACzB,CAAC,CAAC;YAEH,UAAU,GAAG,MAAM,CAAC;YAEpB,UAAU;YACV,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACzE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,CAAC,MAAM,EAAE,gBAAgB,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,cAAc,EAAE,SAAS,CAAC,CAAC;gBAC9G,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,wCAAwC;YACxC,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC9B,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,+CAA+C;YAC/C,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC5C,MAAM,CAAC,OAAO,EAAE,4BAA4B,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;gBACvG,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,2CAA2C;YAC3C,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC7F,OAAO,CAAC,KAAK,CAAC,yBAAyB,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,UAAU,4BAA4B,CAAC,CAAC;QAChH,CAAC;QAED,0EAA0E;QAC1E,MAAM,YAAY,GAAG,gBAAgB,QAAQ,CAAC,MAAM,oBAAoB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpG,OAAO;YACL,OAAO,EAAE,UAAU,YAAY,EAAE;YACjC,UAAU,EAAE,UAAU,EAAE,UAAU,IAAI,GAAG;YACzC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS,CAAC,mBAAmB;gBACnC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACjE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,OAAO,CAAC,KAAK,CAAC,wCAAwC,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,CAAC,CAAC;QAEvG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,OAAO,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,CAAC,CAAC;YAEvG,oFAAoF;YACpF,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,SAAS,EACT,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAC5C,EAAE,CACH,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE1C,oBAAoB;oBACpB,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;wBACxD,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,wCAAwC;oBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChG,OAAO,CAAC,KAAK,CAAC,sCAAsC,GAAG,KAAK,QAAQ,EAAE,CAAC,CAAC;oBAExE,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO,EAAE,+BAA+B,QAAQ,EAAE;wBAClD,UAAU,EAAE,GAAG;wBACf,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,CAAC;gBACH,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,aAAa,CAAC,CAAC;YAC5E,CAAC;YAED,OAAO,CAAC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,CAAC,CAAC;YAEvH,4DAA4D;YAC5D,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,kEAAkE;QAClE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAE1F,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,CAAC;YAED,0CAA0C;YAC1C,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnG,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
package/dist/index.js
CHANGED
|
@@ -10,7 +10,7 @@ import { TOOLS } from './tools/definitions.js';
|
|
|
10
10
|
import { executeTool, getToolCapabilities } from './tools/registry.js';
|
|
11
11
|
import { classifyError, createToolErrorFromStructured } from './utils/errors.js';
|
|
12
12
|
import { SERVER, getCapabilities } from './config/index.js';
|
|
13
|
-
import { initLogger } from './utils/
|
|
13
|
+
import { initLogger } from './utils/logger.js';
|
|
14
14
|
// ============================================================================
|
|
15
15
|
// Capability Detection (uses registry for tool capability mapping)
|
|
16
16
|
// ============================================================================
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,oCAAoC,CAAC;AAE7G,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,oCAAoC,CAAC;AAE7G,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,6BAA6B,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,+EAA+E;AAC/E,mEAAmE;AACnE,+EAA+E;AAE/E,MAAM,YAAY,GAAG,eAAe,EAAE,CAAC;AACvC,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,mBAAmB,EAAE,CAAC;AAEjF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;IAC5B,OAAO,CAAC,KAAK,CAAC,oBAAoB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAC/D,CAAC;AACD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;IAC7B,OAAO,CAAC,KAAK,CAAC,oCAAoC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAChF,CAAC;AACD,IAAI,YAAY,CAAC,QAAQ,IAAI,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC;IACzD,OAAO,CAAC,KAAK,CAAC,sFAAsF,CAAC,CAAC;AACxG,CAAC;AAED,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,EAC9C,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,CAC7C,CAAC;AAEF,UAAU,CAAC,MAAM,CAAC,CAAC;AAEnB,MAAM,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;AAEjF;;;GAGG;AACH,MAAM,CAAC,iBAAiB,CAAC,qBAAqB,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;IAChE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAEjD,IAAI,CAAC;QACH,4DAA4D;QAC5D,OAAO,MAAM,WAAW,CAAC,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,CAAC;IACrD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,kDAAkD;QAClD,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;YAC9B,MAAM,KAAK,CAAC;QACd,CAAC;QAED,0CAA0C;QAC1C,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,CAAC,KAAK,CAAC,sBAAsB,IAAI,UAAU,EAAE;YAClD,IAAI,EAAE,eAAe,CAAC,IAAI;YAC1B,OAAO,EAAE,eAAe,CAAC,OAAO;YAChC,SAAS,EAAE,eAAe,CAAC,SAAS;SACrC,CAAC,CAAC;QACH,OAAO,6BAA6B,CAAC,eAAe,CAAC,CAAC;IACxD,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,+EAA+E;AAC/E,+EAA+E;AAC/E,qFAAqF;AACrF,+EAA+E;AAE/E,kDAAkD;AAClD,IAAI,cAAc,GAAG,KAAK,CAAC;AAE3B;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IAC9C,IAAI,cAAc;QAAE,OAAO;IAC3B,cAAc,GAAG,IAAI,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,KAAK,CAAC,iCAAiC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,UAAU,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;IAClE,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzB,CAAC;AACH,CAAC;AAED,0DAA0D;AAC1D,0DAA0D;AAC1D,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAY,EAAE,EAAE;IAC/C,OAAO,CAAC,KAAK,CAAC,2CAA2C,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACtF,OAAO,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACzC,gBAAgB,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,uEAAuE;AACvE,0DAA0D;AAC1D,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAe,EAAE,EAAE;IACnD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,CAAC,KAAK,CAAC,4CAA4C,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IACvF,OAAO,CAAC,KAAK,CAAC,cAAc,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,KAAK,CAAC,WAAW,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACvC,gBAAgB,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,4DAA4D;AAC5D,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;IACzB,OAAO,CAAC,KAAK,CAAC,oCAAoC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC;IACxG,gBAAgB,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,wEAAwE;AACxE,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE;IAC1B,OAAO,CAAC,KAAK,CAAC,mCAAmC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC;IACvG,gBAAgB,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,+EAA+E;AAC/E,6BAA6B;AAC7B,8EAA8E;AAC9E,2EAA2E;AAC3E,6EAA6E;AAC7E,oEAAoE;AACpE,+EAA+E;AAE/E,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;IAC7B,OAAO,CAAC,KAAK,CAAC,sDAAsD,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;IAC/G,gBAAgB,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;IAC3B,OAAO,CAAC,KAAK,CAAC,qDAAqD,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;IAC9G,gBAAgB,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,8DAA8D;AAC9D,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;IACxD,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;QAChE,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;QAC/F,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC,CAAC;AAEH,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;AAE7C,8BAA8B;AAC9B,IAAI,CAAC;IACH,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,OAAO,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACjC,OAAO,CAAC,KAAK,CAAC,iCAAiC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/tools/scrape.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/tools/scrape.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAiCvF;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,iBAAiB,CAAA;CAAE,CAAC,CAoNpE"}
|
package/dist/tools/scrape.js
CHANGED
|
@@ -9,6 +9,7 @@ import { removeMetaTags } from '../utils/markdown-formatter.js';
|
|
|
9
9
|
import { SCRAPER } from '../config/index.js';
|
|
10
10
|
import { getToolConfig } from '../config/loader.js';
|
|
11
11
|
import { classifyError } from '../utils/errors.js';
|
|
12
|
+
import { pMap } from '../utils/concurrency.js';
|
|
12
13
|
import { mcpLog, formatSuccess, formatError, formatBatchHeader, formatDuration, TOKEN_BUDGETS, calculateTokenAllocation, } from './utils.js';
|
|
13
14
|
// Module-level singleton - MarkdownCleaner is stateless
|
|
14
15
|
const markdownCleaner = new MarkdownCleaner();
|
|
@@ -95,7 +96,7 @@ export async function handleScrapeLinks(params) {
|
|
|
95
96
|
failed++;
|
|
96
97
|
contents.push(`## ${invalidUrl}\n\nā Invalid URL format`);
|
|
97
98
|
}
|
|
98
|
-
|
|
99
|
+
const successItems = [];
|
|
99
100
|
for (let i = 0; i < results.length; i++) {
|
|
100
101
|
const result = results[i];
|
|
101
102
|
if (!result) {
|
|
@@ -115,37 +116,45 @@ export async function handleScrapeLinks(params) {
|
|
|
115
116
|
// Success case
|
|
116
117
|
successful++;
|
|
117
118
|
totalCredits += result.credits;
|
|
118
|
-
// Process content safely
|
|
119
|
+
// Process content safely (CPU-bound, fast)
|
|
119
120
|
let content;
|
|
120
121
|
try {
|
|
121
122
|
content = markdownCleaner.processContent(result.content);
|
|
122
123
|
}
|
|
123
124
|
catch {
|
|
124
|
-
// If markdown cleaning fails, use raw content
|
|
125
125
|
content = result.content;
|
|
126
126
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
127
|
+
successItems.push({ url: result.url, content, index: i });
|
|
128
|
+
}
|
|
129
|
+
// Pass 2: Parallel LLM extraction for successful results (I/O-bound)
|
|
130
|
+
if (params.use_llm && llmProcessor && successItems.length > 0) {
|
|
131
|
+
mcpLog('info', `Starting parallel LLM extraction for ${successItems.length} pages (concurrency: 3)`, 'scrape');
|
|
132
|
+
const llmResults = await pMap(successItems, async (item) => {
|
|
133
|
+
mcpLog('debug', `LLM extracting ${item.url} (${tokensPerUrl} tokens)...`, 'scrape');
|
|
134
|
+
const llmResult = await processContentWithLLM(item.content, { use_llm: params.use_llm, what_to_extract: enhancedInstruction, max_tokens: tokensPerUrl }, llmProcessor);
|
|
131
135
|
if (llmResult.processed) {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
}
|
|
135
|
-
else {
|
|
136
|
-
llmErrors++;
|
|
137
|
-
mcpLog('warning', `[${i + 1}/${results.length}] LLM extraction skipped: ${llmResult.error || 'unknown reason'}`, 'scrape');
|
|
138
|
-
// Continue with original content - graceful degradation
|
|
136
|
+
mcpLog('debug', `LLM extraction complete for ${item.url}`, 'scrape');
|
|
137
|
+
return { ...item, content: llmResult.content };
|
|
139
138
|
}
|
|
139
|
+
llmErrors++;
|
|
140
|
+
mcpLog('warning', `LLM extraction skipped for ${item.url}: ${llmResult.error || 'unknown reason'}`, 'scrape');
|
|
141
|
+
return item; // Graceful degradation ā use original cleaned content
|
|
142
|
+
}, 3);
|
|
143
|
+
// Update successItems with LLM-processed content
|
|
144
|
+
for (let i = 0; i < llmResults.length; i++) {
|
|
145
|
+
successItems[i] = llmResults[i];
|
|
140
146
|
}
|
|
141
|
-
|
|
147
|
+
}
|
|
148
|
+
// Pass 3: Final assembly ā remove meta tags and build content entries
|
|
149
|
+
for (const item of successItems) {
|
|
150
|
+
let content = item.content;
|
|
142
151
|
try {
|
|
143
152
|
content = removeMetaTags(content);
|
|
144
153
|
}
|
|
145
154
|
catch {
|
|
146
155
|
// If this fails, just use the content as-is
|
|
147
156
|
}
|
|
148
|
-
contents.push(`## ${
|
|
157
|
+
contents.push(`## ${item.url}\n\n${content}`);
|
|
149
158
|
}
|
|
150
159
|
const executionTime = Date.now() - startTime;
|
|
151
160
|
mcpLog('info', `Completed: ${successful} successful, ${failed} failed, ${totalCredits} credits used`, 'scrape');
|
package/dist/tools/scrape.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/tools/scrape.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EACL,MAAM,EACN,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAEpB,wDAAwD;AACxD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;AAE9C,8EAA8E;AAC9E,SAAS,mBAAmB;IAC1B,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IAC7C,OAAO,MAAM,EAAE,MAAM,EAAE,iBAA2B,IAAI,OAAO,CAAC,iBAAiB,CAAC;AAClF,CAAC;AAED,SAAS,4BAA4B,CAAC,WAA+B;IACnE,MAAM,IAAI,GAAG,WAAW,IAAI,8DAA8D,CAAC;IAC3F,OAAO,GAAG,IAAI,OAAO,mBAAmB,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAyB;IAEzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,kCAAkC;IAClC,MAAM,mBAAmB,GAAG,CAAC,IAAY,EAAE,OAAe,EAAE,SAAS,GAAG,KAAK,EAA6D,EAAE,CAAC,CAAC;QAC5I,OAAO,EAAE,WAAW,CAAC;YACnB,IAAI;YACJ,OAAO;YACP,SAAS;YACT,QAAQ,EAAE,cAAc;YACxB,QAAQ,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,CAAC,SAAS;SAC9E,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE;gBACR,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;gBAChC,aAAa,EAAE,CAAC;gBAChB,iBAAiB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAC1C;SACF;KACF,CAAC,CAAC;IAEH,kBAAkB;IAClB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,OAAO,mBAAmB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAC5D,CAAC;IAED,gCAAgC;IAChC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACb,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,mBAAmB,CAAC,cAAc,EAAE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,mBAAmB,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,YAAY,GAAG,wBAAwB,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEtE,MAAM,CAAC,MAAM,EAAE,oBAAoB,SAAS,CAAC,MAAM,YAAY,YAAY,gBAAgB,YAAY,YAAY,EAAE,QAAQ,CAAC,CAAC;IAE/H,4BAA4B;IAC5B,IAAI,MAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACjC,OAAO,mBAAmB,CAAC,oBAAoB,EAAE,iCAAiC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAC,CAAC,iCAAiC;IAE5E,MAAM,mBAAmB,GAAG,MAAM,CAAC,OAAO;QACxC,CAAC,CAAC,4BAA4B,CAAC,MAAM,CAAC,eAAe,CAAC;QACtD,CAAC,CAAC,SAAS,CAAC;IAEd,4CAA4C;IAC5C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAEpF,MAAM,CAAC,MAAM,EAAE,iCAAiC,OAAO,CAAC,MAAM,aAAa,EAAE,QAAQ,CAAC,CAAC;IAEvF,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,mCAAmC;IACnC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC;QACT,QAAQ,CAAC,IAAI,CAAC,MAAM,UAAU,0BAA0B,CAAC,CAAC;IAC5D,CAAC;
|
|
1
|
+
{"version":3,"file":"scrape.js","sourceRoot":"","sources":["../../src/tools/scrape.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACzF,OAAO,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EACL,MAAM,EACN,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,aAAa,EACb,wBAAwB,GACzB,MAAM,YAAY,CAAC;AAEpB,wDAAwD;AACxD,MAAM,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;AAE9C,8EAA8E;AAC9E,SAAS,mBAAmB;IAC1B,MAAM,MAAM,GAAG,aAAa,CAAC,cAAc,CAAC,CAAC;IAC7C,OAAO,MAAM,EAAE,MAAM,EAAE,iBAA2B,IAAI,OAAO,CAAC,iBAAiB,CAAC;AAClF,CAAC;AAED,SAAS,4BAA4B,CAAC,WAA+B;IACnE,MAAM,IAAI,GAAG,WAAW,IAAI,8DAA8D,CAAC;IAC3F,OAAO,GAAG,IAAI,OAAO,mBAAmB,EAAE,EAAE,CAAC;AAC/C,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAyB;IAEzB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,kCAAkC;IAClC,MAAM,mBAAmB,GAAG,CAAC,IAAY,EAAE,OAAe,EAAE,SAAS,GAAG,KAAK,EAA6D,EAAE,CAAC,CAAC;QAC5I,OAAO,EAAE,WAAW,CAAC;YACnB,IAAI;YACJ,OAAO;YACP,SAAS;YACT,QAAQ,EAAE,cAAc;YACxB,QAAQ,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,CAAC,SAAS;SAC9E,CAAC;QACF,iBAAiB,EAAE;YACjB,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE;gBACR,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;gBAChC,aAAa,EAAE,CAAC;gBAChB,iBAAiB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAC1C;SACF;KACF,CAAC,CAAC;IAEH,kBAAkB;IAClB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,OAAO,mBAAmB,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;IAC5D,CAAC;IAED,gCAAgC;IAChC,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACb,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,mBAAmB,CAAC,cAAc,EAAE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,mBAAmB,CAAC,CAAC;IAC3F,CAAC;IAED,MAAM,YAAY,GAAG,wBAAwB,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAEtE,MAAM,CAAC,MAAM,EAAE,oBAAoB,SAAS,CAAC,MAAM,YAAY,YAAY,gBAAgB,YAAY,YAAY,EAAE,QAAQ,CAAC,CAAC;IAE/H,4BAA4B;IAC5B,IAAI,MAAqB,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACjC,OAAO,mBAAmB,CAAC,oBAAoB,EAAE,iCAAiC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACnG,CAAC;IAED,MAAM,YAAY,GAAG,kBAAkB,EAAE,CAAC,CAAC,iCAAiC;IAE5E,MAAM,mBAAmB,GAAG,MAAM,CAAC,OAAO;QACxC,CAAC,CAAC,4BAA4B,CAAC,MAAM,CAAC,eAAe,CAAC;QACtD,CAAC,CAAC,SAAS,CAAC;IAEd,4CAA4C;IAC5C,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IAEpF,MAAM,CAAC,MAAM,EAAE,iCAAiC,OAAO,CAAC,MAAM,aAAa,EAAE,QAAQ,CAAC,CAAC;IAEvF,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,mCAAmC;IACnC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC;QACT,QAAQ,CAAC,IAAI,CAAC,MAAM,UAAU,0BAA0B,CAAC,CAAC;IAC5D,CAAC;IAQD,MAAM,YAAY,GAAsB,EAAE,CAAC;IAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QAED,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;QAEnF,6BAA6B;QAC7B,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,EAAE,CAAC;YACxE,MAAM,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,OAAO,IAAI,QAAQ,MAAM,CAAC,UAAU,EAAE,CAAC;YACxF,QAAQ,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,GAAG,2BAA2B,QAAQ,EAAE,CAAC,CAAC;YACrE,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,aAAa,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC;YAChF,SAAS;QACX,CAAC;QAED,eAAe;QACf,UAAU,EAAE,CAAC;QACb,YAAY,IAAI,MAAM,CAAC,OAAO,CAAC;QAE/B,2CAA2C;QAC3C,IAAI,OAAe,CAAC;QACpB,IAAI,CAAC;YACH,OAAO,GAAG,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC3B,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,qEAAqE;IACrE,IAAI,MAAM,CAAC,OAAO,IAAI,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,MAAM,EAAE,wCAAwC,YAAY,CAAC,MAAM,yBAAyB,EAAE,QAAQ,CAAC,CAAC;QAE/G,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACzD,MAAM,CAAC,OAAO,EAAE,kBAAkB,IAAI,CAAC,GAAG,KAAK,YAAY,aAAa,EAAE,QAAQ,CAAC,CAAC;YAEpF,MAAM,SAAS,GAAG,MAAM,qBAAqB,CAC3C,IAAI,CAAC,OAAO,EACZ,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,UAAU,EAAE,YAAY,EAAE,EAC3F,YAAY,CACb,CAAC;YAEF,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;gBACxB,MAAM,CAAC,OAAO,EAAE,+BAA+B,IAAI,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,CAAC;gBACrE,OAAO,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC;YACjD,CAAC;YAED,SAAS,EAAE,CAAC;YACZ,MAAM,CAAC,SAAS,EAAE,8BAA8B,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAAE,EAAE,QAAQ,CAAC,CAAC;YAC9G,OAAO,IAAI,CAAC,CAAC,sDAAsD;QACrE,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,iDAAiD;QACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,YAAY,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC3B,IAAI,CAAC;YACH,OAAO,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,4CAA4C;QAC9C,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,OAAO,OAAO,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAE7C,MAAM,CAAC,MAAM,EAAE,cAAc,UAAU,gBAAgB,MAAM,YAAY,YAAY,eAAe,EAAE,QAAQ,CAAC,CAAC;IAEhH,0BAA0B;IAC1B,MAAM,WAAW,GAAG,iBAAiB,CAAC;QACpC,KAAK,EAAE,oBAAoB,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ;QACrD,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC9B,UAAU;QACV,MAAM;QACN,aAAa,EAAE,YAAY;QAC3B,OAAO,EAAE,YAAY;QACrB,MAAM,EAAE;YACN,cAAc,EAAE,YAAY;YAC5B,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,yBAAyB,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnE;KACF,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG;QAChB,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,4HAA4H,CAAC,CAAC,CAAC,IAAI;QACpJ,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,6EAA6E,CAAC,CAAC,CAAC,IAAI;QACjG,wFAAwF;KACzF,CAAC,MAAM,CAAC,OAAO,CAAa,CAAC;IAE9B,MAAM,gBAAgB,GAAG,aAAa,CAAC;QACrC,KAAK,EAAE,mBAAmB;QAC1B,OAAO,EAAE,WAAW;QACpB,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC;QAClC,SAAS;QACT,QAAQ,EAAE;YACR,gBAAgB,EAAE,cAAc,CAAC,aAAa,CAAC;YAC/C,cAAc,EAAE,aAAa,CAAC,OAAO,CAAC,cAAc,EAAE;SACvD;KACF,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG;QACf,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;QAC9B,UAAU;QACV,MAAM;QACN,aAAa,EAAE,YAAY;QAC3B,iBAAiB,EAAE,aAAa;QAChC,cAAc,EAAE,YAAY;QAC5B,kBAAkB,EAAE,aAAa,CAAC,OAAO;QACzC,iBAAiB,EAAE,YAAY;KAChC,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,EAAE,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,CAAC;AACnG,CAAC"}
|
package/dist/utils/logger.d.ts
CHANGED
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* MCP
|
|
2
|
+
* MCP SDK Structured Logging Utility
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Sends log messages to MCP clients via server.sendLoggingMessage().
|
|
5
|
+
* Falls back to stderr before the server is initialized.
|
|
6
6
|
*/
|
|
7
|
+
import type { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
7
8
|
export type LogLevel = 'debug' | 'info' | 'warning' | 'error';
|
|
8
9
|
/**
|
|
9
|
-
*
|
|
10
|
+
* Initialize the logger with an MCP server reference.
|
|
11
|
+
* Must be called after server creation and before any tool execution.
|
|
12
|
+
*/
|
|
13
|
+
export declare function initLogger(server: Server): void;
|
|
14
|
+
/**
|
|
15
|
+
* Log message via MCP SDK structured logging.
|
|
16
|
+
* Falls back to stderr if the server isn't initialized yet.
|
|
10
17
|
* @param level - Log level
|
|
11
18
|
* @param message - Message to log
|
|
12
|
-
* @param tool - Tool name for context
|
|
19
|
+
* @param tool - Tool/logger name for context
|
|
13
20
|
*/
|
|
14
21
|
export declare function mcpLog(level: LogLevel, message: string, tool?: string): void;
|
|
15
22
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAExE,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC;AAI9D;;;GAGG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAE/C;AAED;;;;;;GAMG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAQ5E;AAED;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAM7E;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM;iBAE5B,MAAM;gBACP,MAAM;mBACH,MAAM;iBACR,MAAM;EAEtB;AAED,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC"}
|
package/dist/utils/logger.js
CHANGED
|
@@ -1,19 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* MCP
|
|
2
|
+
* MCP SDK Structured Logging Utility
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Sends log messages to MCP clients via server.sendLoggingMessage().
|
|
5
|
+
* Falls back to stderr before the server is initialized.
|
|
6
6
|
*/
|
|
7
|
+
let serverRef = null;
|
|
7
8
|
/**
|
|
8
|
-
*
|
|
9
|
+
* Initialize the logger with an MCP server reference.
|
|
10
|
+
* Must be called after server creation and before any tool execution.
|
|
11
|
+
*/
|
|
12
|
+
export function initLogger(server) {
|
|
13
|
+
serverRef = server;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Log message via MCP SDK structured logging.
|
|
17
|
+
* Falls back to stderr if the server isn't initialized yet.
|
|
9
18
|
* @param level - Log level
|
|
10
19
|
* @param message - Message to log
|
|
11
|
-
* @param tool - Tool name for context
|
|
20
|
+
* @param tool - Tool/logger name for context
|
|
12
21
|
*/
|
|
13
22
|
export function mcpLog(level, message, tool) {
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
|
|
23
|
+
const logger = tool ?? 'research-powerpack';
|
|
24
|
+
if (serverRef) {
|
|
25
|
+
serverRef.sendLoggingMessage({ level, data: message, logger }).catch(() => { });
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
// Fallback to stderr before MCP transport is connected
|
|
29
|
+
console.error(`[${logger}] ${message}`);
|
|
30
|
+
}
|
|
17
31
|
}
|
|
18
32
|
/**
|
|
19
33
|
* Safe log that catches any errors (never crashes)
|
package/dist/utils/logger.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,IAAI,SAAS,GAAkB,IAAI,CAAC;AAEpC;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,SAAS,GAAG,MAAM,CAAC;AACrB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,MAAM,CAAC,KAAe,EAAE,OAAe,EAAE,IAAa;IACpE,MAAM,MAAM,GAAG,IAAI,IAAI,oBAAoB,CAAC;IAC5C,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACjF,CAAC;SAAM,CAAC;QACN,uDAAuD;QACvD,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,KAAe,EAAE,OAAe,EAAE,IAAa;IACrE,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,uCAAuC;IACzC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO;QACL,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC;QACnD,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC;QACjD,OAAO,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,CAAC;QACvD,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC;KACpD,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "research-powerpack-mcp",
|
|
3
|
-
"version": "3.6.
|
|
3
|
+
"version": "3.6.1",
|
|
4
4
|
"description": "The ultimate research MCP toolkit: Reddit mining, web search with CTR aggregation, AI-powered deep research, and intelligent web scraping - all in one modular package",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
package/dist/config/env.d.ts
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Centralized Environment Variable Validation
|
|
3
|
-
* Uses Zod for type-safe parsing with helpful error messages
|
|
4
|
-
*/
|
|
5
|
-
import { z } from 'zod';
|
|
6
|
-
declare const envSchema: z.ZodObject<{
|
|
7
|
-
SCRAPEDO_API_KEY: z.ZodOptional<z.ZodString>;
|
|
8
|
-
SERPER_API_KEY: z.ZodOptional<z.ZodString>;
|
|
9
|
-
REDDIT_CLIENT_ID: z.ZodOptional<z.ZodString>;
|
|
10
|
-
REDDIT_CLIENT_SECRET: z.ZodOptional<z.ZodString>;
|
|
11
|
-
OPENROUTER_API_KEY: z.ZodOptional<z.ZodString>;
|
|
12
|
-
OPENROUTER_BASE_URL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
13
|
-
RESEARCH_MODEL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
14
|
-
RESEARCH_FALLBACK_MODEL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
15
|
-
DEFAULT_REASONING_EFFORT: z.ZodDefault<z.ZodOptional<z.ZodEnum<["low", "medium", "high"]>>>;
|
|
16
|
-
DEFAULT_MAX_URLS: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
17
|
-
API_TIMEOUT_MS: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
|
|
18
|
-
LLM_EXTRACTION_MODEL: z.ZodDefault<z.ZodOptional<z.ZodString>>;
|
|
19
|
-
LLM_ENABLE_REASONING: z.ZodDefault<z.ZodOptional<z.ZodEffects<z.ZodString, boolean, string>>>;
|
|
20
|
-
}, "strip", z.ZodTypeAny, {
|
|
21
|
-
OPENROUTER_BASE_URL: string;
|
|
22
|
-
RESEARCH_MODEL: string;
|
|
23
|
-
RESEARCH_FALLBACK_MODEL: string;
|
|
24
|
-
API_TIMEOUT_MS: number;
|
|
25
|
-
DEFAULT_REASONING_EFFORT: "low" | "medium" | "high";
|
|
26
|
-
DEFAULT_MAX_URLS: number;
|
|
27
|
-
LLM_EXTRACTION_MODEL: string;
|
|
28
|
-
LLM_ENABLE_REASONING: boolean;
|
|
29
|
-
SCRAPEDO_API_KEY?: string | undefined;
|
|
30
|
-
SERPER_API_KEY?: string | undefined;
|
|
31
|
-
REDDIT_CLIENT_ID?: string | undefined;
|
|
32
|
-
REDDIT_CLIENT_SECRET?: string | undefined;
|
|
33
|
-
OPENROUTER_API_KEY?: string | undefined;
|
|
34
|
-
}, {
|
|
35
|
-
SCRAPEDO_API_KEY?: string | undefined;
|
|
36
|
-
SERPER_API_KEY?: string | undefined;
|
|
37
|
-
REDDIT_CLIENT_ID?: string | undefined;
|
|
38
|
-
REDDIT_CLIENT_SECRET?: string | undefined;
|
|
39
|
-
OPENROUTER_BASE_URL?: string | undefined;
|
|
40
|
-
RESEARCH_MODEL?: string | undefined;
|
|
41
|
-
RESEARCH_FALLBACK_MODEL?: string | undefined;
|
|
42
|
-
OPENROUTER_API_KEY?: string | undefined;
|
|
43
|
-
API_TIMEOUT_MS?: number | undefined;
|
|
44
|
-
DEFAULT_REASONING_EFFORT?: "low" | "medium" | "high" | undefined;
|
|
45
|
-
DEFAULT_MAX_URLS?: number | undefined;
|
|
46
|
-
LLM_EXTRACTION_MODEL?: string | undefined;
|
|
47
|
-
LLM_ENABLE_REASONING?: string | undefined;
|
|
48
|
-
}>;
|
|
49
|
-
export type EnvConfig = z.infer<typeof envSchema>;
|
|
50
|
-
/**
|
|
51
|
-
* Parse and validate environment variables
|
|
52
|
-
* Returns cached result on subsequent calls
|
|
53
|
-
*/
|
|
54
|
-
export declare function getEnv(): EnvConfig;
|
|
55
|
-
/**
|
|
56
|
-
* Clear cached environment (useful for testing)
|
|
57
|
-
*/
|
|
58
|
-
export declare function clearEnvCache(): void;
|
|
59
|
-
export interface Capabilities {
|
|
60
|
-
reddit: boolean;
|
|
61
|
-
search: boolean;
|
|
62
|
-
scraping: boolean;
|
|
63
|
-
deepResearch: boolean;
|
|
64
|
-
llmExtraction: boolean;
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Get available capabilities based on configured API keys
|
|
68
|
-
*/
|
|
69
|
-
export declare function getCapabilitiesFromEnv(): Capabilities;
|
|
70
|
-
/**
|
|
71
|
-
* Get user-friendly message for missing capability
|
|
72
|
-
*/
|
|
73
|
-
export declare function getMissingCapabilityMessage(capability: keyof Capabilities): string;
|
|
74
|
-
export {};
|
|
75
|
-
//# sourceMappingURL=env.d.ts.map
|
package/dist/config/env.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/config/env.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB,QAAA,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBb,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAQlD;;;GAGG;AACH,wBAAgB,MAAM,IAAI,SAAS,CAelC;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,IAAI,CAEpC;AAMD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,YAAY,EAAE,OAAO,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED;;GAEG;AACH,wBAAgB,sBAAsB,IAAI,YAAY,CASrD;AAED;;GAEG;AACH,wBAAgB,2BAA2B,CAAC,UAAU,EAAE,MAAM,YAAY,GAAG,MAAM,CASlF"}
|
package/dist/config/env.js
DELETED
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Centralized Environment Variable Validation
|
|
3
|
-
* Uses Zod for type-safe parsing with helpful error messages
|
|
4
|
-
*/
|
|
5
|
-
import { z } from 'zod';
|
|
6
|
-
// ============================================================================
|
|
7
|
-
// Environment Schema
|
|
8
|
-
// ============================================================================
|
|
9
|
-
const envSchema = z.object({
|
|
10
|
-
// Scraper API (required for scrape_links)
|
|
11
|
-
SCRAPEDO_API_KEY: z.string().optional(),
|
|
12
|
-
// Search API (required for web_search and search_reddit)
|
|
13
|
-
SERPER_API_KEY: z.string().optional(),
|
|
14
|
-
// Reddit API (required for get_reddit_post)
|
|
15
|
-
REDDIT_CLIENT_ID: z.string().optional(),
|
|
16
|
-
REDDIT_CLIENT_SECRET: z.string().optional(),
|
|
17
|
-
// OpenRouter API (required for deep_research and LLM extraction)
|
|
18
|
-
OPENROUTER_API_KEY: z.string().optional(),
|
|
19
|
-
OPENROUTER_BASE_URL: z.string().url().optional().default('https://openrouter.ai/api/v1'),
|
|
20
|
-
// Research configuration
|
|
21
|
-
RESEARCH_MODEL: z.string().optional().default('x-ai/grok-4-fast'),
|
|
22
|
-
RESEARCH_FALLBACK_MODEL: z.string().optional().default('google/gemini-2.5-flash'),
|
|
23
|
-
DEFAULT_REASONING_EFFORT: z.enum(['low', 'medium', 'high']).optional().default('high'),
|
|
24
|
-
DEFAULT_MAX_URLS: z.coerce.number().min(10).max(200).optional().default(100),
|
|
25
|
-
API_TIMEOUT_MS: z.coerce.number().min(1000).max(3600000).optional().default(1800000),
|
|
26
|
-
// LLM extraction configuration
|
|
27
|
-
LLM_EXTRACTION_MODEL: z.string().optional().default('openai/gpt-oss-120b:nitro'),
|
|
28
|
-
LLM_ENABLE_REASONING: z.string().transform(v => v !== 'false').optional().default('true'),
|
|
29
|
-
});
|
|
30
|
-
// ============================================================================
|
|
31
|
-
// Parse and Validate Environment
|
|
32
|
-
// ============================================================================
|
|
33
|
-
let cachedEnv = null;
|
|
34
|
-
/**
|
|
35
|
-
* Parse and validate environment variables
|
|
36
|
-
* Returns cached result on subsequent calls
|
|
37
|
-
*/
|
|
38
|
-
export function getEnv() {
|
|
39
|
-
if (cachedEnv)
|
|
40
|
-
return cachedEnv;
|
|
41
|
-
try {
|
|
42
|
-
cachedEnv = envSchema.parse(process.env);
|
|
43
|
-
return cachedEnv;
|
|
44
|
-
}
|
|
45
|
-
catch (error) {
|
|
46
|
-
if (error instanceof z.ZodError) {
|
|
47
|
-
const issues = error.issues.map(i => ` - ${i.path.join('.')}: ${i.message}`).join('\n');
|
|
48
|
-
console.error(`[Config] Environment validation failed:\n${issues}`);
|
|
49
|
-
}
|
|
50
|
-
// Return defaults on error to avoid crashing
|
|
51
|
-
cachedEnv = envSchema.parse({});
|
|
52
|
-
return cachedEnv;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Clear cached environment (useful for testing)
|
|
57
|
-
*/
|
|
58
|
-
export function clearEnvCache() {
|
|
59
|
-
cachedEnv = null;
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* Get available capabilities based on configured API keys
|
|
63
|
-
*/
|
|
64
|
-
export function getCapabilitiesFromEnv() {
|
|
65
|
-
const env = getEnv();
|
|
66
|
-
return {
|
|
67
|
-
reddit: !!(env.REDDIT_CLIENT_ID && env.REDDIT_CLIENT_SECRET),
|
|
68
|
-
search: !!env.SERPER_API_KEY,
|
|
69
|
-
scraping: !!env.SCRAPEDO_API_KEY,
|
|
70
|
-
deepResearch: !!env.OPENROUTER_API_KEY,
|
|
71
|
-
llmExtraction: !!env.OPENROUTER_API_KEY,
|
|
72
|
-
};
|
|
73
|
-
}
|
|
74
|
-
/**
|
|
75
|
-
* Get user-friendly message for missing capability
|
|
76
|
-
*/
|
|
77
|
-
export function getMissingCapabilityMessage(capability) {
|
|
78
|
-
const messages = {
|
|
79
|
-
reddit: 'ā **Reddit tools unavailable.** Set `REDDIT_CLIENT_ID` and `REDDIT_CLIENT_SECRET` to enable.\n\nš Create a Reddit app at: https://www.reddit.com/prefs/apps (select "script" type)',
|
|
80
|
-
search: 'ā **Search unavailable.** Set `SERPER_API_KEY` to enable web search and Reddit search.\n\nš Get your free API key at: https://serper.dev (2,500 free queries)',
|
|
81
|
-
scraping: 'ā **Web scraping unavailable.** Set `SCRAPEDO_API_KEY` to enable URL content extraction.\n\nš Sign up at: https://scrape.do (1,000 free credits)',
|
|
82
|
-
deepResearch: 'ā **Deep research unavailable.** Set `OPENROUTER_API_KEY` to enable AI-powered research.\n\nš Get your API key at: https://openrouter.ai/keys',
|
|
83
|
-
llmExtraction: 'ā ļø **AI extraction disabled.** The `use_llm` and `what_to_extract` features require `OPENROUTER_API_KEY`.\n\nScraping will work but without intelligent content filtering.',
|
|
84
|
-
};
|
|
85
|
-
return messages[capability];
|
|
86
|
-
}
|
|
87
|
-
//# sourceMappingURL=env.js.map
|
package/dist/config/env.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../src/config/env.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,+EAA+E;AAC/E,qBAAqB;AACrB,+EAA+E;AAE/E,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IACzB,0CAA0C;IAC1C,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAEvC,yDAAyD;IACzD,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAErC,4CAA4C;IAC5C,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAE3C,iEAAiE;IACjE,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzC,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,8BAA8B,CAAC;IAExF,yBAAyB;IACzB,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC;IACjE,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC;IACjF,wBAAwB,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;IACtF,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;IAC5E,cAAc,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IAEpF,+BAA+B;IAC/B,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,2BAA2B,CAAC;IAChF,oBAAoB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;CAC1F,CAAC,CAAC;AAIH,+EAA+E;AAC/E,iCAAiC;AACjC,+EAA+E;AAE/E,IAAI,SAAS,GAAqB,IAAI,CAAC;AAEvC;;;GAGG;AACH,MAAM,UAAU,MAAM;IACpB,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,IAAI,CAAC;QACH,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACzC,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzF,OAAO,CAAC,KAAK,CAAC,4CAA4C,MAAM,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,6CAA6C;QAC7C,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa;IAC3B,SAAS,GAAG,IAAI,CAAC;AACnB,CAAC;AAcD;;GAEG;AACH,MAAM,UAAU,sBAAsB;IACpC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;IACrB,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,oBAAoB,CAAC;QAC5D,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,cAAc;QAC5B,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,gBAAgB;QAChC,YAAY,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB;QACtC,aAAa,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB;KACxC,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,2BAA2B,CAAC,UAA8B;IACxE,MAAM,QAAQ,GAAuC;QACnD,MAAM,EAAE,qLAAqL;QAC7L,MAAM,EAAE,gKAAgK;QACxK,QAAQ,EAAE,mJAAmJ;QAC7J,YAAY,EAAE,gJAAgJ;QAC9J,aAAa,EAAE,4KAA4K;KAC5L,CAAC;IACF,OAAO,QAAQ,CAAC,UAAU,CAAC,CAAC;AAC9B,CAAC"}
|
|
File without changes
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* MCP SDK structured logging wrapper
|
|
3
|
-
* Sends log messages to MCP clients via server.sendLoggingMessage()
|
|
4
|
-
*/
|
|
5
|
-
import type { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
6
|
-
export declare function initLogger(server: Server): void;
|
|
7
|
-
export declare function mcpLog(level: 'debug' | 'info' | 'warning' | 'error', message: string, logger?: string): void;
|
|
8
|
-
//# sourceMappingURL=mcp-logger.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-logger.d.ts","sourceRoot":"","sources":["../../src/utils/mcp-logger.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AAIxE,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAE/C;AAED,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAG5G"}
|
package/dist/utils/mcp-logger.js
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* MCP SDK structured logging wrapper
|
|
3
|
-
* Sends log messages to MCP clients via server.sendLoggingMessage()
|
|
4
|
-
*/
|
|
5
|
-
let serverRef = null;
|
|
6
|
-
export function initLogger(server) {
|
|
7
|
-
serverRef = server;
|
|
8
|
-
}
|
|
9
|
-
export function mcpLog(level, message, logger) {
|
|
10
|
-
if (!serverRef)
|
|
11
|
-
return;
|
|
12
|
-
serverRef.sendLoggingMessage({ level, data: message, logger: logger ?? 'research-powerpack' }).catch(() => { });
|
|
13
|
-
}
|
|
14
|
-
//# sourceMappingURL=mcp-logger.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"mcp-logger.js","sourceRoot":"","sources":["../../src/utils/mcp-logger.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,IAAI,SAAS,GAAkB,IAAI,CAAC;AAEpC,MAAM,UAAU,UAAU,CAAC,MAAc;IACvC,SAAS,GAAG,MAAM,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,KAA6C,EAAE,OAAe,EAAE,MAAe;IACpG,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,SAAS,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,oBAAoB,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;AACjH,CAAC"}
|