rrcrawl 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/dist/config.js +1 -1
- package/dist/config.js.map +1 -1
- package/dist/http.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/providers/firecrawl.d.ts +1 -1
- package/dist/providers/firecrawl.js +40 -29
- package/dist/providers/firecrawl.js.map +1 -1
- package/dist/providers/scrapedo.js.map +1 -1
- package/dist/providers/tavily.js.map +1 -1
- package/dist/proxy.js.map +1 -1
- package/dist/router.js +1 -1
- package/dist/router.js.map +1 -1
- package/dist/server.d.ts +2 -2
- package/dist/server.js +12 -6
- package/dist/server.js.map +1 -1
- package/package.json +8 -7
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.1.1] - 2026-10-19
|
|
4
|
+
|
|
5
|
+
Changes since v0.1.0; replaces the previous, unpublished v0.1.1 tag.
|
|
6
|
+
|
|
7
|
+
### Fixed
|
|
8
|
+
|
|
9
|
+
- Honor the requested Firecrawl crawl page limit, counting only usable documents and continuing through paginated results when needed.
|
|
10
|
+
- Perform a final crawl status poll after the polling deadline so completed results are not missed.
|
|
11
|
+
- Reject non-HTTP(S) scrape and crawl URLs before calling a provider.
|
|
12
|
+
- Trim whitespace around `RRCRAWL_AUTH_MODE` values.
|
|
13
|
+
- Return a rejected promise when no crawl-capable provider is configured.
|
|
14
|
+
- Report the package version in the MCP server handshake.
|
|
15
|
+
|
|
16
|
+
### Maintenance
|
|
17
|
+
|
|
18
|
+
- Update the MCP SDK, dotenv, undici, Zod, TypeScript, Vitest, and supporting development dependencies.
|
|
19
|
+
- Expand regression coverage for provider limits, polling, HTTP errors, configuration, routing, and MCP tool calls.
|
|
20
|
+
- Publish to npm from version-tag pushes, verifying that the tag matches the package version, and run branch CI on `master`.
|
|
21
|
+
|
|
22
|
+
[0.1.1]: https://github.com/romanbsd/rrcrawl/compare/v0.1.0...v0.1.1
|
package/dist/config.js
CHANGED
|
@@ -19,7 +19,7 @@ function positiveInteger(env, name, fallback) {
|
|
|
19
19
|
return parsed;
|
|
20
20
|
}
|
|
21
21
|
function parseAuthMode(env) {
|
|
22
|
-
const configured = env.RRCRAWL_AUTH_MODE ?? "auto";
|
|
22
|
+
const configured = (env.RRCRAWL_AUTH_MODE ?? "auto").trim();
|
|
23
23
|
if (configured === "auto") {
|
|
24
24
|
return env.ONECLI_URL ? "onecli" : "env";
|
|
25
25
|
}
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,IAAI,GAAG,CAAe;IAC1C,WAAW;IACX,QAAQ;IACR,UAAU;CACX,CAAC,CAAC;AAEH,SAAS,eAAe,CACtB,GAAgB,EAChB,IAAY,EACZ,QAAgB;IAEhB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,8EAA8E;IAC9E,uCAAuC;IACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,GAAgB;IACrC,MAAM,UAAU,GAAG,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,IAAI,GAAG,CAAe;IAC1C,WAAW;IACX,QAAQ;IACR,UAAU;CACX,CAAC,CAAC;AAEH,SAAS,eAAe,CACtB,GAAgB,EAChB,IAAY,EACZ,QAAgB;IAEhB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,QAAQ,CAAC;IAC5B,8EAA8E;IAC9E,uCAAuC;IACvC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,GAAgB;IACrC,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,iBAAiB,IAAI,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;IAC3C,CAAC;IACD,IAAI,UAAU,KAAK,KAAK,IAAI,UAAU,KAAK,QAAQ,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,MAAM,GAAG,KAAK;SACjB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;SAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,MAAM,EAAE,CAAC;QAC9B,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAwB,CAAC,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,0CAA0C,QAAQ,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAmB,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAG,GAAgB,OAAO,CAAC,GAAG;IACvD,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,YAAY,GAAG,GAAG,CAAC,iBAAiB,CAAC;IAC3C,MAAM,SAAS,GAAG,GAAG,CAAC,cAAc,CAAC;IACrC,MAAM,aAAa,GACjB,GAAG,CAAC,kBAAkB,IAAI,GAAG,CAAC,mBAAmB,IAAI,GAAG,CAAC,eAAe,CAAC;IAE3E,MAAM,mBAAmB,GAAG,GAAG,CAAC,iBAAiB;QAC/C,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,iBAAiB,CAAC;QAC1C,CAAC,CAAC,SAAS,CAAC;IAEd,IAAI,SAAyB,CAAC;IAC9B,IAAI,mBAAmB,EAAE,CAAC;QACxB,SAAS,GAAG,mBAAmB,CAAC;IAClC,CAAC;SAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,SAAS,GAAG,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;SAAM,CAAC;QACN,SAAS,GAAG;YACV,GAAG,CAAC,YAAY,CAAC,CAAC,CAAE,CAAC,WAAW,CAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAW,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,aAAa,CAAC,CAAC,CAAE,CAAC,UAAU,CAAW,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,iFAAiF,CAClF,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAC9B,CAAC,QAAQ,EAAE,EAAE,CACX,CAAC,QAAQ,KAAK,WAAW,IAAI,CAAC,YAAY,CAAC;YAC3C,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC;YACrC,CAAC,QAAQ,KAAK,UAAU,IAAI,CAAC,aAAa,CAAC,CAC9C,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CACb,iDAAiD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACtE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ;QACR,SAAS;QACT,gBAAgB,EAAE,eAAe,CAC/B,GAAG,EACH,4BAA4B,EAC5B,MAAM,CACP;QACD,cAAc,EAAE,eAAe,CAC7B,GAAG,EACH,0BAA0B,EAC1B,OAAO,CACR;QACD,cAAc,EAAE,eAAe,CAC7B,GAAG,EACH,0BAA0B,EAC1B,KAAK,CACN;QACD,SAAS,EAAE;YACT,MAAM,EAAE,CAAC,GAAG,CAAC,iBAAiB,IAAI,2BAA2B,CAAC,CAAC,OAAO,CACpE,KAAK,EACL,EAAE,CACH;YACD,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;SACtD;QACD,MAAM,EAAE;YACN,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,IAAI,wBAAwB,CAAC,CAAC,OAAO,CAC9D,KAAK,EACL,EAAE,CACH;YACD,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACnD;QACD,QAAQ,EAAE;YACR,MAAM,EAAE,GAAG,CAAC,gBAAgB,IAAI,wBAAwB;YACxD,QAAQ,EAAE,QAAQ,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;SACzD;KACF,CAAC;AACJ,CAAC"}
|
package/dist/http.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,SAAU,SAAQ,KAAK;IAEvB;
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,SAAU,SAAQ,KAAK;IAEvB,MAAM;IACN,YAAY;IAFvB,YACW,MAAc,EACd,YAAoB,EAC7B,GAAW;QAEX,KAAK,CAAC,QAAQ,MAAM,SAAS,GAAG,KAAK,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;sBAJ1D,MAAM;4BACN,YAAY;QAIrB,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IAC1B,CAAC;CACF;AAED,2EAA2E;AAC3E,8EAA8E;AAC9E,6BAA6B;AAC7B,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAEhC,QAAQ;IACR,MAAM;IAFjB,YACW,QAAgB,EAChB,MAAc;QAEvB,KAAK,CAAC,GAAG,QAAQ,0BAA0B,MAAM,GAAG,CAAC,CAAC;wBAH7C,QAAQ;sBACR,MAAM;QAGf,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;IACnC,CAAC;CACF;AAED,6EAA6E;AAC7E,sEAAsE;AACtE,MAAM,UAAU,YAAY,CAC1B,QAAgB,EAChB,KAAc,EACd,aAAgC;IAEhC,IAAI,KAAK,YAAY,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,kBAAkB,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,KAAK,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,0EAA0E;AAC1E,2EAA2E;AAC3E,MAAM,UAAU,iBAAiB,CAAC,KAAyB;IACzD,IAAI,CAAC,KAAK;QAAE,OAAO,KAAK,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QACpC,OAAO,QAAQ,KAAK,OAAO,IAAI,QAAQ,KAAK,QAAQ,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,oDAAoD;AACpD,KAAK,UAAU,WAAW,CACxB,OAAkB,EAClB,GAAW,EACX,IAAiB,EACjB,SAAiB;IAEjB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;YAClC,GAAG,IAAI;YACP,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAkB,EAClB,GAAW,EACX,IAAiB,EACjB,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAM,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAkB,EAClB,GAAW,EACX,IAAiB,EACjB,SAAiB;IAEjB,OAAO,WAAW,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;AACpD,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -27,7 +27,7 @@ See https://www.npmjs.com/package/rrcrawl for details.`);
|
|
|
27
27
|
return;
|
|
28
28
|
}
|
|
29
29
|
configureProxy();
|
|
30
|
-
await runServer(loadConfig());
|
|
30
|
+
await runServer(loadConfig(), version);
|
|
31
31
|
}
|
|
32
32
|
main().catch((error) => {
|
|
33
33
|
console.error(error instanceof Error ? error.message : String(error));
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,MAAM,OAAO,GAAG,aAAa,CAAC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,iBAAiB,CAAwB,CAAC;AAEtE,KAAK,UAAU,IAAI;IACjB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAClD,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IACD,IAAI,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,WAAW,OAAO;;;;;;;;;;uDAUqB,CAAC,CAAC;QACrD,OAAO;IACT,CAAC;IACD,cAAc,EAAE,CAAC;IACjB,MAAM,SAAS,CAAC,UAAU,EAAE,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -18,7 +18,7 @@ export declare class FirecrawlProvider implements ScrapeProvider, CrawlProvider
|
|
|
18
18
|
private headers;
|
|
19
19
|
scrape(request: ScrapeRequest): Promise<ScrapeResult>;
|
|
20
20
|
crawl(request: CrawlRequest): Promise<CrawlResult>;
|
|
21
|
-
private
|
|
21
|
+
private completedPages;
|
|
22
22
|
private json;
|
|
23
23
|
private documentUrl;
|
|
24
24
|
private page;
|
|
@@ -62,49 +62,60 @@ export class FirecrawlProvider {
|
|
|
62
62
|
if (!started.id) {
|
|
63
63
|
throw new Error("Firecrawl did not return a crawl job id");
|
|
64
64
|
}
|
|
65
|
+
const statusUrl = `${this.options.apiUrl}/v2/crawl/${encodeURIComponent(started.id)}`;
|
|
65
66
|
const deadline = Date.now() + this.options.crawlTimeoutMs;
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
const poll = () => this.json(statusUrl, {
|
|
68
|
+
method: "GET",
|
|
69
|
+
headers: this.headers(),
|
|
70
|
+
});
|
|
71
|
+
for (;;) {
|
|
72
|
+
const status = await poll();
|
|
68
73
|
if (status.status === "completed") {
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
.map((document) => {
|
|
74
|
-
const url = this.documentUrl(document);
|
|
75
|
-
return document.markdown && isAbsoluteHttpUrl(url)
|
|
76
|
-
? this.page(document, url)
|
|
77
|
-
: undefined;
|
|
78
|
-
})
|
|
79
|
-
.filter((page) => page !== undefined);
|
|
80
|
-
if (pages.length === 0) {
|
|
81
|
-
throw new Error("Firecrawl crawl completed without page content");
|
|
82
|
-
}
|
|
83
|
-
return { provider: this.name, pages };
|
|
74
|
+
return {
|
|
75
|
+
provider: this.name,
|
|
76
|
+
pages: await this.completedPages(status, request.limit),
|
|
77
|
+
};
|
|
84
78
|
}
|
|
85
79
|
if (status.status === "failed" || status.status === "cancelled") {
|
|
86
80
|
throw new Error(`Firecrawl crawl ${status.status}: ${status.error ?? "unknown error"}`);
|
|
87
81
|
}
|
|
82
|
+
if (Date.now() >= deadline) {
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
88
85
|
await this.sleep(this.options.pollIntervalMs);
|
|
89
86
|
}
|
|
90
87
|
throw new Error(`Firecrawl crawl timed out after ${this.options.crawlTimeoutMs}ms`);
|
|
91
88
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
89
|
+
async completedPages(first, limit) {
|
|
90
|
+
const pages = [];
|
|
91
|
+
let batch = first;
|
|
92
|
+
// Firecrawl paginates large results through `next`. Continue until enough
|
|
93
|
+
// usable pages have been collected, rather than counting malformed entries
|
|
94
|
+
// against the caller's page limit.
|
|
95
|
+
// Each crawl page must carry its own absolute URL. If Firecrawl omits it we
|
|
96
|
+
// drop the page rather than mislabel every one with the crawl root.
|
|
97
|
+
for (let i = 0; i <= limit; i += 1) {
|
|
98
|
+
for (const document of batch.data ?? []) {
|
|
99
|
+
const url = this.documentUrl(document);
|
|
100
|
+
if (document.markdown && isAbsoluteHttpUrl(url)) {
|
|
101
|
+
pages.push(this.page(document, url));
|
|
102
|
+
if (pages.length === limit) {
|
|
103
|
+
return pages;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (!batch.next || i === limit) {
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
batch = await this.json(batch.next, {
|
|
101
111
|
method: "GET",
|
|
102
112
|
headers: this.headers(),
|
|
103
113
|
});
|
|
104
|
-
documents.push(...(batch.data ?? []));
|
|
105
|
-
next = batch.next;
|
|
106
114
|
}
|
|
107
|
-
|
|
115
|
+
if (pages.length === 0) {
|
|
116
|
+
throw new Error("Firecrawl crawl completed without page content");
|
|
117
|
+
}
|
|
118
|
+
return pages;
|
|
108
119
|
}
|
|
109
120
|
// 402 Payment Required is Firecrawl's insufficient-credits signal (permanent).
|
|
110
121
|
// 429 rate limits are transient and deliberately left to normal failover.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../src/providers/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,YAAY,GAEb,MAAM,YAAY,CAAC;AAkDpB,MAAM,OAAO,iBAAiB;IAKC;
|
|
1
|
+
{"version":3,"file":"firecrawl.js","sourceRoot":"","sources":["../../src/providers/firecrawl.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,YAAY,GAEb,MAAM,YAAY,CAAC;AAkDpB,MAAM,OAAO,iBAAiB;IAKC,OAAO;IAJ3B,IAAI,GAAG,WAAoB,CAAC;IACpB,OAAO,CAAY;IACnB,KAAK,CAA0C;IAEhE,YAA6B,OAAyB;uBAAzB,OAAO;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;QACxC,IAAI,CAAC,KAAK;YACR,OAAO,CAAC,KAAK;gBACb,CAAC,CAAC,YAAY,EAAE,EAAE,CAChB,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,OAAO;QACb,OAAO;YACL,cAAc,EAAE,kBAAkB;YAClC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM;gBACrB,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE;gBACpD,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAC9B,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,YAAY,EAClC;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACvB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,CAAC,UAAU,CAAC;gBACrB,eAAe,EAAE,IAAI;aACtB,CAAC;SACH,CACF,CAAC;QAEF,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,4BAA4B,QAAQ,CAAC,KAAK,IAAI,eAAe,EAAE,CAChE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CACpB,QAAQ,CAAC,IAAI,EACb,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAC3C,CAAC;QACF,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAC7B,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,WAAW,EACjC;YACE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACvB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,iBAAiB,EAAE,OAAO,CAAC,QAAQ;gBACnC,YAAY,EACV,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;gBACpE,kBAAkB,EAAE,OAAO,CAAC,aAAa;gBACzC,MAAM,EAAE,OAAO,CAAC,YAAY;gBAC5B,aAAa,EAAE;oBACb,OAAO,EAAE,CAAC,UAAU,CAAC;oBACrB,eAAe,EAAE,IAAI;iBACtB;aACF,CAAC;SACH,CACF,CAAC;QAEF,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,oCAAoC,OAAO,CAAC,KAAK,IAAI,eAAe,EAAE,CACvE,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,aAAa,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QACtF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC;QAC1D,MAAM,IAAI,GAAG,GAAqC,EAAE,CAClD,IAAI,CAAC,IAAI,CAA0B,SAAS,EAAE;YAC5C,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;SACxB,CAAC,CAAC;QAEL,SAAS,CAAC;YACR,MAAM,MAAM,GAAG,MAAM,IAAI,EAAE,CAAC;YAC5B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO;oBACL,QAAQ,EAAE,IAAI,CAAC,IAAI;oBACnB,KAAK,EAAE,MAAM,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC;iBACxD,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAChE,MAAM,IAAI,KAAK,CACb,mBAAmB,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CACvE,CAAC;YACJ,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM;YACR,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,IAAI,KAAK,CACb,mCAAmC,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CACnE,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,KAA8B,EAC9B,KAAa;QAEb,MAAM,KAAK,GAAW,EAAE,CAAC;QACzB,IAAI,KAAK,GAAG,KAAK,CAAC;QAElB,0EAA0E;QAC1E,2EAA2E;QAC3E,mCAAmC;QACnC,4EAA4E;QAC5E,oEAAoE;QACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;gBACvC,IAAI,QAAQ,CAAC,QAAQ,IAAI,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC;oBAChD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;oBACrC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;wBAC3B,OAAO,KAAK,CAAC;oBACf,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;gBAC/B,MAAM;YACR,CAAC;YACD,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAA0B,KAAK,CAAC,IAAI,EAAE;gBAC3D,MAAM,EAAE,KAAK;gBACb,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;aACxB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,+EAA+E;IAC/E,0EAA0E;IAClE,KAAK,CAAC,IAAI,CAAI,GAAW,EAAE,IAAiB;QAClD,IAAI,CAAC;YACH,OAAO,MAAM,WAAW,CACtB,IAAI,CAAC,OAAO,EACZ,GAAG,EACH,IAAI,EACJ,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,QAA2B;QAC7C,OAAO,CACL,QAAQ,CAAC,QAAQ,EAAE,SAAS;YAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS;YAC5B,QAAQ,CAAC,QAAQ,EAAE,GAAG;YACtB,SAAS,CACV,CAAC;IACJ,CAAC;IAEO,IAAI,CAAC,QAA2B,EAAE,GAAW;QACnD,OAAO;YACL,GAAG;YACH,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK;gBAC1B,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE;gBACpC,CAAC,CAAC,EAAE,CAAC;YACP,QAAQ,EAAE,QAAQ,CAAC,QAAQ,IAAI,EAAE;SAClC,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrapedo.js","sourceRoot":"","sources":["../../src/providers/scrapedo.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,WAAW,GAEZ,MAAM,YAAY,CAAC;AAcpB,MAAM,OAAO,gBAAgB;IAIE;
|
|
1
|
+
{"version":3,"file":"scrapedo.js","sourceRoot":"","sources":["../../src/providers/scrapedo.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,kBAAkB,EAClB,WAAW,GAEZ,MAAM,YAAY,CAAC;AAcpB,MAAM,OAAO,gBAAgB;IAIE,OAAO;IAH3B,IAAI,GAAG,UAAmB,CAAC;IACnB,OAAO,CAAY;IAEpC,YAA6B,OAAwB;uBAAxB,OAAO;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9C,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9C,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC1B,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC5D,CAAC;QAED,oEAAoE;QACpE,uDAAuD;QACvD,4CAA4C;QAC5C,uEAAuE;QACvE,uEAAuE;QACvE,6DAA6D;QAC7D,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,WAAW,CAC1B,IAAI,CAAC,OAAO,EACZ,QAAQ,CAAC,QAAQ,EAAE,EACnB,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,gCAAgC,EAAE,EAAE,EACxE,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;gBAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACzB,MAAM,IAAI,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC/C,CAAC;gBACD,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACzE,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tavily.js","sourceRoot":"","sources":["../../src/providers/tavily.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,YAAY,GAEb,MAAM,YAAY,CAAC;AA2BpB,MAAM,OAAO,cAAc;IAII;
|
|
1
|
+
{"version":3,"file":"tavily.js","sourceRoot":"","sources":["../../src/providers/tavily.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,YAAY,GAEb,MAAM,YAAY,CAAC;AA2BpB,MAAM,OAAO,cAAc;IAII,OAAO;IAH3B,IAAI,GAAG,QAAiB,CAAC;IACjB,OAAO,CAAY;IAEpC,YAA6B,OAAsB;uBAAtB,OAAO;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC1C,CAAC;IAEO,OAAO;QACb,OAAO;YACL,cAAc,EAAE,kBAAkB;YAClC,MAAM,EAAE,kBAAkB;YAC1B,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM;gBACrB,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE;gBACpD,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAiB,UAAU,EAAE;YAC3D,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;YACnB,MAAM,EAAE,UAAU;YAClB,aAAa,EAAE,OAAO;SACvB,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,OAAO;YACL,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,GAAG,EAAE,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG;YAC7D,QAAQ,EAAE,MAAM,CAAC,WAAW;SAC7B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAiB,QAAQ,EAAE;YACzD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,SAAS,EAAE,OAAO,CAAC,QAAQ;YAC3B,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EACV,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;YACpE,cAAc,EAAE,OAAO,CAAC,aAAa;YACrC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,MAAM,EAAE,UAAU;YAClB,aAAa,EAAE,OAAO;SACvB,CAAC,CAAC;QACH,uEAAuE;QACvE,2EAA2E;QAC3E,+BAA+B;QAC/B,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;aACnC,GAAG,CAAC,CAAC,MAAM,EAAoB,EAAE,CAChC,MAAM,CAAC,WAAW,IAAI,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC;YACjD,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE;YACnD,CAAC,CAAC,SAAS,CACd;aACA,MAAM,CAAC,CAAC,IAAI,EAAgB,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;QACtD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;IACxC,CAAC;IAED,6EAA6E;IAC7E,4EAA4E;IACpE,KAAK,CAAC,IAAI,CAChB,IAAY,EACZ,IAA6B;QAE7B,IAAI,CAAC;YACH,OAAO,MAAM,WAAW,CACtB,IAAI,CAAC,OAAO,EACZ,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,EAAE,EAC/B;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;gBACvB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAC9B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;CACF"}
|
package/dist/proxy.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"proxy.js","sourceRoot":"","sources":["../src/proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,QAAQ,CAAC;AAIzD,gFAAgF;AAChF,yEAAyE;AACzE,oEAAoE;AACpE,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,UAAU,QAAQ,CAAC,GAAgB;IACvC,OAAO,CACL,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,UAAU;QACd,GAAG,CAAC,UAAU;QACd,SAAS,CACV,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAC5B,
|
|
1
|
+
{"version":3,"file":"proxy.js","sourceRoot":"","sources":["../src/proxy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,QAAQ,CAAC;AAIzD,gFAAgF;AAChF,yEAAyE;AACzE,oEAAoE;AACpE,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,UAAU,QAAQ,CAAC,GAAgB;IACvC,OAAO,CACL,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,UAAU;QACd,GAAG,CAAC,UAAU;QACd,SAAS,CACV,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAC5B,GAAG,GAAgB,OAAO,CAAC,GAAG;IAE9B,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,GAAG,EAAE,CAAC;QACR,mBAAmB,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/dist/router.js
CHANGED
|
@@ -41,7 +41,7 @@ export class RoundRobinRouter {
|
|
|
41
41
|
scrape(request) {
|
|
42
42
|
return this.tryProviders("scrape", this.scrapeProviders, (provider) => provider.scrape(request));
|
|
43
43
|
}
|
|
44
|
-
crawl(request) {
|
|
44
|
+
async crawl(request) {
|
|
45
45
|
if (this.crawlProviders.length === 0) {
|
|
46
46
|
throw new Error("No crawl-capable provider configured; enable Firecrawl or Tavily");
|
|
47
47
|
}
|
package/dist/router.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAW/C,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAErC;
|
|
1
|
+
{"version":3,"file":"router.js","sourceRoot":"","sources":["../src/router.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAW/C,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAErC,SAAS;IACT,QAAQ;IAFnB,YACW,SAA6B,EAC7B,QAA4D;QAErE,KAAK,CACH,OAAO,SAAS,sBAAsB,QAAQ;aAC3C,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,GAAG,QAAQ,KAAK,OAAO,EAAE,CAAC;aACzD,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;yBAPO,SAAS;wBACT,QAAQ;QAOjB,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED,+EAA+E;AAC/E,wDAAwD;AACxD,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAC3B,SAAS;IAA9B,YAAqB,SAA6B;QAChD,KAAK,CACH,4BAA4B,SAAS,uCAAuC,CAC7E,CAAC;yBAHiB,SAAS;QAI5B,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAED,MAAM,OAAO,gBAAgB;IAUR,eAAe;IACf,cAAc;IAVhB,MAAM,GAAuC;QAC5D,MAAM,EAAE,CAAC;QACT,KAAK,EAAE,CAAC;KACT,CAAC;IACF,6EAA6E;IAC7E,4EAA4E;IAC3D,QAAQ,GAAG,IAAI,GAAG,EAAgB,CAAC;IAEpD,YACmB,eAAiC,EACjC,cAA+B;+BAD/B,eAAe;8BACf,cAAc;QAE/B,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,MAAM,CAAC,OAAsB;QAC3B,OAAO,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,QAAQ,EAAE,EAAE,CACpE,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CACzB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAqB;QAC/B,IAAI,IAAI,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,QAAQ,EAAE,EAAE,CAClE,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CACxB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,SAA6B,EAC7B,SAAsB,EACtB,OAAkD;QAElD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAC7B,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAChD,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACrD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAuD,EAAE,CAAC;QAExE,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;YACzD,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,IAAI,CAAC;gBACH,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;oBACxC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACnC,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,QAAQ,CAAC,IAAI;oBACvB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAChE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,0CAA0C;QAC1C,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,uBAAuB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,IAAI,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACzD,CAAC;CACF"}
|
package/dist/server.d.ts
CHANGED
|
@@ -2,5 +2,5 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
2
2
|
import type { AppConfig } from "./config.js";
|
|
3
3
|
import { RoundRobinRouter } from "./router.js";
|
|
4
4
|
export declare function createRouter(config: AppConfig): RoundRobinRouter;
|
|
5
|
-
export declare function createServer(router: RoundRobinRouter): McpServer;
|
|
6
|
-
export declare function runServer(config: AppConfig): Promise<void>;
|
|
5
|
+
export declare function createServer(router: RoundRobinRouter, version: string): McpServer;
|
|
6
|
+
export declare function runServer(config: AppConfig, version: string): Promise<void>;
|
package/dist/server.js
CHANGED
|
@@ -10,6 +10,12 @@ const pageSchema = z.object({
|
|
|
10
10
|
title: z.string().optional(),
|
|
11
11
|
markdown: z.string(),
|
|
12
12
|
});
|
|
13
|
+
// The tool descriptions promise HTTP(S) only; z.string().url() alone would
|
|
14
|
+
// accept ftp:// or file:// which providers may not scrape correctly.
|
|
15
|
+
const httpUrl = z
|
|
16
|
+
.string()
|
|
17
|
+
.url()
|
|
18
|
+
.regex(/^https?:\/\//i, "must be an HTTP or HTTPS URL");
|
|
13
19
|
function toErrorResult(error) {
|
|
14
20
|
return {
|
|
15
21
|
content: [
|
|
@@ -50,16 +56,16 @@ export function createRouter(config) {
|
|
|
50
56
|
}
|
|
51
57
|
return new RoundRobinRouter(scrapeProviders, crawlProviders);
|
|
52
58
|
}
|
|
53
|
-
export function createServer(router) {
|
|
59
|
+
export function createServer(router, version) {
|
|
54
60
|
const server = new McpServer({
|
|
55
61
|
name: "rrcrawl",
|
|
56
|
-
version
|
|
62
|
+
version,
|
|
57
63
|
});
|
|
58
64
|
server.registerTool("scrape", {
|
|
59
65
|
title: "Scrape one URL",
|
|
60
66
|
description: "Fetch one URL as normalized Markdown. Requests rotate across all configured providers and fail over on errors.",
|
|
61
67
|
inputSchema: {
|
|
62
|
-
url:
|
|
68
|
+
url: httpUrl.describe("HTTP or HTTPS URL to scrape"),
|
|
63
69
|
},
|
|
64
70
|
outputSchema: pageSchema.extend({
|
|
65
71
|
provider: z.enum(["firecrawl", "tavily", "scrapedo"]),
|
|
@@ -90,7 +96,7 @@ export function createServer(router) {
|
|
|
90
96
|
title: "Crawl a website",
|
|
91
97
|
description: "Crawl multiple pages as normalized Markdown. Requests rotate between configured crawl-capable providers (Firecrawl and Tavily) and fail over on errors.",
|
|
92
98
|
inputSchema: {
|
|
93
|
-
url:
|
|
99
|
+
url: httpUrl.describe("Root HTTP or HTTPS URL"),
|
|
94
100
|
limit: z
|
|
95
101
|
.number()
|
|
96
102
|
.int()
|
|
@@ -153,8 +159,8 @@ export function createServer(router) {
|
|
|
153
159
|
});
|
|
154
160
|
return server;
|
|
155
161
|
}
|
|
156
|
-
export async function runServer(config) {
|
|
157
|
-
const server = createServer(createRouter(config));
|
|
162
|
+
export async function runServer(config, version) {
|
|
163
|
+
const server = createServer(createRouter(config), version);
|
|
158
164
|
await server.connect(new StdioServerTransport());
|
|
159
165
|
}
|
|
160
166
|
//# sourceMappingURL=server.js.map
|
package/dist/server.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAM/C,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACrB,CAAC,CAAC;AAEH,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D;SACF;QACD,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAiB;IAC5C,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;YACrC,GAAG,MAAM,CAAC,SAAS;YACnB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,GAAG,MAAM,CAAC,MAAM;YAChB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,eAAe,CAAC,IAAI,CAClB,IAAI,gBAAgB,CAAC;YACnB,GAAG,MAAM,CAAC,QAAQ;YAClB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,gBAAgB,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,YAAY,
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAM/C,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACrB,CAAC,CAAC;AAEH,2EAA2E;AAC3E,qEAAqE;AACrE,MAAM,OAAO,GAAG,CAAC;KACd,MAAM,EAAE;KACR,GAAG,EAAE;KACL,KAAK,CAAC,eAAe,EAAE,8BAA8B,CAAC,CAAC;AAE1D,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO;QACL,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC7D;SACF;QACD,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAiB;IAC5C,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC;YACrC,GAAG,MAAM,CAAC,SAAS;YACnB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAI,cAAc,CAAC;YAClC,GAAG,MAAM,CAAC,MAAM;YAChB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IACD,IAAI,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1C,eAAe,CAAC,IAAI,CAClB,IAAI,gBAAgB,CAAC;YACnB,GAAG,MAAM,CAAC,QAAQ;YAClB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;SAC1C,CAAC,CACH,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,gBAAgB,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,MAAwB,EACxB,OAAe;IAEf,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;QAC3B,IAAI,EAAE,SAAS;QACf,OAAO;KACR,CAAC,CAAC;IAEH,MAAM,CAAC,YAAY,CACjB,QAAQ,EACR;QACE,KAAK,EAAE,gBAAgB;QACvB,WAAW,EACT,gHAAgH;QAClH,WAAW,EAAE;YACX,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAC;SACrD;QACD,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC;YAC9B,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC;SACtD,CAAC;QACF,WAAW,EAAE;YACX,YAAY,EAAE,IAAI;YAClB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;KACF,EACD,KAAK,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;QAChB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5C,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,iBAAiB,EAAE;oBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChD,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,MAAM,CAAC,YAAY,CACjB,OAAO,EACP;QACE,KAAK,EAAE,iBAAiB;QACxB,WAAW,EACT,yJAAyJ;QAC3J,WAAW,EAAE;YACX,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC;YAC/C,KAAK,EAAE,CAAC;iBACL,MAAM,EAAE;iBACR,GAAG,EAAE;iBACL,GAAG,CAAC,CAAC,CAAC;iBACN,GAAG,CAAC,GAAG,CAAC;iBACR,OAAO,CAAC,EAAE,CAAC;iBACX,QAAQ,CAAC,yBAAyB,CAAC;YACtC,QAAQ,EAAE,CAAC;iBACR,MAAM,EAAE;iBACR,GAAG,EAAE;iBACL,GAAG,CAAC,CAAC,CAAC;iBACN,GAAG,CAAC,CAAC,CAAC;iBACN,OAAO,CAAC,CAAC,CAAC;iBACV,QAAQ,CAAC,sCAAsC,CAAC;YACnD,YAAY,EAAE,CAAC;iBACZ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;iBACjB,GAAG,CAAC,EAAE,CAAC;iBACP,OAAO,CAAC,EAAE,CAAC;iBACX,QAAQ,CAAC,wCAAwC,CAAC;YACrD,aAAa,EAAE,CAAC;iBACb,OAAO,EAAE;iBACT,OAAO,CAAC,KAAK,CAAC;iBACd,QAAQ,CAAC,qCAAqC,CAAC;YAClD,YAAY,EAAE,CAAC;iBACZ,MAAM,EAAE;iBACR,GAAG,CAAC,KAAK,CAAC;iBACV,QAAQ,EAAE;iBACV,QAAQ,CAAC,+CAA+C,CAAC;SAC7D;QACD,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC;YACrB,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YACzC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC;SAC3B,CAAC;QACF,WAAW,EAAE;YACX,YAAY,EAAE,IAAI;YAClB,cAAc,EAAE,IAAI;YACpB,aAAa,EAAE,IAAI;SACpB;KACF,EACD,KAAK,EAAE,EACL,GAAG,EACH,KAAK,EACL,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,YAAY,GACb,EAAE,EAAE;QACH,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC;gBAChC,GAAG;gBACH,KAAK;gBACL,QAAQ;gBACR,YAAY;gBACZ,aAAa;gBACb,YAAY;aACb,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,iBAAiB,EAAE;oBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;iBACpB;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,MAAiB,EACjB,OAAe;IAEf,MAAM,MAAM,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;AACnD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rrcrawl",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "Tiny round-robin crawl MCP backed by Firecrawl, Tavily, and Scrape.do",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Roman Shterenzon <roman.shterenzon@gmail.com>",
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"files": [
|
|
29
29
|
"dist",
|
|
30
30
|
"README.md",
|
|
31
|
+
"CHANGELOG.md",
|
|
31
32
|
".env.example"
|
|
32
33
|
],
|
|
33
34
|
"scripts": {
|
|
@@ -44,16 +45,16 @@
|
|
|
44
45
|
"node": ">=22"
|
|
45
46
|
},
|
|
46
47
|
"dependencies": {
|
|
47
|
-
"@modelcontextprotocol/sdk": "1.
|
|
48
|
-
"dotenv": "^
|
|
48
|
+
"@modelcontextprotocol/sdk": "^1.30.0",
|
|
49
|
+
"dotenv": "^18.0.0",
|
|
49
50
|
"undici": "^8.8.0",
|
|
50
|
-
"zod": "^
|
|
51
|
+
"zod": "^4.4.3"
|
|
51
52
|
},
|
|
52
53
|
"devDependencies": {
|
|
53
|
-
"@types/node": "^
|
|
54
|
+
"@types/node": "^26.1.1",
|
|
54
55
|
"tsx": "^4.21.0",
|
|
55
|
-
"typescript": "^
|
|
56
|
-
"vitest": "^
|
|
56
|
+
"typescript": "^7.0.2",
|
|
57
|
+
"vitest": "^5.0.0"
|
|
57
58
|
},
|
|
58
59
|
"overrides": {
|
|
59
60
|
"@hono/node-server": "2.0.11"
|