web-access-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +270 -0
- package/README_CN.md +268 -0
- package/THIRD_PARTY_NOTICES.md +48 -0
- package/dist/cli.js +197 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/config.js +463 -0
- package/dist/config/config.js.map +1 -0
- package/dist/config/edit.js +126 -0
- package/dist/config/edit.js.map +1 -0
- package/dist/core/diagnostics.js +94 -0
- package/dist/core/diagnostics.js.map +1 -0
- package/dist/core/errors.js +47 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/router.js +320 -0
- package/dist/core/router.js.map +1 -0
- package/dist/core/schema.js +120 -0
- package/dist/core/schema.js.map +1 -0
- package/dist/core/types.js +22 -0
- package/dist/core/types.js.map +1 -0
- package/dist/providers/anysearch.js +225 -0
- package/dist/providers/anysearch.js.map +1 -0
- package/dist/providers/common.js +210 -0
- package/dist/providers/common.js.map +1 -0
- package/dist/providers/extract.js +155 -0
- package/dist/providers/extract.js.map +1 -0
- package/dist/providers/registry.js +31 -0
- package/dist/providers/registry.js.map +1 -0
- package/dist/providers/rsc.js +109 -0
- package/dist/providers/rsc.js.map +1 -0
- package/dist/providers/search.js +178 -0
- package/dist/providers/search.js.map +1 -0
- package/dist/providers/xcrawl.js +170 -0
- package/dist/providers/xcrawl.js.map +1 -0
- package/dist/transport/http.js +175 -0
- package/dist/transport/http.js.map +1 -0
- package/dist/version.js +2 -0
- package/dist/version.js.map +1 -0
- package/package.json +61 -0
- package/schemas/config.schema.json +210 -0
- package/schemas/extractRequest.schema.json +21 -0
- package/schemas/output.schema.json +435 -0
- package/schemas/searchRequest.schema.json +57 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/providers/common.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAS/D,MAAM,UAAU,GAAG,CAAC,QAA0B;IAC7C,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAA0B;IAC3D,IAAI,CAAC,QAAQ,CAAC,MAAM;QACnB,MAAM,IAAI,cAAc,CACvB,sBAAsB,EACtB,GAAG,QAAQ,CAAC,EAAE,aAAa,EAC3B,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC3B,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,QAA0B;IACxD,IAAI,CAAC,QAAQ,CAAC,OAAO;QACpB,MAAM,IAAI,cAAc,CACvB,sBAAsB,EACtB,GAAG,QAAQ,CAAC,EAAE,aAAa,EAC3B,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC3B,CAAC;IACH,OAAO,QAAQ,CAAC,OAAO,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAClC,QAAsB,EACtB,QAA0B,EAC1B,UAAoB,EAAE;IAEtB,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC/B,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAC;IAC9D,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACtC,OAAO,IAAI,cAAc,CACxB,YAAY,EACZ,GAAG,QAAQ,CAAC,EAAE,YAAY,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAC5D;YACC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC;YACvB,UAAU,EAAE,MAAM;YAClB,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CACD,CAAC;IACH,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACpB,OAAO,IAAI,cAAc,CACxB,gBAAgB,EAChB,GAAG,QAAQ,CAAC,EAAE,eAAe,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EACtD;YACC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC;YACvB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CACD,CAAC;IACH,CAAC;IACD,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACxD,OAAO,IAAI,cAAc,CACxB,cAAc,EACd,GAAG,QAAQ,CAAC,EAAE,YAAY,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAC5D;YACC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC;YACvB,UAAU,EAAE,MAAM;YAClB,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CACD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,cAAc,CACxB,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,EACnD,GAAG,QAAQ,CAAC,EAAE,YAAY,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAC5D;QACC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC;QACvB,UAAU,EAAE,MAAM;QAClB,SAAS,EAAE,MAAM,IAAI,GAAG;QACxB,GAAG,EAAE,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC;KAChC,CACD,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACxC,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,QAAsB,EACtB,QAA0B;IAE1B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE;QACxB,MAAM,IAAI,cAAc,CAAC,kBAAkB,EAAE,GAAG,QAAQ,CAAC,EAAE,QAAQ,EAAE;YACpE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC;YACvB,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,GAAG,EAAE,QAAQ,CAAC,IAAI;SAClB,CAAC,CAAC;IACJ,IAAI,CAAC;QACJ,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;QAClC,OAAO,KAAgC,CAAC;IACzC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,IAAI,cAAc,CACvB,kBAAkB,EAClB,GAAG,QAAQ,CAAC,EAAE,eAAe,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACrF;YACC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC;YACvB,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,SAAS,EAAE,IAAI;YACf,GAAG,EAAE,QAAQ,CAAC,IAAI;SAClB,CACD,CAAC;IACH,CAAC;AACF,CAAC;AAED,MAAM,UAAU,QAAQ,CACvB,QAAsB,EACtB,QAA0B;IAE1B,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG;QACtE,MAAM,mBAAmB,CACxB,QAAQ,EACR,QAAQ,EACR,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CACxC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,GAAW;IAC1C,IAAI,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAC7B,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,SAAS,GAAG,EAAE,CAAC,CAAC;IAC3D,IAAI,CAAC;QACJ,KAAK,GAAG,IAAI,GAAG,CACd,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,KAAK,EAAE,CAClD,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,SAAS,GAAG,EAAE,CAAC,CAAC;IAC3D,CAAC;IACD,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;IACxC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;QACvD,MAAM,IAAI,cAAc,CAAC,eAAe,EAAE,SAAS,GAAG,EAAE,CAAC,CAAC;IAC3D,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAgB;IAChD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CACrC,KAAa,EACb,cAAwB,EACxB,cAAwB;IAExB,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;IACtB,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACpE,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC;QACjC,KAAK,CAAC,IAAI,CACT,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CACpE,CAAC;IACH,KAAK,MAAM,MAAM,IAAI,cAAc;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,MAAc;IAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACvD,OAAO,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAChC,GAAW,EACX,cAAwB,EACxB,cAAwB;IAExB,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;YAC9D,OAAO,KAAK,CAAC;QACd,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,KAAK,CAAC;IACd,CAAC;IACD,IACC,cAAc,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAE/D,OAAO,KAAK,CAAC;IACd,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,aAAa,CAC5B,KAAc,EACd,KAAa,EACb,cAAwB,EACxB,cAAwB;IAExB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAAE,SAAS;QACvE,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,IACC,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ;YAC3B,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,cAAc,EAAE,cAAc,CAAC;YAE3D,SAAS;QACV,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChC,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACjB,GAAG,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,SAAS;QACV,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GACV,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;YAChD,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE;YAClB,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,OAAO,GACZ,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;YAC9B,CAAC,CAAC,GAAG,CAAC,OAAO;YACb,CAAC,CAAC,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ;gBACpC,CAAC,CAAC,GAAG,CAAC,WAAW;gBACjB,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;oBAChC,CAAC,CAAC,GAAG,CAAC,OAAO;oBACb,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;wBAC7B,CAAC,CAAC,GAAG,CAAC,IAAI;wBACV,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;4BAC9B,CAAC,CAAC,GAAG,CAAC,UAAU;iCACb,MAAM,CACN,CAAC,KAAK,EAAmB,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CACrD;iCACA,IAAI,CAAC,GAAG,CAAC;4BACZ,CAAC,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;YACvB,KAAK;YACL,GAAG;YACH,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;SAC5C,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,MAAM,IAAI,KAAK;YAAE,MAAM;IACnC,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,MAAM,UAAU,kBAAkB,CACjC,KAAsB,EACtB,GAAG,GAAG,IAAI,IAAI,EAAE;IAEhB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,KAAK,KAAK,KAAK;QAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK,KAAK,OAAO;QAAE,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAChE,IAAI,KAAK,KAAK,MAAM;QAAE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC;IACrE,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,eAAe,CAC9B,QAA0B,EAC1B,SAAiC;IAEjC,OAAO,EAAE,GAAG,SAAS,EAAE,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC5C,QAA0B,EAC1B,SAAiC;IAEjC,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACrD,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrB,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO;YAC5B,OAAO,CAAC,sBAAsB,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;;YAC9C,OAAO,CAAC,aAAa,GAAG,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC1D,CAAC;IACD,OAAO,OAAO,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { Readability } from "@mozilla/readability";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
import TurndownService from "turndown";
|
|
4
|
+
import { WebAccessError } from "../core/errors.js";
|
|
5
|
+
import { buildEndpoint } from "../transport/http.js";
|
|
6
|
+
import { assertOk, parseJsonResponse, providerHeadersWithCredential, ref, requireBaseUrl, requireCredential, } from "./common.js";
|
|
7
|
+
import { extractRscMarkdown } from "./rsc.js";
|
|
8
|
+
function documentFromContent(content, request, title = "") {
|
|
9
|
+
const normalized = content
|
|
10
|
+
.replace(/\r\n/g, "\n")
|
|
11
|
+
.replace(/[ \t]+\n/g, "\n")
|
|
12
|
+
.trim();
|
|
13
|
+
return {
|
|
14
|
+
document: {
|
|
15
|
+
sourceUrl: request.url,
|
|
16
|
+
title: title.trim(),
|
|
17
|
+
content: normalized,
|
|
18
|
+
contentType: "text/markdown",
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
function stringField(value) {
|
|
23
|
+
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
|
24
|
+
}
|
|
25
|
+
function parseJinaDocument(body) {
|
|
26
|
+
const declaredTitle = /^Title:\s*(.+)$/im.exec(body)?.[1]?.trim() ?? "";
|
|
27
|
+
const marker = "Markdown Content:";
|
|
28
|
+
const markerIndex = body.indexOf(marker);
|
|
29
|
+
const content = (markerIndex >= 0 ? body.slice(markerIndex + marker.length) : body).trim();
|
|
30
|
+
const headingTitle = /^#\s+(.+)$/m.exec(content)?.[1]?.trim() ?? "";
|
|
31
|
+
return { content, title: declaredTitle || headingTitle };
|
|
32
|
+
}
|
|
33
|
+
const firecrawl = {
|
|
34
|
+
type: "firecrawl",
|
|
35
|
+
capabilities: ["extract"],
|
|
36
|
+
isConfigured: (instance) => !!instance.baseUrl &&
|
|
37
|
+
(!!instance.apiKey || instance.baseUrl !== "https://api.firecrawl.dev"),
|
|
38
|
+
async extract(request) {
|
|
39
|
+
if (requireBaseUrl(request.instance) === "https://api.firecrawl.dev")
|
|
40
|
+
requireCredential(request.instance);
|
|
41
|
+
const response = await request.transport.request(buildEndpoint(requireBaseUrl(request.instance), "v2/scrape"), {
|
|
42
|
+
method: "POST",
|
|
43
|
+
headers: providerHeadersWithCredential(request.instance, {
|
|
44
|
+
"Content-Type": "application/json",
|
|
45
|
+
Accept: "application/json",
|
|
46
|
+
}),
|
|
47
|
+
body: JSON.stringify({ url: request.url, formats: ["markdown"] }),
|
|
48
|
+
signal: request.signal,
|
|
49
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
50
|
+
});
|
|
51
|
+
assertOk(response, request.instance);
|
|
52
|
+
const parsed = parseJsonResponse(response, request.instance);
|
|
53
|
+
const root = parsed.data &&
|
|
54
|
+
typeof parsed.data === "object" &&
|
|
55
|
+
!Array.isArray(parsed.data)
|
|
56
|
+
? parsed.data
|
|
57
|
+
: parsed;
|
|
58
|
+
const content = stringField(root.markdown) ?? stringField(root.content);
|
|
59
|
+
if (!content)
|
|
60
|
+
throw new WebAccessError("no_usable_content", "Firecrawl 没有返回 Markdown 正文", { provider: ref(request.instance), retryable: true, raw: parsed });
|
|
61
|
+
const metadata = root.metadata &&
|
|
62
|
+
typeof root.metadata === "object" &&
|
|
63
|
+
!Array.isArray(root.metadata)
|
|
64
|
+
? root.metadata
|
|
65
|
+
: undefined;
|
|
66
|
+
return {
|
|
67
|
+
data: documentFromContent(content, request, stringField(metadata?.title) ?? stringField(root.title)),
|
|
68
|
+
raw: parsed,
|
|
69
|
+
};
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
const jina = {
|
|
73
|
+
type: "jina",
|
|
74
|
+
capabilities: ["extract"],
|
|
75
|
+
isConfigured: (instance) => !!instance.baseUrl,
|
|
76
|
+
async extract(request) {
|
|
77
|
+
const base = requireBaseUrl(request.instance).replace(/\/+$/, "");
|
|
78
|
+
const target = `${base}/${request.url}`;
|
|
79
|
+
const response = await request.transport.request(target, {
|
|
80
|
+
headers: providerHeadersWithCredential(request.instance, {
|
|
81
|
+
Accept: "text/markdown",
|
|
82
|
+
}),
|
|
83
|
+
signal: request.signal,
|
|
84
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
85
|
+
});
|
|
86
|
+
assertOk(response, request.instance);
|
|
87
|
+
const parsed = parseJinaDocument(response.body);
|
|
88
|
+
return {
|
|
89
|
+
data: documentFromContent(parsed.content, request, parsed.title),
|
|
90
|
+
raw: response.body,
|
|
91
|
+
};
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
const turndown = new TurndownService({
|
|
95
|
+
headingStyle: "atx",
|
|
96
|
+
bulletListMarker: "-",
|
|
97
|
+
codeBlockStyle: "fenced",
|
|
98
|
+
});
|
|
99
|
+
function extractHttpDocument(body, contentType) {
|
|
100
|
+
const isHtml = /html|xhtml/i.test(contentType) ||
|
|
101
|
+
/^\s*(?:<!doctype html|<html|<head|<body)/i.test(body);
|
|
102
|
+
if (!isHtml)
|
|
103
|
+
return { markdown: body, title: "" };
|
|
104
|
+
const { document } = parseHTML(body);
|
|
105
|
+
let title = document.querySelector("title")?.textContent?.trim() ?? "";
|
|
106
|
+
let html = body;
|
|
107
|
+
try {
|
|
108
|
+
const article = new Readability(document).parse();
|
|
109
|
+
if (article?.content) {
|
|
110
|
+
html = article.content;
|
|
111
|
+
title = article.title?.trim() || title;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
html = body;
|
|
116
|
+
}
|
|
117
|
+
const readability = turndown.turndown(html);
|
|
118
|
+
const rsc = extractRscMarkdown(body);
|
|
119
|
+
return {
|
|
120
|
+
markdown: rsc.length > readability.length ? rsc : readability,
|
|
121
|
+
title,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
const http = {
|
|
125
|
+
type: "http",
|
|
126
|
+
capabilities: ["extract"],
|
|
127
|
+
isConfigured: () => true,
|
|
128
|
+
async extract(request) {
|
|
129
|
+
const response = await request.transport.request(request.url, {
|
|
130
|
+
headers: {
|
|
131
|
+
Accept: "text/html,application/xhtml+xml,text/plain,text/markdown;q=0.9,*/*;q=0.1",
|
|
132
|
+
...request.instance.headers,
|
|
133
|
+
},
|
|
134
|
+
signal: request.signal,
|
|
135
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
136
|
+
});
|
|
137
|
+
assertOk(response, request.instance);
|
|
138
|
+
const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
|
|
139
|
+
if (/^(?:image|audio|video)\//.test(contentType) ||
|
|
140
|
+
/application\/(?:zip|octet-stream|pdf)/.test(contentType)) {
|
|
141
|
+
throw new WebAccessError("unsupported_content", `不支持的内容类型: ${contentType.split(";")[0]}`, {
|
|
142
|
+
provider: ref(request.instance),
|
|
143
|
+
retryable: false,
|
|
144
|
+
raw: response.body,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
const parsed = extractHttpDocument(response.body, contentType);
|
|
148
|
+
return {
|
|
149
|
+
data: documentFromContent(parsed.markdown, request, parsed.title),
|
|
150
|
+
raw: response.body,
|
|
151
|
+
};
|
|
152
|
+
},
|
|
153
|
+
};
|
|
154
|
+
export const EXTRACT_ADAPTERS = [firecrawl, jina, http];
|
|
155
|
+
//# sourceMappingURL=extract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/providers/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAOnD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EACN,QAAQ,EACR,iBAAiB,EACjB,6BAA6B,EAC7B,GAAG,EACH,cAAc,EACd,iBAAiB,GACjB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAE9C,SAAS,mBAAmB,CAC3B,OAAe,EACf,OAA8B,EAC9B,KAAK,GAAG,EAAE;IAEV,MAAM,UAAU,GAAG,OAAO;SACxB,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC;SACtB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,IAAI,EAAE,CAAC;IACT,OAAO;QACN,QAAQ,EAAE;YACT,SAAS,EAAE,OAAO,CAAC,GAAG;YACtB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;YACnB,OAAO,EAAE,UAAU;YACnB,WAAW,EAAE,eAAe;SAC5B;KACD,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IAClC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACtC,MAAM,aAAa,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxE,MAAM,MAAM,GAAG,mBAAmB,CAAC;IACnC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,CACf,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CACjE,CAAC,IAAI,EAAE,CAAC;IACT,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,IAAI,YAAY,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,SAAS,GAAoB;IAClC,IAAI,EAAE,WAAW;IACjB,YAAY,EAAE,CAAC,SAAS,CAAC;IACzB,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAC1B,CAAC,CAAC,QAAQ,CAAC,OAAO;QAClB,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,OAAO,KAAK,2BAA2B,CAAC;IACxE,KAAK,CAAC,OAAO,CAAC,OAAO;QACpB,IAAI,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,2BAA2B;YACnE,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAC/C,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC,EAC5D;YACC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,6BAA6B,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACxD,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;aAC1B,CAAC;YACF,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC;YACjE,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CACD,CAAC;QACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,MAAM,IAAI,GACT,MAAM,CAAC,IAAI;YACX,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;YAC/B,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;YAC1B,CAAC,CAAE,MAAM,CAAC,IAAgC;YAC1C,CAAC,CAAC,MAAM,CAAC;QACX,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxE,IAAI,CAAC,OAAO;YACX,MAAM,IAAI,cAAc,CACvB,mBAAmB,EACnB,4BAA4B,EAC5B,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,CACjE,CAAC;QACH,MAAM,QAAQ,GACb,IAAI,CAAC,QAAQ;YACb,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;YACjC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC5B,CAAC,CAAE,IAAI,CAAC,QAAoC;YAC5C,CAAC,CAAC,SAAS,CAAC;QACd,OAAO;YACN,IAAI,EAAE,mBAAmB,CACxB,OAAO,EACP,OAAO,EACP,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CACvD;YACD,GAAG,EAAE,MAAM;SACX,CAAC;IACH,CAAC;CACD,CAAC;AAEF,MAAM,IAAI,GAAoB;IAC7B,IAAI,EAAE,MAAM;IACZ,YAAY,EAAE,CAAC,SAAS,CAAC;IACzB,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO;IAC9C,KAAK,CAAC,OAAO,CAAC,OAAO;QACpB,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,MAAM,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE;YACxD,OAAO,EAAE,6BAA6B,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACxD,MAAM,EAAE,eAAe;aACvB,CAAC;YACF,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAChD,OAAO;YACN,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;YAChE,GAAG,EAAE,QAAQ,CAAC,IAAI;SAClB,CAAC;IACH,CAAC;CACD,CAAC;AAEF,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;IACpC,YAAY,EAAE,KAAK;IACnB,gBAAgB,EAAE,GAAG;IACrB,cAAc,EAAE,QAAQ;CACxB,CAAC,CAAC;AAEH,SAAS,mBAAmB,CAC3B,IAAY,EACZ,WAAmB;IAEnB,MAAM,MAAM,GACX,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC;QAC/B,2CAA2C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAClD,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACvE,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAA+B,CAAC,CAAC,KAAK,EAAE,CAAC;QACzE,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YACtB,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC;YACvB,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC;QACxC,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,IAAI,GAAG,IAAI,CAAC;IACb,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,GAAG,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO;QACN,QAAQ,EAAE,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW;QAC7D,KAAK;KACL,CAAC;AACH,CAAC;AAED,MAAM,IAAI,GAAoB;IAC7B,IAAI,EAAE,MAAM;IACZ,YAAY,EAAE,CAAC,SAAS,CAAC;IACzB,YAAY,EAAE,GAAG,EAAE,CAAC,IAAI;IACxB,KAAK,CAAC,OAAO,CAAC,OAAO;QACpB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE;YAC7D,OAAO,EAAE;gBACR,MAAM,EACL,0EAA0E;gBAC3E,GAAG,OAAO,CAAC,QAAQ,CAAC,OAAO;aAC3B;YACD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,WAAW,GAChB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC3D,IACC,0BAA0B,CAAC,IAAI,CAAC,WAAW,CAAC;YAC5C,uCAAuC,CAAC,IAAI,CAAC,WAAW,CAAC,EACxD,CAAC;YACF,MAAM,IAAI,cAAc,CACvB,qBAAqB,EACrB,aAAa,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EACxC;gBACC,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC/B,SAAS,EAAE,KAAK;gBAChB,GAAG,EAAE,QAAQ,CAAC,IAAI;aAClB,CACD,CAAC;QACH,CAAC;QACD,MAAM,MAAM,GAAG,mBAAmB,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAC/D,OAAO;YACN,IAAI,EAAE,mBAAmB,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC;YACjE,GAAG,EAAE,QAAQ,CAAC,IAAI;SAClB,CAAC;IACH,CAAC;CACD,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAsB,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { ANYSEARCH_ADAPTER } from "./anysearch.js";
|
|
2
|
+
import { EXTRACT_ADAPTERS } from "./extract.js";
|
|
3
|
+
import { EXA_ADAPTER, SEARCH_ADAPTERS } from "./search.js";
|
|
4
|
+
import { XCRAWL_ADAPTER } from "./xcrawl.js";
|
|
5
|
+
const adapters = [...SEARCH_ADAPTERS, ...EXTRACT_ADAPTERS].reduce((map, adapter) => {
|
|
6
|
+
const existing = map.get(adapter.type);
|
|
7
|
+
if (!existing)
|
|
8
|
+
map.set(adapter.type, adapter);
|
|
9
|
+
else
|
|
10
|
+
map.set(adapter.type, {
|
|
11
|
+
...existing,
|
|
12
|
+
capabilities: [
|
|
13
|
+
...new Set([...existing.capabilities, ...adapter.capabilities]),
|
|
14
|
+
],
|
|
15
|
+
search: existing.search ?? adapter.search,
|
|
16
|
+
extract: existing.extract ?? adapter.extract,
|
|
17
|
+
});
|
|
18
|
+
return map;
|
|
19
|
+
}, new Map([
|
|
20
|
+
[EXA_ADAPTER.type, EXA_ADAPTER],
|
|
21
|
+
[ANYSEARCH_ADAPTER.type, ANYSEARCH_ADAPTER],
|
|
22
|
+
[XCRAWL_ADAPTER.type, XCRAWL_ADAPTER],
|
|
23
|
+
]));
|
|
24
|
+
export function getAdapter(type, capability) {
|
|
25
|
+
const adapter = adapters.get(type);
|
|
26
|
+
return adapter?.capabilities.includes(capability) ? adapter : undefined;
|
|
27
|
+
}
|
|
28
|
+
export function getAdapters() {
|
|
29
|
+
return [...adapters.values()];
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/providers/registry.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE7C,MAAM,QAAQ,GAAG,CAAC,GAAG,eAAe,EAAE,GAAG,gBAAgB,CAAC,CAAC,MAAM,CAGhE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE;IAChB,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,CAAC,QAAQ;QAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;;QAE7C,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE;YACrB,GAAG,QAAQ;YACX,YAAY,EAAE;gBACb,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,YAAY,EAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;aAC/D;YACD,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM;YACzC,OAAO,EAAE,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO;SAC5C,CAAC,CAAC;IACJ,OAAO,GAAG,CAAC;AACZ,CAAC,EACD,IAAI,GAAG,CAAC;IACP,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC;IAC/B,CAAC,iBAAiB,CAAC,IAAI,EAAE,iBAAiB,CAAC;IAC3C,CAAC,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC;CACrC,CAAC,CACF,CAAC;AAEF,MAAM,UAAU,UAAU,CACzB,IAAkB,EAClB,UAAsB;IAEtB,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,OAAO,OAAO,EAAE,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,WAAW;IAC1B,OAAO,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/** 从 Next.js 内嵌的 React Server Components flight payload 中提取 Markdown。 */
|
|
2
|
+
export function extractRscMarkdown(html) {
|
|
3
|
+
if (!html.includes("self.__next_f.push"))
|
|
4
|
+
return "";
|
|
5
|
+
const chunks = new Map();
|
|
6
|
+
const scriptPattern = /self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)/g;
|
|
7
|
+
for (const match of html.matchAll(scriptPattern)) {
|
|
8
|
+
try {
|
|
9
|
+
const decoded = JSON.parse(`"${match[1]}"`);
|
|
10
|
+
for (const line of decoded.split("\n")) {
|
|
11
|
+
const separator = line.indexOf(":");
|
|
12
|
+
if (separator < 1 || separator > 6)
|
|
13
|
+
continue;
|
|
14
|
+
const id = line.slice(0, separator);
|
|
15
|
+
if (!/^[0-9a-f]+$/i.test(id))
|
|
16
|
+
continue;
|
|
17
|
+
const payload = line.slice(separator + 1);
|
|
18
|
+
if (!payload.startsWith("[") && !payload.startsWith("{"))
|
|
19
|
+
continue;
|
|
20
|
+
try {
|
|
21
|
+
chunks.set(id, JSON.parse(payload));
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
// 单个 chunk 损坏时继续处理其他 chunk。
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
// 非法脚本不是整个页面的致命错误。
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const active = new Set();
|
|
33
|
+
const render = (node, inCode = false) => {
|
|
34
|
+
if (node === null || node === undefined || typeof node === "boolean")
|
|
35
|
+
return "";
|
|
36
|
+
if (typeof node === "number")
|
|
37
|
+
return String(node);
|
|
38
|
+
if (typeof node === "string") {
|
|
39
|
+
const reference = /^\$L([0-9a-f]+)$/i.exec(node);
|
|
40
|
+
if (reference) {
|
|
41
|
+
const id = reference[1];
|
|
42
|
+
if (active.has(id))
|
|
43
|
+
return "";
|
|
44
|
+
active.add(id);
|
|
45
|
+
const result = render(chunks.get(id), inCode);
|
|
46
|
+
active.delete(id);
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
if (!inCode &&
|
|
50
|
+
(node === "$" || node === "$undefined" || /^\$[A-Z]/.test(node)))
|
|
51
|
+
return "";
|
|
52
|
+
return node;
|
|
53
|
+
}
|
|
54
|
+
if (!Array.isArray(node)) {
|
|
55
|
+
if (typeof node === "object") {
|
|
56
|
+
const object = node;
|
|
57
|
+
return render(object.children ?? object.content ?? object.text, inCode);
|
|
58
|
+
}
|
|
59
|
+
return "";
|
|
60
|
+
}
|
|
61
|
+
if (node[0] !== "$" || typeof node[1] !== "string")
|
|
62
|
+
return node.map((child) => render(child, inCode)).join("");
|
|
63
|
+
const tag = node[1];
|
|
64
|
+
const props = node[3] && typeof node[3] === "object"
|
|
65
|
+
? node[3]
|
|
66
|
+
: {};
|
|
67
|
+
if ([
|
|
68
|
+
"script",
|
|
69
|
+
"style",
|
|
70
|
+
"svg",
|
|
71
|
+
"path",
|
|
72
|
+
"nav",
|
|
73
|
+
"footer",
|
|
74
|
+
"aside",
|
|
75
|
+
"button",
|
|
76
|
+
"input",
|
|
77
|
+
].includes(tag))
|
|
78
|
+
return "";
|
|
79
|
+
const content = render(props.children, inCode).trim();
|
|
80
|
+
if (/^h[1-6]$/.test(tag))
|
|
81
|
+
return `${"#".repeat(Number(tag[1]))} ${content}\n\n`;
|
|
82
|
+
if (tag === "p")
|
|
83
|
+
return `${content}\n\n`;
|
|
84
|
+
if (tag === "li")
|
|
85
|
+
return `- ${content}\n`;
|
|
86
|
+
if (tag === "blockquote")
|
|
87
|
+
return `> ${content}\n\n`;
|
|
88
|
+
if (tag === "strong" || tag === "b")
|
|
89
|
+
return `**${content}**`;
|
|
90
|
+
if (tag === "em" || tag === "i")
|
|
91
|
+
return `*${content}*`;
|
|
92
|
+
if (tag === "code")
|
|
93
|
+
return inCode ? content : `\`${content}\``;
|
|
94
|
+
if (tag === "pre")
|
|
95
|
+
return `\`\`\`\n${render(props.children, true)}\n\`\`\`\n\n`;
|
|
96
|
+
if (tag === "a" &&
|
|
97
|
+
typeof props.href === "string" &&
|
|
98
|
+
!props.href.startsWith("#"))
|
|
99
|
+
return `[${content}](${props.href})`;
|
|
100
|
+
return content;
|
|
101
|
+
};
|
|
102
|
+
const candidates = [...chunks.values()]
|
|
103
|
+
.map((chunk) => render(chunk)
|
|
104
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
105
|
+
.trim())
|
|
106
|
+
.filter((content) => content.length > 100);
|
|
107
|
+
return candidates.sort((left, right) => right.length - left.length)[0] ?? "";
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=rsc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rsc.js","sourceRoot":"","sources":["../../src/providers/rsc.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QAAE,OAAO,EAAE,CAAC;IACpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC1C,MAAM,aAAa,GAAG,6CAA6C,CAAC;IACpE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC;YACJ,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAW,CAAC;YACtD,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpC,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC;oBAAE,SAAS;gBAC7C,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBACpC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAE,SAAS;gBACvC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACnE,IAAI,CAAC;oBACJ,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACR,4BAA4B;gBAC7B,CAAC;YACF,CAAC;QACF,CAAC;QAAC,MAAM,CAAC;YACR,mBAAmB;QACpB,CAAC;IACF,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,MAAM,GAAG,CAAC,IAAa,EAAE,MAAM,GAAG,KAAK,EAAU,EAAE;QACxD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS,IAAI,OAAO,IAAI,KAAK,SAAS;YACnE,OAAO,EAAE,CAAC;QACX,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,SAAS,GAAG,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,SAAS,EAAE,CAAC;gBACf,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBACxB,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAAE,OAAO,EAAE,CAAC;gBAC9B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACf,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;gBAC9C,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAClB,OAAO,MAAM,CAAC;YACf,CAAC;YACD,IACC,CAAC,MAAM;gBACP,CAAC,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,YAAY,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAEhE,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACb,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,IAA+B,CAAC;gBAC/C,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACzE,CAAC;YACD,OAAO,EAAE,CAAC;QACX,CAAC;QACD,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ;YACjD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,KAAK,GACV,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ;YACrC,CAAC,CAAE,IAAI,CAAC,CAAC,CAA6B;YACtC,CAAC,CAAC,EAAE,CAAC;QACP,IACC;YACC,QAAQ;YACR,OAAO;YACP,KAAK;YACL,MAAM;YACN,KAAK;YACL,QAAQ;YACR,OAAO;YACP,QAAQ;YACR,OAAO;SACP,CAAC,QAAQ,CAAC,GAAG,CAAC;YAEf,OAAO,EAAE,CAAC;QACX,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC;YACvB,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,MAAM,CAAC;QACvD,IAAI,GAAG,KAAK,GAAG;YAAE,OAAO,GAAG,OAAO,MAAM,CAAC;QACzC,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,KAAK,OAAO,IAAI,CAAC;QAC1C,IAAI,GAAG,KAAK,YAAY;YAAE,OAAO,KAAK,OAAO,MAAM,CAAC;QACpD,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,GAAG;YAAE,OAAO,KAAK,OAAO,IAAI,CAAC;QAC7D,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,GAAG;YAAE,OAAO,IAAI,OAAO,GAAG,CAAC;QACvD,IAAI,GAAG,KAAK,MAAM;YAAE,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,IAAI,CAAC;QAC/D,IAAI,GAAG,KAAK,KAAK;YAChB,OAAO,WAAW,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,cAAc,CAAC;QAC9D,IACC,GAAG,KAAK,GAAG;YACX,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;YAC9B,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAE3B,OAAO,IAAI,OAAO,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC;QACtC,OAAO,OAAO,CAAC;IAChB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;SACrC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACd,MAAM,CAAC,KAAK,CAAC;SACX,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CACR;SACA,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IAC5C,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9E,CAAC"}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { WebAccessError } from "../core/errors.js";
|
|
2
|
+
import { buildEndpoint, mergeHeaders } from "../transport/http.js";
|
|
3
|
+
import { assertOk, freshnessStartDate, normalizeHits, parseJsonResponse, providerHeaders, providerHeadersWithCredential, ref, requireBaseUrl, requireCredential, searchQueryWithDomains, } from "./common.js";
|
|
4
|
+
function requestBody(data) {
|
|
5
|
+
return JSON.stringify(data);
|
|
6
|
+
}
|
|
7
|
+
function mapSearchResult(response, request) {
|
|
8
|
+
const results = Array.isArray(response.results) ? response.results : [];
|
|
9
|
+
return {
|
|
10
|
+
results: normalizeHits(results, request.limit, request.includeDomains, request.excludeDomains),
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
function mapBraveResult(response, request) {
|
|
14
|
+
const web = response.web;
|
|
15
|
+
const results = web && typeof web === "object" && !Array.isArray(web)
|
|
16
|
+
? web.results
|
|
17
|
+
: [];
|
|
18
|
+
return {
|
|
19
|
+
results: normalizeHits(results, request.limit, request.includeDomains, request.excludeDomains),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
const tavily = {
|
|
23
|
+
type: "tavily",
|
|
24
|
+
capabilities: ["search"],
|
|
25
|
+
isConfigured: (instance) => !!instance.apiKey && !!instance.baseUrl,
|
|
26
|
+
async search(request) {
|
|
27
|
+
requireCredential(request.instance);
|
|
28
|
+
const body = {
|
|
29
|
+
query: request.query,
|
|
30
|
+
search_depth: "basic",
|
|
31
|
+
max_results: request.limit,
|
|
32
|
+
include_answer: false,
|
|
33
|
+
include_raw_content: false,
|
|
34
|
+
...(request.freshness ? { time_range: request.freshness } : {}),
|
|
35
|
+
...(request.includeDomains.length
|
|
36
|
+
? { include_domains: request.includeDomains }
|
|
37
|
+
: {}),
|
|
38
|
+
...(request.excludeDomains.length
|
|
39
|
+
? { exclude_domains: request.excludeDomains }
|
|
40
|
+
: {}),
|
|
41
|
+
};
|
|
42
|
+
const response = await request.transport.request(buildEndpoint(requireBaseUrl(request.instance), "search"), {
|
|
43
|
+
method: "POST",
|
|
44
|
+
headers: providerHeadersWithCredential(request.instance, mergeHeaders({
|
|
45
|
+
"Content-Type": "application/json",
|
|
46
|
+
Accept: "application/json",
|
|
47
|
+
})),
|
|
48
|
+
body: requestBody(body),
|
|
49
|
+
signal: request.signal,
|
|
50
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
51
|
+
});
|
|
52
|
+
assertOk(response, request.instance);
|
|
53
|
+
const parsed = parseJsonResponse(response, request.instance);
|
|
54
|
+
return { data: mapSearchResult(parsed, request), raw: parsed };
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
const exa = {
|
|
58
|
+
type: "exa",
|
|
59
|
+
capabilities: ["search", "extract"],
|
|
60
|
+
isConfigured: (instance) => !!instance.apiKey && !!instance.baseUrl,
|
|
61
|
+
async search(request) {
|
|
62
|
+
const key = requireCredential(request.instance);
|
|
63
|
+
const body = {
|
|
64
|
+
query: request.query,
|
|
65
|
+
type: "auto",
|
|
66
|
+
numResults: request.limit,
|
|
67
|
+
contents: { highlights: { maxCharacters: 500 } },
|
|
68
|
+
...(request.includeDomains.length
|
|
69
|
+
? { includeDomains: request.includeDomains }
|
|
70
|
+
: {}),
|
|
71
|
+
...(request.excludeDomains.length
|
|
72
|
+
? { excludeDomains: request.excludeDomains }
|
|
73
|
+
: {}),
|
|
74
|
+
...(request.freshness
|
|
75
|
+
? { startPublishedDate: freshnessStartDate(request.freshness) }
|
|
76
|
+
: {}),
|
|
77
|
+
};
|
|
78
|
+
const response = await request.transport.request(buildEndpoint(requireBaseUrl(request.instance), "search"), {
|
|
79
|
+
method: "POST",
|
|
80
|
+
headers: providerHeaders(request.instance, {
|
|
81
|
+
"Content-Type": "application/json",
|
|
82
|
+
Accept: "application/json",
|
|
83
|
+
"x-api-key": key,
|
|
84
|
+
}),
|
|
85
|
+
body: requestBody(body),
|
|
86
|
+
signal: request.signal,
|
|
87
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
88
|
+
});
|
|
89
|
+
assertOk(response, request.instance);
|
|
90
|
+
const parsed = parseJsonResponse(response, request.instance);
|
|
91
|
+
return { data: mapSearchResult(parsed, request), raw: parsed };
|
|
92
|
+
},
|
|
93
|
+
async extract(request) {
|
|
94
|
+
const key = requireCredential(request.instance);
|
|
95
|
+
const body = { urls: [request.url], text: true };
|
|
96
|
+
const response = await request.transport.request(buildEndpoint(requireBaseUrl(request.instance), "contents"), {
|
|
97
|
+
method: "POST",
|
|
98
|
+
headers: providerHeaders(request.instance, {
|
|
99
|
+
"Content-Type": "application/json",
|
|
100
|
+
Accept: "application/json",
|
|
101
|
+
"x-api-key": key,
|
|
102
|
+
}),
|
|
103
|
+
body: requestBody(body),
|
|
104
|
+
signal: request.signal,
|
|
105
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
106
|
+
});
|
|
107
|
+
assertOk(response, request.instance);
|
|
108
|
+
const parsed = parseJsonResponse(response, request.instance);
|
|
109
|
+
const items = Array.isArray(parsed.results) ? parsed.results : [];
|
|
110
|
+
const item = items.find((candidate) => candidate &&
|
|
111
|
+
typeof candidate === "object" &&
|
|
112
|
+
typeof candidate.text === "string");
|
|
113
|
+
if (!item || typeof item.text !== "string" || !item.text.trim())
|
|
114
|
+
throw new WebAccessError("no_usable_content", "Exa Contents 没有返回正文", { provider: ref(request.instance), retryable: true, raw: parsed });
|
|
115
|
+
return {
|
|
116
|
+
data: {
|
|
117
|
+
document: {
|
|
118
|
+
sourceUrl: typeof item.url === "string" ? item.url : request.url,
|
|
119
|
+
title: typeof item.title === "string" ? item.title : "",
|
|
120
|
+
content: item.text.trim(),
|
|
121
|
+
contentType: "text/markdown",
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
raw: parsed,
|
|
125
|
+
};
|
|
126
|
+
},
|
|
127
|
+
};
|
|
128
|
+
const brave = {
|
|
129
|
+
type: "brave",
|
|
130
|
+
capabilities: ["search"],
|
|
131
|
+
isConfigured: (instance) => !!instance.apiKey && !!instance.baseUrl,
|
|
132
|
+
async search(request) {
|
|
133
|
+
const key = requireCredential(request.instance);
|
|
134
|
+
const url = new URL(buildEndpoint(requireBaseUrl(request.instance), "res/v1/web/search"));
|
|
135
|
+
url.searchParams.set("q", searchQueryWithDomains(request.query, request.includeDomains, request.excludeDomains));
|
|
136
|
+
url.searchParams.set("count", String(request.includeDomains.length || request.excludeDomains.length
|
|
137
|
+
? 20
|
|
138
|
+
: request.limit));
|
|
139
|
+
if (request.freshness)
|
|
140
|
+
url.searchParams.set("freshness", { day: "pd", month: "pm", year: "py" }[request.freshness]);
|
|
141
|
+
const response = await request.transport.request(url, {
|
|
142
|
+
headers: providerHeaders(request.instance, {
|
|
143
|
+
Accept: "application/json",
|
|
144
|
+
"X-Subscription-Token": key,
|
|
145
|
+
}),
|
|
146
|
+
signal: request.signal,
|
|
147
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
148
|
+
});
|
|
149
|
+
assertOk(response, request.instance);
|
|
150
|
+
const parsed = parseJsonResponse(response, request.instance);
|
|
151
|
+
return { data: mapBraveResult(parsed, request), raw: parsed };
|
|
152
|
+
},
|
|
153
|
+
};
|
|
154
|
+
const searxng = {
|
|
155
|
+
type: "searxng",
|
|
156
|
+
capabilities: ["search"],
|
|
157
|
+
isConfigured: (instance) => !!instance.baseUrl,
|
|
158
|
+
async search(request) {
|
|
159
|
+
const url = new URL(buildEndpoint(requireBaseUrl(request.instance), "search"));
|
|
160
|
+
url.searchParams.set("q", searchQueryWithDomains(request.query, request.includeDomains, request.excludeDomains));
|
|
161
|
+
url.searchParams.set("format", "json");
|
|
162
|
+
if (request.freshness)
|
|
163
|
+
url.searchParams.set("time_range", request.freshness);
|
|
164
|
+
const response = await request.transport.request(url, {
|
|
165
|
+
headers: providerHeadersWithCredential(request.instance, {
|
|
166
|
+
Accept: "application/json",
|
|
167
|
+
}),
|
|
168
|
+
signal: request.signal,
|
|
169
|
+
maxResponseBytes: request.maxResponseBytes,
|
|
170
|
+
});
|
|
171
|
+
assertOk(response, request.instance);
|
|
172
|
+
const parsed = parseJsonResponse(response, request.instance);
|
|
173
|
+
return { data: mapSearchResult(parsed, request), raw: parsed };
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
export const SEARCH_ADAPTERS = [tavily, exa, brave, searxng];
|
|
177
|
+
export const EXA_ADAPTER = exa;
|
|
178
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/providers/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AASnD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACnE,OAAO,EACN,QAAQ,EACR,kBAAkB,EAClB,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,6BAA6B,EAC7B,GAAG,EACH,cAAc,EACd,iBAAiB,EACjB,sBAAsB,GACtB,MAAM,aAAa,CAAC;AAErB,SAAS,WAAW,CAAC,IAA6B;IACjD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CACvB,QAAiC,EACjC,OAA6B;IAE7B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IACxE,OAAO;QACN,OAAO,EAAE,aAAa,CACrB,OAAO,EACP,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,cAAc,CACtB;KACD,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CACtB,QAAiC,EACjC,OAA6B;IAE7B,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;IACzB,MAAM,OAAO,GACZ,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QACpD,CAAC,CAAE,GAA+B,CAAC,OAAO;QAC1C,CAAC,CAAC,EAAE,CAAC;IACP,OAAO;QACN,OAAO,EAAE,aAAa,CACrB,OAAO,EACP,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,cAAc,CACtB;KACD,CAAC;AACH,CAAC;AAED,MAAM,MAAM,GAAoB;IAC/B,IAAI,EAAE,QAAQ;IACd,YAAY,EAAE,CAAC,QAAQ,CAAC;IACxB,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO;IACnE,KAAK,CAAC,MAAM,CAAC,OAAO;QACnB,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,IAAI,GAA4B;YACrC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,YAAY,EAAE,OAAO;YACrB,WAAW,EAAE,OAAO,CAAC,KAAK;YAC1B,cAAc,EAAE,KAAK;YACrB,mBAAmB,EAAE,KAAK;YAC1B,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/D,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM;gBAChC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE;gBAC7C,CAAC,CAAC,EAAE,CAAC;YACN,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM;gBAChC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE;gBAC7C,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAC/C,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,EACzD;YACC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,6BAA6B,CACrC,OAAO,CAAC,QAAQ,EAChB,YAAY,CAAC;gBACZ,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;aAC1B,CAAC,CACF;YACD,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CACD,CAAC;QACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IAChE,CAAC;CACD,CAAC;AAEF,MAAM,GAAG,GAAoB;IAC5B,IAAI,EAAE,KAAK;IACX,YAAY,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC;IACnC,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO;IACnE,KAAK,CAAC,MAAM,CAAC,OAAO;QACnB,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAA4B;YACrC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,IAAI,EAAE,MAAM;YACZ,UAAU,EAAE,OAAO,CAAC,KAAK;YACzB,QAAQ,EAAE,EAAE,UAAU,EAAE,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE;YAChD,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM;gBAChC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE;gBAC5C,CAAC,CAAC,EAAE,CAAC;YACN,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM;gBAChC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE;gBAC5C,CAAC,CAAC,EAAE,CAAC;YACN,GAAG,CAAC,OAAO,CAAC,SAAS;gBACpB,CAAC,CAAC,EAAE,kBAAkB,EAAE,kBAAkB,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC/D,CAAC,CAAC,EAAE,CAAC;SACN,CAAC;QACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAC/C,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,EACzD;YACC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE;gBAC1C,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;gBAC1B,WAAW,EAAE,GAAG;aAChB,CAAC;YACF,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CACD,CAAC;QACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IAChE,CAAC;IACD,KAAK,CAAC,OAAO,CACZ,OAA8B;QAE9B,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAC/C,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAC,EAC3D;YACC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE;gBAC1C,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;gBAC1B,WAAW,EAAE,GAAG;aAChB,CAAC;YACF,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC;YACvB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CACD,CAAC;QACF,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CACtB,CAAC,SAAS,EAAE,EAAE,CACb,SAAS;YACT,OAAO,SAAS,KAAK,QAAQ;YAC7B,OAAQ,SAAqC,CAAC,IAAI,KAAK,QAAQ,CACzB,CAAC;QACzC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAC9D,MAAM,IAAI,cAAc,CACvB,mBAAmB,EACnB,qBAAqB,EACrB,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,CACjE,CAAC;QACH,OAAO;YACN,IAAI,EAAE;gBACL,QAAQ,EAAE;oBACT,SAAS,EAAE,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG;oBAChE,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;oBACvD,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACzB,WAAW,EAAE,eAAe;iBAC5B;aACD;YACD,GAAG,EAAE,MAAM;SACX,CAAC;IACH,CAAC;CACD,CAAC;AAEF,MAAM,KAAK,GAAoB;IAC9B,IAAI,EAAE,OAAO;IACb,YAAY,EAAE,CAAC,QAAQ,CAAC;IACxB,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO;IACnE,KAAK,CAAC,MAAM,CAAC,OAAO;QACnB,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,IAAI,GAAG,CAClB,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,mBAAmB,CAAC,CACpE,CAAC;QACF,GAAG,CAAC,YAAY,CAAC,GAAG,CACnB,GAAG,EACH,sBAAsB,CACrB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,cAAc,CACtB,CACD,CAAC;QACF,GAAG,CAAC,YAAY,CAAC,GAAG,CACnB,OAAO,EACP,MAAM,CACL,OAAO,CAAC,cAAc,CAAC,MAAM,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM;YAC7D,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,OAAO,CAAC,KAAK,CAChB,CACD,CAAC;QACF,IAAI,OAAO,CAAC,SAAS;YACpB,GAAG,CAAC,YAAY,CAAC,GAAG,CACnB,WAAW,EACX,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CACzD,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE;YACrD,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,QAAQ,EAAE;gBAC1C,MAAM,EAAE,kBAAkB;gBAC1B,sBAAsB,EAAE,GAAG;aAC3B,CAAC;YACF,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IAC/D,CAAC;CACD,CAAC;AAEF,MAAM,OAAO,GAAoB;IAChC,IAAI,EAAE,SAAS;IACf,YAAY,EAAE,CAAC,QAAQ,CAAC;IACxB,YAAY,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO;IAC9C,KAAK,CAAC,MAAM,CAAC,OAAO;QACnB,MAAM,GAAG,GAAG,IAAI,GAAG,CAClB,aAAa,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CACzD,CAAC;QACF,GAAG,CAAC,YAAY,CAAC,GAAG,CACnB,GAAG,EACH,sBAAsB,CACrB,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,cAAc,EACtB,OAAO,CAAC,cAAc,CACtB,CACD,CAAC;QACF,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,IAAI,OAAO,CAAC,SAAS;YACpB,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,EAAE;YACrD,OAAO,EAAE,6BAA6B,CAAC,OAAO,CAAC,QAAQ,EAAE;gBACxD,MAAM,EAAE,kBAAkB;aAC1B,CAAC;YACF,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;SAC1C,CAAC,CAAC;QACH,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7D,OAAO,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;IAChE,CAAC;CACD,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAsB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AAChF,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC"}
|