smart-web-mcp 1.10.6 → 1.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/shared.d.ts +9 -0
- package/dist/shared.js.map +1 -1
- package/dist/smartfetch/archive-fallback.d.ts +0 -8
- package/dist/smartfetch/archive-fallback.js +263 -125
- package/dist/smartfetch/archive-fallback.js.map +1 -1
- package/dist/smartfetch/provider-policy.d.ts +2 -0
- package/dist/smartfetch/provider-policy.js +3 -0
- package/dist/smartfetch/provider-policy.js.map +1 -1
- package/dist/smartfetch.js +51 -28
- package/dist/smartfetch.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"archive-fallback.js","sourceRoot":"","sources":["../../src/smartfetch/archive-fallback.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,kBAAkB,EAAE,WAAW,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACvH,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AAC3E,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AAExE,MAAM,UAAU,GAAG,iHAAiH,CAAA;AAEpI,SAAS,sBAAsB;IAC7B,OAAO,aAAa,CAAC,6BAA6B,EAAE,IAAI,CAAC,CAAA;AAC3D,CAAC;AAED,SAAS,cAAc;IACrB,OAAO,aAAa,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAA;AACnD,CAAC;AAED,SAAS,gBAAgB;IACvB,OAAO,aAAa,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAA;AACrD,CAAC;AAED,SAAS,aAAa;IACpB,OAAO,aAAa,CAAC,mBAAmB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,SAAiB;IACxD,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAC1C,6CAA6C,kBAAkB,CAAC,GAAG,CAAC,EAAE,EACtE,SAAS,EACT;QACE,OAAO,EAAE;YACP,MAAM,EAAE,iCAAiC;YACzC,YAAY,EAAE,UAAU;SACzB;KACF,EACD;QACE,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,GAAG;KACf,CACF,CAAA;IAED,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,CAAC;QACrB,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,OAAO,EAAsB;SACzI,CAAA;IACH,CAAC;IAED,MAAM,WAAW,GAAI,YAAY,CAAC,IAAY,EAAE,kBAAkB,EAAE,OAAO,EAAE,GAAG,CAAA;IAChF,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,0BAA0B,EAAE,OAAO,EAAE,+BAA+B,EAAsB;SACnI,CAAA;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CACtC,MAAM,CAAC,WAAW,CAAC,EACnB,SAAS,EACT;QACE,OAAO,EAAE;YACP,MAAM,EAAE,0BAA0B;YAClC,YAAY,EAAE,UAAU;SACzB;KACF,EACD;QACE,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,GAAG;KACf,CACF,CAAA;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAsB;SAC7H,CAAA;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,iBAAiB;YACzB,OAAO,EAAE,QAAQ,CAAC,IAAI;YACtB,KAAK,EAAE,UAAU,CAAC,CAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;SACjE;QACzB,KAAK,EAAE,IAAI;KACZ,CAAA;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,SAAiB;IAC1D,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CACtC,sBAAsB,GAAG,EAAE,EAC3B,SAAS,EACT;QACE,OAAO,EAAE;YACP,MAAM,EAAE,0BAA0B;YAClC,YAAY,EAAE,UAAU;SACzB;KACF,EACD;QACE,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,GAAG;KACf,CACF,CAAA;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAsB;SAC7H,CAAA;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE;YACN,EAAE,EAAE,IAAI;YACR,MAAM,EAAE,YAAY;YACpB,OAAO,EAAE,QAAQ,CAAC,IAAI;YACtB,KAAK,EAAE,UAAU,CAAC,CAAC,GAAG,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;SACjE;QACzB,KAAK,EAAE,IAAI;KACZ,CAAA;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,OAAsB,EAAE,SAAwB;IAC9E,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAC9C,MAAM,aAAa,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAClD,IAAI,aAAa,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,KAAK,CAAA;IAC5C,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,IAAI,6BAA6B,CAAC,aAAa,CAAC;QAAE,OAAO,KAAK,CAAA;IACnG,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,aAAa,CAAC,IAAI,iBAAiB,CAAC,SAAS,CAAC,CAAA;IACxF,IAAI,CAAC,gBAAgB;QAAE,OAAO,aAAa,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,CAAA;IACxE,OAAO,aAAa,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAA;AACtD,CAAC;AAED,SAAS,6BAA6B,CAAC,KAAa;IAClD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1E,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAA;IACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAA;IAC/C,MAAM,sBAAsB,GAAG,+DAA+D,CAAC,IAAI,CAAC,IAAI,CAAC;WACpG,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CAAC;WAC5D,OAAO,CAAC,QAAQ,CAAC,uCAAuC,CAAC,CAAA;IAC9D,MAAM,kBAAkB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;WACzF,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAA;IAC1E,sEAAsE;IACtE,OAAO,sBAAsB,IAAI,kBAAkB,CAAA;AACrD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,GAAW,EAAE,MAAqB,EAAE,SAAiB;IACjG,IAAI,CAAC,sBAAsB,EAAE,IAAI,aAAa,EAAE,EAAE,CAAC;QACjD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAiB,EAAE,CAAA;IAC9C,CAAC;IACD,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,EAAiB,EAAE,CAAA;IAC9C,CAAC;IAED,MAAM,MAAM,GAAgB,EAAE,CAAA;IAE9B,IAAI,cAAc,EAAE,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;QAClD,IAAI,OAAO,CAAC,MAAM,IAAI,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACrE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;QAC3C,CAAC;QACD,IAAI,OAAO,CAAC,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;IAC/C,CAAC;IAED,IAAI,gBAAgB,EAAE,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;QACtD,IAAI,SAAS,CAAC,MAAM,IAAI,sBAAsB,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACzE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;QAC7C,CAAC;QACD,IAAI,SAAS,CAAC,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACnD,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,8BAA8B,EAAE,OAAO,EAAE,uCAAuC,EAAE,CAAC,CAAA;IAClI,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC3B,CAAC"}
|
|
1
|
+
{"version":3,"file":"archive-fallback.js","sourceRoot":"","sources":["../../src/smartfetch/archive-fallback.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAEpC,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,kBAAkB,EAAE,WAAW,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACzH,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAA;AAExE,MAAM,WAAW,GAAG,CAAC,YAAY,EAAE,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC,CAAA;AACzH,MAAM,OAAO,GAAG,CAAC,aAAa,EAAE,iBAAiB,EAAE,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,IAAI,MAAM,CAAC,CAAA;AACrG,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,4CAA4C,EAAE,YAAY,EAAE,4CAA4C,EAAE,CAAA;AAEpI,SAAS,KAAK,CAAC,IAAY,EAAE,OAAe,EAAE,QAAQ,GAA0B,aAAa;IAC3F,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;AACpC,CAAC;AAED,4EAA4E;AAC5E,oDAAoD;AACpD,SAAS,YAAY,CAAC,SAAiB,EAAE,QAAgB;IACvD,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAA;QACnD,CAAC,CAAC,IAAI,GAAG,EAAE,CAAA;QACX,CAAC,CAAC,IAAI,GAAG,EAAE,CAAA;QACX,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAA;IACxF,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,KAAK,CAAA;IAAC,CAAC;AAC1B,CAAC;AAED,SAAS,WAAW,CAAC,MAAc;IACjC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,CAAA;IACvC,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAA;IAC3J,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAA;IAC1B,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAA;AACjF,CAAC;AAED,SAAS,cAAc,CAAC,KAAa,EAAE,QAAgB,EAAE,QAAuC;IAC9F,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;QAC1B,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAA;QAC5F,MAAM,SAAS,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,KAAK,iBAAiB,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QAClH,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAA;QAC3B,MAAM,KAAK,GAAG,QAAQ,KAAK,SAAS;YAClC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,gDAAgD,CAAC;YACtE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAA;QACjF,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAA;QACvB,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAA;QAC/D,IAAI,CAAC,UAAU,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAA;QACzF,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAA;QACvB,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,CAAA;IAC9C,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,IAAI,CAAA;IAAC,CAAC;AACzB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAAY;IACnD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QAChC,OAAO,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;eACnF,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI;YACxG,CAAC,CAAC,WAAW,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IACnD,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,EAAE,CAAA;IAAC,CAAC;AACvB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,QAAgB;IACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IACpC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,gBAAgB,CAAC,uCAAuC,CAAC;QAAE,IAAI,CAAC,MAAM,EAAE,CAAA;IACpG,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;QACxD,IAAI,oGAAoG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAAE,IAAI,CAAC,MAAM,EAAE,CAAA;IAChK,CAAC;IACD,IAAI,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC,gEAAgE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,IAAI,iDAAiD,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7R,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,IAAI,EAAE,CAAA;IAChE,IAAI,yIAAyI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7K,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,gBAAgB,CAAC,+CAA+C,CAAC,EAAE,CAAC;QAC9F,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAChF,IAAI,QAAQ,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,eAAe,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,EAAE,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAA;IACnN,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,gBAAgB,CAAC,yHAAyH,CAAC;QAAE,IAAI,CAAC,MAAM,EAAE,CAAA;IACtL,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,4BAA4B,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAA;IACxH,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACjE,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,6UAA6U,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IAC9X,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;IACpG,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,IAAI,CAAA;IAC3C,OAAO,IAAI,CAAC,SAAS,CAAA;AACvB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAe,EAAE,QAAgB,EAAE,QAAuC,EAAE,WAAmB,EAAE,UAAkB;IAC7I,OAAO;QACL,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,QAAQ;QAChG,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;QACrF,OAAO,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE;KACpH,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,GAAW,EAAE,MAAqB,EAAE,SAAiB;IACjG,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,IAAI,CAAC,aAAa,CAAC,6BAA6B,EAAE,IAAI,CAAC,IAAI,aAAa,CAAC,mBAAmB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,GAAG,EAAE,MAAM,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;IAC3L,4EAA4E;IAC5E,wEAAwE;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAA;IACvE,MAAM,cAAc,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAA;IAC5C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,MAAsC,CAAA;IAC1C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;QAC1C,MAAM,GAAG,UAAU,CAAC,GAAG,EAAE;YACvB,UAAU,CAAC,KAAK,EAAE,CAAA;YAClB,OAAO,CAAC,IAAI,CAAC,CAAA;QACf,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IACF,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC/C,MAAM,cAAc,GAAG,CAAC,KAAa,EAAE,EAAE;QACvC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAA;YAC7B,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAA;YACnG,IAAI,MAAM,CAAC,QAAQ,KAAK,aAAa;gBAAE,OAAO,MAAM,CAAC,QAAQ,KAAK,oBAAoB,IAAI,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,KAAK,CAAC,CAAA;YAC/L,IAAI,MAAM,CAAC,QAAQ,KAAK,iBAAiB;gBAAE,OAAO,OAAO,CAAC,cAAc,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;YAChG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAAE,OAAO,KAAK,CAAA;YACxD,OAAO,CAAC,WAAW,cAAc,EAAE,EAAE,IAAI,cAAc,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,cAAc,CAAC,KAAK,EAAE,GAAG,EAAE,eAAe,CAAC,CAAC,CAAA;QAChM,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAA;QAAC,CAAC;IAC1B,CAAC,CAAA;IACD,MAAM,OAAO,GAAG,KAAK,EAAE,QAAgB,EAAE,EAAE;QACzC,IAAI,SAAS,EAAE,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAA;QAC9D,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAA;QACnC,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;QACjE,IAAI,KAAqC,CAAA;QACzC,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;YAC3C,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBACtB,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,OAAO,CAAC,IAAI,CAAC,CAAA;YACf,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QACF,IAAI,CAAC;YACH,sEAAsE;YACtE,qEAAqE;YACrE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAA;YACtP,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,yBAAyB,EAAE,4CAA4C,EAAE,SAAS,CAAC,CAAC,CAAA;gBACtG,OAAO,IAAI,CAAA;YACb,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;gBAC1E,OAAO,IAAI,CAAA;YACb,CAAC;YACD,OAAO,QAAQ,CAAA;QACjB,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,KAAK,CAAC,KAAK,EAAE,CAAA;QACf,CAAC;IACH,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,KAAK,IAAmC,EAAE;QACrD,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,GAAG,CAAC,CAAA;QACjD,IAAI,SAAS,EAAE,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAA;QAC9D,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAA;YACzE,OAAO,IAAI,CAAA;QACb,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,QAAuC,EAAE,WAAmB,EAAE,UAAkB,EAAE,EAAE;YAChH,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;YAC3C,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC5I,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,6BAA6B,EAAE,mEAAmE,CAAC,CAAC,CAAA;gBACtH,OAAO,IAAI,CAAA;YACb,CAAC;YACD,OAAO,kBAAkB,CAAC,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAC,CAAA;QAC5E,CAAC,CAAA;QACD,IAAI,aAAa,CAAC,qBAAqB,EAAE,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,6CAA6C,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;YACvG,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,OAAgB,CAAA;gBACpB,IAAI,CAAC;oBAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAA;gBAAC,CAAC;gBAAC,MAAM,CAAC;oBAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,yBAAyB,EAAE,yCAAyC,CAAC,CAAC,CAAA;gBAAC,CAAC;gBAC5K,MAAM,QAAQ,GAAG,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAkC,CAAC,CAAC,CAAC,EAAE,CAAA;gBACjG,MAAM,MAAM,GAAG,OAAO,QAAQ,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;gBACrG,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC,SAAS,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,KAAK,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,IAAI,QAAQ,CAAC,SAAS,KAAK,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC;oBACzP,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,oDAAoD,CAAC,CAAC,CAAA;gBACtG,CAAC;qBAAM,CAAC;oBACN,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;oBAClD,MAAM,MAAM,GAAG,QAAQ,IAAI,cAAc,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;oBAC7E,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;wBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;wBACtF,IAAI,MAAM;4BAAE,OAAO,MAAM,CAAA;oBAC3B,CAAC;yBAAM,IAAI,QAAQ;wBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,2BAA2B,EAAE,gDAAgD,CAAC,CAAC,CAAA;gBACxH,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,aAAa,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC;YACjD,6EAA6E;YAC7E,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;gBAC/B,IAAI,SAAS,EAAE,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAK;gBACxD,IAAI,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,IAAI,WAAW,cAAc,EAAE,CAAC,CAAA;gBACtE,mEAAmE;gBACnE,mEAAmE;gBACnE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,+BAA+B,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,KAAK,YAAY;oBAAE,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,IAAI,IAAI,cAAc,EAAE,CAAC,CAAA;gBAC9K,IAAI,CAAC,MAAM;oBAAE,SAAQ;gBACrB,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAA;gBACtB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAA;gBAC3C,IAAI,QAAQ,GAAG,CAAC,CAAA;gBAChB,OAAO,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,GAAG,CAAC,EAAE,CAAC;oBACvC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAG,CAAA;oBAC3B,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;oBACzC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAA;oBAC1D,MAAM,QAAQ,GAAG,QAAQ,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;oBACnH,MAAM,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC,WAAW,CAAC,CAAA;oBACxD,MAAM,QAAQ,GAAG,YAAY,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;oBAC/H,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,eAAe,CAAC,IAAI,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,eAAe,CAAC,CAAA;oBACrH,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;oBAC9D,IAAI,MAAM,IAAI,CAAC,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;wBAC5H,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;wBAC7F,IAAI,MAAM;4BAAE,OAAO,MAAM,CAAA;oBAC3B,CAAC;oBACD,sEAAsE;oBACtE,uEAAuE;oBACvE,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;wBAC1D,IAAI,QAAQ,IAAI,CAAC,IAAI,SAAS,EAAE,IAAI,CAAC;4BAAE,MAAK;wBAC5C,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;wBAC9C,IAAI,QAAQ,GAAG,EAAE,CAAA;wBACjB,IAAI,CAAC;4BAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAA;wBAAC,CAAC;wBAAC,MAAM,CAAC;4BAAC,SAAQ;wBAAC,CAAC;wBACxE,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,eAAe,CAAC,EAAE,WAAW,CAAA;wBACvH,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;4BAAE,SAAQ;wBAClD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;wBACtB,QAAQ,EAAE,CAAA;wBACV,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,CAAA;wBACzC,IAAI,QAAQ;4BAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC,CAAA;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC,CAAA;QACpD,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,SAAS,EAAE,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,2BAA2B,EAAE,0CAA0C,EAAE,SAAS,CAAC,CAAC,CAAA;QACzJ,IAAI,MAAM,IAAI,SAAS,EAAE,GAAG,CAAC;YAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;QAChE,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,8BAA8B,EAAE,4DAA4D,CAAC,CAAC,CAAA;QACpI,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,EAAE,CAAA;IACxC,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,MAAM,CAAC,CAAA;QACpB,UAAU,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;AACH,CAAC"}
|
|
@@ -6,6 +6,8 @@ type ProviderRequestOptions = {
|
|
|
6
6
|
allowedOrigins?: string[];
|
|
7
7
|
/** Optional absolute budget shared by JSON policy, redirects and body reads. */
|
|
8
8
|
deadlineAt?: number;
|
|
9
|
+
/** Optional route guard, applied before every request including redirects. */
|
|
10
|
+
allowedUrl?: (url: string) => boolean;
|
|
9
11
|
};
|
|
10
12
|
type ProviderFailure = {
|
|
11
13
|
ok: false;
|
|
@@ -41,6 +41,9 @@ function sanitizeCrossOriginRedirectHeaders(headers) {
|
|
|
41
41
|
return next;
|
|
42
42
|
}
|
|
43
43
|
async function enforcePolicy(url, options) {
|
|
44
|
+
if (options.allowedUrl && !options.allowedUrl(url)) {
|
|
45
|
+
return { ok: false, error: { category: "block", code: "provider_route_blocked", message: "Provider URL is not an allowed retrieval route" } };
|
|
46
|
+
}
|
|
44
47
|
const validation = await validateProviderUrl(url, options);
|
|
45
48
|
if (!validation.ok) {
|
|
46
49
|
return { ok: false, error: { category: "block", code: validation.reason, message: validation.message } };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provider-policy.js","sourceRoot":"","sources":["../../src/smartfetch/provider-policy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"provider-policy.js","sourceRoot":"","sources":["../../src/smartfetch/provider-policy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAoCjE,SAAS,aAAa;IACpB,OAAO,aAAa,CAAC,mBAAmB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;AACrE,CAAC;AAED,SAAS,mBAAmB,CAAC,GAAW,EAAE,OAA+B;IACvE,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAA;AACjC,CAAC;AAED,SAAS,eAAe,CAAC,GAAQ;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IACnG,OAAO,GAAG,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAA;AACnD,CAAC;AAED,SAAS,YAAY,CAAC,SAAiB,EAAE,SAAiB;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAA;QACjC,OAAO,eAAe,CAAC,MAAM,CAAC,KAAK,eAAe,CAAC,MAAM,CAAC,CAAA;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,SAAiB,EAAE,OAA+B;IACzE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAA;QAClD,OAAO,MAAM,KAAK,eAAe,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAClH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,SAAS,kCAAkC,CAAC,OAAgC;IAC1E,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IAC3C,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;IACrC,MAAM,IAAI,GAAG,IAAI,OAAO,EAAE,CAAA;IAC1B,KAAK,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAChC,IAAI,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAClC,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,OAA+B;IACvE,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACnD,OAAO,EAAE,EAAE,EAAE,KAAc,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAgB,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,gDAAgD,EAAE,EAAE,CAAA;IACjK,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,mBAAmB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IAC1D,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,OAAO,EAAE,EAAE,EAAE,KAAc,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAgB,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,EAAE,CAAA;IAC5H,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;QACrE,OAAO;YACL,EAAE,EAAE,KAAc;YAClB,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAgB,EAAE,IAAI,EAAE,+BAA+B,EAAE,OAAO,EAAE,8CAA8C,EAAE;SACtI,CAAA;IACH,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,IAAI,aAAa,EAAE,EAAE,CAAC;QAChD,OAAO;YACL,EAAE,EAAE,KAAc;YAClB,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAgB,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,+DAA+D,EAAE;SACnJ,CAAA;IACH,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAa,EAAE,CAAA;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,SAAiB,EAAE,IAA6B,EAAE,OAA+B;IACpI,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAA;IACnG,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAA;IAC7D,IAAI,OAAO,GAAG,GAAG,CAAA;IACjB,IAAI,CAAC;QACH,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;YAClC,MAAM,CAAC,cAAc,EAAE,CAAA;YACvB,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YACrD,IAAI,CAAC,OAAO,CAAC,EAAE;gBAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAA;YAChF,MAAM,CAAC,cAAc,EAAE,CAAA;YACvB,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,kCAAkC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAC9G,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;YAC/G,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;gBACjD,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;gBAC5C,IAAI,GAAG,KAAK,CAAC;oBAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,6BAA6B,EAAE,EAAE,CAAA;gBACxK,IAAI,CAAC,QAAQ;oBAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,oCAAoC,EAAE,EAAE,CAAA;gBACtL,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAA;gBAC/C,SAAQ;YACV,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAA;gBAC5D,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;gBAC5C,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,aAAa,UAAU,CAAC,CAAC,CAAC,iBAAiB,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAA;YACnL,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YAClC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,UAAU,QAAQ,CAAC,MAAM,EAAE,EAAE,EAAE,CAAA;YAC9K,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,CAAA;QACxE,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,6BAA6B,EAAE,EAAE,CAAA;IAC7I,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAA;IAC/K,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAA;IACrB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,SAAiB,EAAE,IAA6B,EAAE,OAA+B;IACpI,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,wBAAwB,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;IACpG,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACjD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,IAAI,EAAE,MAAM,EAAE,OAAO;QAAE,OAAO,uBAAuB,EAAE,CAAA;IAC5G,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,OAAiC,CAAA;IACrC,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAA;IACnG,IAAI,CAAC;QACH,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC;YACxB,wBAAwB,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC;YACtE,IAAI,OAAO,CAAkB,OAAO,CAAC,EAAE;gBACrC,OAAO,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAAA;gBAClD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;gBACzD,IAAI,MAAM,CAAC,OAAO;oBAAE,OAAO,EAAE,CAAA;gBAC7B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAA;YACzD,CAAC,CAAC;SACH,CAAC,CAAA;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,OAAO;YAAE,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IAC3D,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB;IAC9B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,4BAA4B,EAAE,OAAO,EAAE,yDAAyD,EAAE,EAAE,CAAA;AAC/L,CAAC;AAED,KAAK,UAAU,wBAAwB,CAAC,GAAW,EAAE,SAAiB,EAAE,IAA6B,EAAE,OAA+B;IACpI,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC,UAAU,CAAC,CAAA;IACrH,IAAI,OAAO,EAAE;QAAE,OAAO,uBAAuB,EAAE,CAAA;IAC/C,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAA;IACrE,IAAI,OAAO,EAAE;QAAE,OAAO,uBAAuB,EAAE,CAAA;IAC/C,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,EAAE,GAAG,MAAM,EAAE,IAAI,EAAE,IAAe,EAAE,CAAA;IAC3D,IAAI,CAAC;QACH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACnH,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,CAAA;IAChN,CAAC;AACH,CAAC"}
|
package/dist/smartfetch.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { absolutizeUrl, assetFromUrl, asString, cleanNormalizedLinks, dedupeUrls, configFlag, extractAnchorHrefs, extractUrls, inferTarget, needsBotChallengeBrowserUpgrade, needsDynamicCrawl, hasActiveRetryAfter, needsSpaBrowserFallback, resolveValidatedUrl, validateOutboundUrl, } from "./shared.js";
|
|
1
|
+
import { absolutizeUrl, assetFromUrl, asString, cleanNormalizedLinks, dedupeUrls, configFlag, extractAnchorHrefs, extractUrls, inferTarget, needsBotChallengeBrowserUpgrade, needsDynamicCrawl, hasActiveRetryAfter, needsSpaBrowserFallback, resolveValidatedUrl, stripTags, validateOutboundUrl, } from "./shared.js";
|
|
2
2
|
import { createAssessment, hasTerminalSafetyBlock } from "./assessment.js";
|
|
3
3
|
import { runBrowserSession, resolveProfileDir } from "./browser-session.js";
|
|
4
4
|
import { projectSmartfetchOutput, resolveSmartfetchBudgetPolicy } from "./extraction/budget.js";
|
|
@@ -10,6 +10,7 @@ import { maybeUseAcademicFallback } from "./smartfetch/academic-fallback.js";
|
|
|
10
10
|
import { maybeUseAdaptiveScraper } from "./smartfetch/adaptive-scraper.js";
|
|
11
11
|
import { discoverAssets } from "./smartfetch/assets.js";
|
|
12
12
|
import { maybeUseArchiveFallback } from "./smartfetch/archive-fallback.js";
|
|
13
|
+
import { hasPaywallMarker, isKnownPaywallHost, looksPaywalledOrBlocked } from "./smartfetch/paywall.js";
|
|
13
14
|
import { maybeUseJinaReader } from "./smartfetch/jina-reader.js";
|
|
14
15
|
import { extractGenericDiscussion } from "./smartfetch/generic-discussion.js";
|
|
15
16
|
import { runCoupang } from "./korea/coupang.js";
|
|
@@ -889,6 +890,32 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
889
890
|
}
|
|
890
891
|
}
|
|
891
892
|
let active;
|
|
893
|
+
let archiveTried = false;
|
|
894
|
+
const recoverArchive = async (budget) => {
|
|
895
|
+
archiveTried = true;
|
|
896
|
+
const beforeMethod = active.method;
|
|
897
|
+
const archived = await maybeUseArchiveFallback(active.final_url || options.url, active, budget);
|
|
898
|
+
remaining();
|
|
899
|
+
active = archived.active;
|
|
900
|
+
output.errors.push(...archived.errors);
|
|
901
|
+
const archiveErrors = archived.errors.filter((error) => /archive|wayback/i.test(error.code));
|
|
902
|
+
const archiveSucceeded = Boolean(active.archive);
|
|
903
|
+
archiveAttempt.attempted = archiveSucceeded || archiveErrors.length > 0 || active.method !== beforeMethod;
|
|
904
|
+
archiveAttempt.ok = archiveSucceeded;
|
|
905
|
+
if (archiveSucceeded)
|
|
906
|
+
archiveAttempt.method = active.method;
|
|
907
|
+
else
|
|
908
|
+
delete archiveAttempt.method;
|
|
909
|
+
archiveAttempt.reason = archiveSucceeded
|
|
910
|
+
? "Archive fallback improved weak or blocked direct retrieval"
|
|
911
|
+
: archiveErrors.length > 0
|
|
912
|
+
? archiveErrors[0]?.message || "Archive fallback did not improve the page"
|
|
913
|
+
: "Archive fallback was eligible but was not used or not needed";
|
|
914
|
+
if (archiveSucceeded) {
|
|
915
|
+
usedArchive = true;
|
|
916
|
+
output.retrieval_method.push(active.method);
|
|
917
|
+
}
|
|
918
|
+
};
|
|
892
919
|
let expectedPostUrl = nativeNaverTarget(options.url) === target ? options.url : undefined;
|
|
893
920
|
if (acquirePlan.lane === "browser") {
|
|
894
921
|
const nativeUrl = target === "naver_blog" ? naverBlogMobilePostUrl(options.url)
|
|
@@ -963,6 +990,16 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
963
990
|
const needsRedirectBrowserUpgrade = Boolean(redirectUrl)
|
|
964
991
|
&& (redirectProvider.capabilities.includes("requires_browser") || redirectProvider.capabilities.includes("interactive_surface"));
|
|
965
992
|
const mustNotRetry = hasTerminalSafetyBlock([active.error]) || active.http_status === 429 || hasActiveRetryAfter(active.retry_after);
|
|
993
|
+
const archiveUrl = active.final_url || options.url;
|
|
994
|
+
// An existing public capture is independent of an origin login wall. Try
|
|
995
|
+
// it before the browser can exhaust the budget or establish participation.
|
|
996
|
+
// Ordinary transport/SPA failures retain browser-first recovery.
|
|
997
|
+
if (!mustNotRetry && !needsRedirectBrowserUpgrade && !profileDir && target === "generic"
|
|
998
|
+
&& acquirePlan.allowArchiveFallback && redirectProvider.policy.allowArchiveFallback
|
|
999
|
+
&& looksPaywalledOrBlocked(archiveUrl, active)
|
|
1000
|
+
&& (isKnownPaywallHost(archiveUrl) || hasPaywallMarker(stripTags(active.content)))) {
|
|
1001
|
+
await recoverArchive(configText("fetch.browserCdpUrl") ? Math.max(1, Math.floor(remaining() * 0.6)) : remaining());
|
|
1002
|
+
}
|
|
966
1003
|
if (mustNotRetry) {
|
|
967
1004
|
browserAttempt.reason = hasTerminalSafetyBlock([active.error])
|
|
968
1005
|
? "terminal URL safety rejection — no browser continuation"
|
|
@@ -989,7 +1026,7 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
989
1026
|
if (browserResult.ok || expectedPostUrl || hasTerminalSafetyBlock([browserResult.error]))
|
|
990
1027
|
active = browserResult;
|
|
991
1028
|
}
|
|
992
|
-
else if (operatorRecoveryReason(active) || needsBotChallengeBrowserUpgrade(active)) {
|
|
1029
|
+
else if (!usedArchive && (operatorRecoveryReason(active) || needsBotChallengeBrowserUpgrade(active))) {
|
|
993
1030
|
// Direct-client failure does not prove the approved browser cannot read it.
|
|
994
1031
|
const browserResult = await runPlaywrightFetchWithRuntime(options.url, remaining(), target, runtime, profileDir, options.wait_for, options.includeComments);
|
|
995
1032
|
const rejected = target === "generic" ? await genericBrowserRecoveryReason(browserResult, options.url) : "";
|
|
@@ -1048,7 +1085,7 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
1048
1085
|
// A transport-success shell is not a fulfilled generic read. Use the approved
|
|
1049
1086
|
// owned browser once before optional relays, without asking for force_dynamic.
|
|
1050
1087
|
// Preserve cheap structured/feed recovery and real participation boundaries.
|
|
1051
|
-
if (target === "generic" && effectiveLane === "direct" && !browserAttempt.attempted && !suppressRetries
|
|
1088
|
+
if (target === "generic" && effectiveLane === "direct" && !usedArchive && !browserAttempt.attempted && !suppressRetries
|
|
1052
1089
|
&& !options.includeComments && !new URL(options.url).hash
|
|
1053
1090
|
&& !profileDir && configText("fetch.browserCdpUrl") && active.ok
|
|
1054
1091
|
&& !/self\.__next_f\.push\(|<script[^>]*id=["']__NEXT_DATA__["']|<link[^>]*type=["']application\/(?:rss|atom)\+xml["']/i.test(active.content)) {
|
|
@@ -1130,6 +1167,7 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
1130
1167
|
&& effectiveLane === "direct"
|
|
1131
1168
|
&& Boolean(archiveProvider.policy.allowArchiveFallback);
|
|
1132
1169
|
const allowJinaReader = acquirePlan.allowJinaReader
|
|
1170
|
+
&& !usedArchive
|
|
1133
1171
|
&& !suppressRetries
|
|
1134
1172
|
&& !lockedBlindPreview
|
|
1135
1173
|
&& effectiveLane === "direct"
|
|
@@ -1139,7 +1177,7 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
1139
1177
|
? "Jina Reader relay was not eligible for the effective provider/lane"
|
|
1140
1178
|
: "Jina Reader relay was not allowed by plan, privacy profile, or settings";
|
|
1141
1179
|
}
|
|
1142
|
-
if (!allowArchiveFallback) {
|
|
1180
|
+
if (!allowArchiveFallback && !archiveTried) {
|
|
1143
1181
|
archiveAttempt.reason = acquirePlan.allowArchiveFallback
|
|
1144
1182
|
? "archive fallback was not eligible for the effective provider/lane"
|
|
1145
1183
|
: "archive fallback was not allowed by plan, privacy profile, or settings";
|
|
@@ -1168,30 +1206,10 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
1168
1206
|
if (jinaSucceeded)
|
|
1169
1207
|
output.retrieval_method.push(active.method);
|
|
1170
1208
|
}
|
|
1171
|
-
if (allowArchiveFallback && effectiveLane === "direct") {
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
remaining();
|
|
1175
|
-
active = archived.active;
|
|
1176
|
-
output.errors.push(...archived.errors);
|
|
1177
|
-
const archiveErrors = archived.errors.filter((error) => /archive|wayback/i.test(error.code));
|
|
1178
|
-
const archiveSucceeded = active.method === "wayback_archive" || active.method === "archive_md";
|
|
1179
|
-
archiveAttempt.attempted = archiveSucceeded || archiveErrors.length > 0 || active.method !== beforeMethod;
|
|
1180
|
-
archiveAttempt.ok = archiveSucceeded;
|
|
1181
|
-
if (archiveSucceeded)
|
|
1182
|
-
archiveAttempt.method = active.method;
|
|
1183
|
-
else
|
|
1184
|
-
delete archiveAttempt.method;
|
|
1185
|
-
archiveAttempt.reason = archiveSucceeded
|
|
1186
|
-
? "Archive fallback improved weak or blocked direct retrieval"
|
|
1187
|
-
: archiveErrors.length > 0
|
|
1188
|
-
? archiveErrors[0]?.message || "Archive fallback did not improve the page"
|
|
1189
|
-
: "Archive fallback was eligible but was not used or not needed";
|
|
1190
|
-
if (archiveSucceeded) {
|
|
1209
|
+
if (allowArchiveFallback && effectiveLane === "direct" && !archiveTried) {
|
|
1210
|
+
await recoverArchive(remaining());
|
|
1211
|
+
if (usedArchive)
|
|
1191
1212
|
genericRecoveryIncomplete = false;
|
|
1192
|
-
usedArchive = true;
|
|
1193
|
-
output.retrieval_method.push(active.method);
|
|
1194
|
-
}
|
|
1195
1213
|
}
|
|
1196
1214
|
if (effectiveLane === "direct" && !suppressRetries && (needsDynamicCrawl(active) || needsSpaBrowserFallback(active))) {
|
|
1197
1215
|
const scraped = await maybeUseAdaptiveScraper(active.final_url || options.url, active, remaining());
|
|
@@ -1343,6 +1361,8 @@ async function runSmartfetchTask(options, runtime, deadlineAt, output) {
|
|
|
1343
1361
|
}
|
|
1344
1362
|
output.target = effectiveTarget;
|
|
1345
1363
|
output.risk_level = riskLevelForTarget(effectiveTarget);
|
|
1364
|
+
if (active.archive)
|
|
1365
|
+
output.archive = { ...active.archive };
|
|
1346
1366
|
output.pipeline.resolve = buildResolveStage({
|
|
1347
1367
|
url: options.url,
|
|
1348
1368
|
requestedTarget,
|
|
@@ -1639,6 +1659,7 @@ export function renderSmartfetch(output, format) {
|
|
|
1639
1659
|
const post = projected.post || {};
|
|
1640
1660
|
const title = String(post?.title || "").trim();
|
|
1641
1661
|
const text = String(post?.text || post?.selftext || "").trim();
|
|
1662
|
+
const archiveReceipt = projected.archive ? `Historical archive snapshot (${projected.archive.provider}); captured_at=${projected.archive.captured_at}\nOriginal: ${projected.archive.original_url}\nSnapshot: ${projected.archive.snapshot_url}\nNot live content; the archived copy may be stale.` : "";
|
|
1642
1663
|
const exported = projected.comment_export;
|
|
1643
1664
|
const exportReceipt = exported ? `Comment export: ${exported.path}\ncomments=${exported.comments_count} complete=${exported.complete} bytes=${exported.bytes} sha256=${exported.sha256}` : "";
|
|
1644
1665
|
const bodyExported = projected.body_export;
|
|
@@ -1708,7 +1729,7 @@ export function renderSmartfetch(output, format) {
|
|
|
1708
1729
|
const continuation = assessment.continuation ? `continuation: ${assessment.continuation.status} — ${assessment.continuation.reason}` : "";
|
|
1709
1730
|
const budget = projected.budget?.truncated ? `budget: truncated fields=${Object.keys(projected.budget.fields).join(",")}` : "";
|
|
1710
1731
|
const compactBudget = renderBudget ? `text_view: ${fullPolicy ? "full" : "compact"} renderer omitted ${renderBudget}` : "";
|
|
1711
|
-
return [title, meta, exportReceipt, bodyExportReceipt, tocText, textBody.text, results.join("\n"), comments.join("\n"), linkItems.visible.join("\n"), download, assets.join("\n"), handoff, continuation, budget, compactBudget].filter(Boolean).join("\n\n").trim();
|
|
1732
|
+
return [title, meta, archiveReceipt, exportReceipt, bodyExportReceipt, tocText, textBody.text, results.join("\n"), comments.join("\n"), linkItems.visible.join("\n"), download, assets.join("\n"), handoff, continuation, budget, compactBudget].filter(Boolean).join("\n\n").trim();
|
|
1712
1733
|
}
|
|
1713
1734
|
const parts = [];
|
|
1714
1735
|
const url = String(post?.url || projected.url || "").trim();
|
|
@@ -1731,6 +1752,8 @@ export function renderSmartfetch(output, format) {
|
|
|
1731
1752
|
parts.push(`# ${title}`);
|
|
1732
1753
|
if (url)
|
|
1733
1754
|
parts.push(`Source: ${url}`);
|
|
1755
|
+
if (archiveReceipt)
|
|
1756
|
+
parts.push(archiveReceipt);
|
|
1734
1757
|
if (exportReceipt)
|
|
1735
1758
|
parts.push(exportReceipt);
|
|
1736
1759
|
if (bodyExportReceipt)
|