webpeel 0.14.3 → 0.14.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +11 -4
- package/dist/cache.js.map +1 -1
- package/dist/cli.bundle.cjs +159248 -0
- package/dist/core/agent.js +12 -8
- package/dist/core/agent.js.map +1 -1
- package/dist/core/application-tracker.js +3 -2
- package/dist/core/application-tracker.js.map +1 -1
- package/dist/core/auto-extract.js +6 -4
- package/dist/core/auto-extract.js.map +1 -1
- package/dist/core/browser-fetch.d.ts +90 -0
- package/dist/core/browser-fetch.d.ts.map +1 -0
- package/dist/core/browser-fetch.js +599 -0
- package/dist/core/browser-fetch.js.map +1 -0
- package/dist/core/browser-pool.d.ts +70 -0
- package/dist/core/browser-pool.d.ts.map +1 -0
- package/dist/core/browser-pool.js +378 -0
- package/dist/core/browser-pool.js.map +1 -0
- package/dist/core/change-tracking.js +3 -2
- package/dist/core/change-tracking.js.map +1 -1
- package/dist/core/diff.js +3 -2
- package/dist/core/diff.js.map +1 -1
- package/dist/core/domain-extractors.js +3 -2
- package/dist/core/domain-extractors.js.map +1 -1
- package/dist/core/extract-inline.js +6 -4
- package/dist/core/extract-inline.js.map +1 -1
- package/dist/core/fetcher.d.ts +9 -118
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +10 -1525
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/http-fetch.d.ts +37 -0
- package/dist/core/http-fetch.d.ts.map +1 -0
- package/dist/core/http-fetch.js +618 -0
- package/dist/core/http-fetch.js.map +1 -0
- package/dist/core/metadata.js +18 -12
- package/dist/core/metadata.js.map +1 -1
- package/dist/core/pipeline.d.ts +104 -0
- package/dist/core/pipeline.d.ts.map +1 -0
- package/dist/core/pipeline.js +623 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/profiles.js +15 -10
- package/dist/core/profiles.js.map +1 -1
- package/dist/core/quick-answer.d.ts.map +1 -1
- package/dist/core/quick-answer.js +120 -9
- package/dist/core/quick-answer.js.map +1 -1
- package/dist/core/rate-governor.js +3 -2
- package/dist/core/rate-governor.js.map +1 -1
- package/dist/core/research.js +9 -6
- package/dist/core/research.js.map +1 -1
- package/dist/core/search-provider.js +12 -8
- package/dist/core/search-provider.js.map +1 -1
- package/dist/core/youtube.js +3 -2
- package/dist/core/youtube.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -487
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/server/middleware/auth.js +3 -2
- package/dist/server/middleware/auth.js.map +1 -1
- package/dist/server/routes/compat.js +3 -2
- package/dist/server/routes/compat.js.map +1 -1
- package/dist/server/routes/fetch.d.ts.map +1 -1
- package/dist/server/routes/fetch.js +44 -4
- package/dist/server/routes/fetch.js.map +1 -1
- package/dist/server/routes/health.js +3 -2
- package/dist/server/routes/health.js.map +1 -1
- package/dist/server/routes/mcp.js +1 -1
- package/dist/server/routes/mcp.js.map +1 -1
- package/dist/server/routes/search.js +6 -4
- package/dist/server/routes/search.js.map +1 -1
- package/dist/server/routes/users.js +3 -2
- package/dist/server/routes/users.js.map +1 -1
- package/package.json +1 -1
package/dist/core/youtube.js
CHANGED
|
@@ -199,8 +199,9 @@ export function extractPlayerResponse(html) {
|
|
|
199
199
|
try {
|
|
200
200
|
return JSON.parse(jsonStr);
|
|
201
201
|
}
|
|
202
|
-
catch {
|
|
203
|
-
|
|
202
|
+
catch (e) {
|
|
203
|
+
if (process.env.DEBUG)
|
|
204
|
+
console.debug('[webpeel]', 'player response parse failed:', e instanceof Error ? e.message : e);
|
|
204
205
|
}
|
|
205
206
|
}
|
|
206
207
|
}
|
package/dist/core/youtube.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../src/core/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA4CzD,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAErF,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,4BAA4B;QAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC;IAED,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,oBAAoB;QACpB,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,kBAAkB;QAClB,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,mBAAmB;QACnB,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,iCAAiC;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAEnD,MAAM,YAAY,GAAG,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC;IACxD,MAAM,WAAW,GAAG,cAAc,EAAE,WAAW,EAAE,yBAAyB,IAAI,EAAE,CAAC;IAEjF,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,IAAI,EAAE,CAAC;IAC3C,MAAM,KAAK,GACT,YAAY,CAAC,KAAK;QAClB,WAAW,CAAC,KAAK,EAAE,UAAU;QAC7B,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC;QAChC,EAAE,CAAC;IACL,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,WAAW,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAC1E,MAAM,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,IAAI,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC;IACxE,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,IAAI,WAAW,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5E,MAAM,WAAW,GACf,YAAY,CAAC,gBAAgB;QAC7B,WAAW,CAAC,WAAW,EAAE,UAAU;QACnC,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC;QACtC,EAAE,CAAC;IACL,MAAM,SAAS,GACb,YAAY,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG;QACrD,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG;QACpD,8BAA8B,OAAO,oBAAoB,CAAC;IAE5D,gDAAgD;IAChD,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,IAAI,EAAE,CAAC;IAE/C,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,WAAW;QACX,QAAQ,EAAE,cAAc,CAAC,aAAa,CAAC;QACvC,WAAW;QACX,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAAW,EACX,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAE/C,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,mCAAmC,OAAO,EAAE,CAAC;IAC9D,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAClE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;QACxB,wEAAwE;QACxE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;QAChE,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,0BAA0B;IAC1B,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,qBAAqB;IACrB,MAAM,YAAY,GAAG,cAAc,CAAC,YAAY,IAAI,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC;IAC1C,MAAM,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IAEtE,yBAAyB;IACzB,MAAM,aAAa,GAAmB,oBAAoB,CAAC,cAAc,CAAC,CAAC;IAC3E,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,kBAAkB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAElE,kFAAkF;IAClF,MAAM,aAAa,GAAG,eAAe,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IAEpE,wBAAwB;IACxB,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAEhE,iBAAiB;IACjB,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAEjF,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,QAAQ,EAAE,cAAc,CAAC,aAAa,CAAC;QACvC,QAAQ,EAAE,aAAa,CAAC,YAAY;QACpC,QAAQ;QACR,QAAQ;QACR,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,qDAAqD;IACrD,MAAM,QAAQ,GAAG;QACf,+CAA+C;QAC/C,oEAAoE;QACpE,6DAA6D;QAC7D,6DAA6D;KAC9D,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;gBAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;gBACtD,IAAI,KAAK,KAAK,CAAC,CAAC;oBAAE,SAAS;gBAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,UAAU,KAAK,CAAC,CAAC;oBAAE,SAAS;gBAChC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACpD,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;oBAAC,MAAM,CAAC;wBACP,kCAAkC;oBACpC,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,yCAAyC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACpD,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7B,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,GAAW,EAAE,KAAa;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAElB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,KAAK,CAAC;YACf,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,QAAQ,GAAG,CAAC,QAAQ,CAAC;YACrB,SAAS;QACX,CAAC;QAED,IAAI,QAAQ;YAAE,SAAS;QAEvB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,EAAE,CAAC;aACjC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAClC,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,cAAmC;IAC/D,IAAI,CAAC;QACH,MAAM,MAAM,GACV,cAAc,EAAE,QAAQ,EAAE,+BAA+B,EAAE,aAAa,CAAC;QAC3E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,YAAY,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE;YACzD,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,EAAE;YAC3E,eAAe,EACb,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC;gBACpC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC/D,KAAK;SACR,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAsB,EAAE,aAAqB;IACpE,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB;IAEzE,kCAAkC;IAClC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,sBAAsB;IACtB,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACvD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,kCAAkC;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAAC,OAAe;IAC5C,gDAAgD;IAChD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,6BAA6B;IACnE,iCAAiC;IACjC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,QAAQ,GAAwB,EAAE,CAAC;IAEzC,kDAAkD;IAClD,MAAM,SAAS,GAAG,qCAAqC,CAAC;IACxD,IAAI,KAA6B,CAAC;IAElC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEzB,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhD,IAAI,IAAI,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,YAAY,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI;QACT,gFAAgF;SAC/E,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QACxB,+BAA+B;SAC9B,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;SAC1E,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;SAClF,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,QAAgB;IACpD,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,mCAAmC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,mCAAmC,EACnG,GAAG,CACJ,CAAC;IACF,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CACvC,IAAI,MAAM,CAAC,gEAAgE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CACrH,CAAC;IACF,OAAO,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}
|
|
1
|
+
{"version":3,"file":"youtube.js","sourceRoot":"","sources":["../../src/core/youtube.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AA4CzD,8EAA8E;AAC9E,cAAc;AACd,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEjD,IAAI,MAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAErF,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,4BAA4B;QAC5B,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,OAAO,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACxC,CAAC;IAED,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;QAC3B,oBAAoB;QACpB,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClE,MAAM,EAAE,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,kBAAkB;QAClB,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,mBAAmB;QACnB,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;QAED,iCAAiC;QACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzC,OAAO,EAAE,IAAI,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,EAAU;IAChC,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAEnD,MAAM,YAAY,GAAG,cAAc,EAAE,YAAY,IAAI,EAAE,CAAC;IACxD,MAAM,WAAW,GAAG,cAAc,EAAE,WAAW,EAAE,yBAAyB,IAAI,EAAE,CAAC;IAEjF,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,IAAI,EAAE,CAAC;IAC3C,MAAM,KAAK,GACT,YAAY,CAAC,KAAK;QAClB,WAAW,CAAC,KAAK,EAAE,UAAU;QAC7B,cAAc,CAAC,IAAI,EAAE,UAAU,CAAC;QAChC,EAAE,CAAC;IACL,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,WAAW,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAC1E,MAAM,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,IAAI,WAAW,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IACnG,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,IAAI,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC;IACxE,MAAM,WAAW,GAAG,WAAW,CAAC,WAAW,IAAI,WAAW,CAAC,UAAU,IAAI,EAAE,CAAC;IAC5E,MAAM,WAAW,GACf,YAAY,CAAC,gBAAgB;QAC7B,WAAW,CAAC,WAAW,EAAE,UAAU;QACnC,cAAc,CAAC,IAAI,EAAE,gBAAgB,CAAC;QACtC,EAAE,CAAC;IACL,MAAM,SAAS,GACb,YAAY,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG;QACrD,WAAW,CAAC,SAAS,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG;QACpD,8BAA8B,OAAO,oBAAoB,CAAC;IAE5D,gDAAgD;IAChD,MAAM,SAAS,GAAG,YAAY,CAAC,SAAS,IAAI,EAAE,CAAC;IAE/C,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,WAAW;QACX,QAAQ,EAAE,cAAc,CAAC,aAAa,CAAC;QACvC,WAAW;QACX,SAAS;QACT,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAAW,EACX,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,aAAa,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAE/C,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,mCAAmC,OAAO,EAAE,CAAC;IAC9D,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAClE,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;QACxB,wEAAwE;QACxE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;QAChE,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,0BAA0B;IAC1B,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,+CAA+C,OAAO,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,qBAAqB;IACrB,MAAM,YAAY,GAAG,cAAc,CAAC,YAAY,IAAI,EAAE,CAAC;IACvD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;IACvC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,EAAE,CAAC;IAC1C,MAAM,aAAa,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;IAEtE,yBAAyB;IACzB,MAAM,aAAa,GAAmB,oBAAoB,CAAC,cAAc,CAAC,CAAC;IAC3E,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,kBAAkB,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAElE,kFAAkF;IAClF,MAAM,aAAa,GAAG,eAAe,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IAEpE,wBAAwB;IACxB,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAEhE,iBAAiB;IACjB,MAAM,QAAQ,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IAE7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAEjF,OAAO;QACL,OAAO;QACP,KAAK;QACL,OAAO;QACP,QAAQ,EAAE,cAAc,CAAC,aAAa,CAAC;QACvC,QAAQ,EAAE,aAAa,CAAC,YAAY;QACpC,QAAQ;QACR,QAAQ;QACR,kBAAkB;KACnB,CAAC;AACJ,CAAC;AAaD;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,qDAAqD;IACrD,MAAM,QAAQ,GAAG;QACf,+CAA+C;QAC/C,oEAAoE;QACpE,6DAA6D;QAC7D,6DAA6D;KAC9D,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBACP,0DAA0D;gBAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;gBACtD,IAAI,KAAK,KAAK,CAAC,CAAC;oBAAE,SAAS;gBAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,UAAU,KAAK,CAAC,CAAC;oBAAE,SAAS;gBAChC,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;gBACpD,IAAI,OAAO,EAAE,CAAC;oBACZ,IAAI,CAAC;wBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC7B,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;4BAAE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,+BAA+B,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACzH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACnD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;QACtB,yCAAyC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACpD,IAAI,OAAO,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC7B,CAAC;gBAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,GAAW,EAAE,KAAa;IACnD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAElB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,GAAG,KAAK,CAAC;YACf,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,GAAG,IAAI,CAAC;YACd,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YACf,QAAQ,GAAG,CAAC,QAAQ,CAAC;YACrB,SAAS;QACX,CAAC;QAED,IAAI,QAAQ;YAAE,SAAS;QAEvB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG;YAAE,KAAK,EAAE,CAAC;aACjC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAClC,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;gBAChB,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,cAAmC;IAC/D,IAAI,CAAC;QACH,MAAM,MAAM,GACV,cAAc,EAAE,QAAQ,EAAE,+BAA+B,EAAE,aAAa,CAAC;QAC3E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,YAAY,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE;YACzD,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,YAAY,IAAI,EAAE;YAC3E,eAAe,EACb,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;gBAClB,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC;gBACpC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAC/D,KAAK;SACR,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAsB,EAAE,aAAqB;IACpE,MAAM,IAAI,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB;IAEzE,kCAAkC;IAClC,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3F,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACxF,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAE9B,sBAAsB;IACtB,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;IACvD,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAEhC,kCAAkC;IAClC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,eAAe,CAAC,OAAe;IAC5C,gDAAgD;IAChD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC7B,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAE,6BAA6B;IACnE,iCAAiC;IACjC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACnE,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,GAAW;IACzC,MAAM,QAAQ,GAAwB,EAAE,CAAC;IAEzC,kDAAkD;IAClD,MAAM,SAAS,GAAG,qCAAqC,CAAC;IACxD,IAAI,KAA6B,CAAC;IAElC,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEzB,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;QAC7D,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAEhD,IAAI,IAAI,EAAE,CAAC;YACT,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,KAAa,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,YAAY,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI;QACT,gFAAgF;SAC/E,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;QACxB,+BAA+B;SAC9B,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;SAC1E,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;SAClF,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IAEnC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5E,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,QAAgB;IACpD,MAAM,KAAK,GAAG,IAAI,MAAM,CACtB,mCAAmC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,mCAAmC,EACnG,GAAG,CACJ,CAAC;IACF,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CACvC,IAAI,MAAM,CAAC,gEAAgE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CACrH,CAAC;IACF,OAAO,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAa1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,KAAK,mBAAmB,EAAE,KAAK,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACpI,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAE,eAAe,EAAE,KAAK,UAAU,EAAE,KAAK,aAAa,EAAE,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,SAAS,EAAE,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,eAAe,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC3E,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,KAAK,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAClI,OAAO,EAAE,iBAAiB,EAAE,KAAK,oBAAoB,EAAE,KAAK,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAClH,OAAO,EAAE,QAAQ,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,aAAa,EAAE,KAAK,gBAAgB,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7J,OAAO,EAAE,gBAAgB,EAAE,KAAK,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,gBAAgB,GACtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,cAAc,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,UAAU,GAChB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,UAAU,EAAE,KAAK,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACvH,OAAO,EACL,YAAY,EACZ,cAAc,EACd,KAAK,UAAU,EACf,KAAK,SAAS,EACd,KAAK,cAAc,GACpB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,GACvB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,UAAU,EACV,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,uBAAuB,EACvB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,WAAW,EAChB,KAAK,iBAAiB,IAAI,sBAAsB,GACjD,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,qBAAqB,EACrB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,GACtB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC5F,OAAO,EACL,KAAK,EACL,aAAa,EACb,cAAc,EACd,KAAK,YAAY,EACjB,KAAK,SAAS,EACd,KAAK,gBAAgB,EACrB,KAAK,eAAe,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,OAAO,EACP,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,UAAU,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,sBAAsB,EAAE,KAAK,iBAAiB,EAAE,KAAK,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAChH,OAAO,EAAE,WAAW,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACtG,OAAO,EAAE,KAAK,EAAE,KAAK,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAE9D;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAmBtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,CAAC;AAC1E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,qBAAqB,EACrB,WAAW,EACX,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EACL,UAAU,EACV,cAAc,EACd,SAAS,EACT,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,oBAAoB,EACpB,aAAa,EACb,WAAW,EACX,eAAe,EACf,WAAW,EACX,KAAK,WAAW,GACjB,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -3,21 +3,8 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Main library export
|
|
5
5
|
*/
|
|
6
|
-
import { createHash } from 'crypto';
|
|
7
|
-
import { smartFetch } from './core/strategies.js';
|
|
8
|
-
import { htmlToMarkdown, htmlToText, estimateTokens, selectContent, detectMainContent, calculateQuality, truncateToTokenBudget, filterByTags } from './core/markdown.js';
|
|
9
|
-
import { pruneContent } from './core/content-pruner.js';
|
|
10
|
-
import { distillToBudget } from './core/budget.js';
|
|
11
|
-
import { extractMetadata, extractLinks, extractImages } from './core/metadata.js';
|
|
12
6
|
import { cleanup, warmup, closePool, scrollAndWait, closeProfileBrowser } from './core/fetcher.js';
|
|
13
|
-
import {
|
|
14
|
-
import { extractStructured } from './core/extract.js';
|
|
15
|
-
import { isPdfContentType, isDocxContentType, extractDocumentToFormat } from './core/documents.js';
|
|
16
|
-
import { parseYouTubeUrl, getYouTubeTranscript } from './core/youtube.js';
|
|
17
|
-
import { extractDomainData, getDomainExtractor } from './core/domain-extractors.js';
|
|
18
|
-
import { extractReadableContent } from './core/readability.js';
|
|
19
|
-
import { quickAnswer as runQuickAnswer } from './core/quick-answer.js';
|
|
20
|
-
import { Timer } from './core/timing.js';
|
|
7
|
+
import { createContext, normalizeOptions, handleYouTube, fetchContent, detectContentType, parseContent, postProcess, finalize, buildResult, } from './core/pipeline.js';
|
|
21
8
|
export * from './types.js';
|
|
22
9
|
export { getDomainExtractor, extractDomainData } from './core/domain-extractors.js';
|
|
23
10
|
export { crawl } from './core/crawler.js';
|
|
@@ -64,480 +51,18 @@ export { Timer } from './core/timing.js';
|
|
|
64
51
|
* ```
|
|
65
52
|
*/
|
|
66
53
|
export async function peel(url, options = {}) {
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (
|
|
71
|
-
|
|
72
|
-
options.budget = 4000;
|
|
73
|
-
if (options.format === undefined)
|
|
74
|
-
options.format = 'markdown';
|
|
75
|
-
}
|
|
76
|
-
let { render = false, stealth = false, wait = 0, format = 'markdown', timeout = 30000, userAgent, screenshot = false, screenshotFullPage = false, selector, exclude, includeTags, excludeTags, headers, cookies, raw = false, actions, extract, maxTokens, images: extractImagesFlag = false, location: _location, stream: _stream, profileDir, headed = false, storageState, proxy, fullPage = false, autoScroll: autoScrollOption, } = options;
|
|
77
|
-
void _stream;
|
|
78
|
-
// Normalize autoScroll option
|
|
79
|
-
const autoScrollOpts = autoScrollOption
|
|
80
|
-
? (typeof autoScrollOption === 'boolean' ? {} : autoScrollOption)
|
|
81
|
-
: undefined;
|
|
82
|
-
// NOTE: PDFs/DOCX are now handled via simpleFetch + document parser.
|
|
83
|
-
// No need to force browser rendering for them.
|
|
84
|
-
// If screenshot is requested, force render mode
|
|
85
|
-
if (screenshot) {
|
|
86
|
-
render = true;
|
|
87
|
-
}
|
|
88
|
-
// If stealth is requested, force render mode
|
|
89
|
-
if (stealth) {
|
|
90
|
-
render = true;
|
|
91
|
-
}
|
|
92
|
-
// If actions are provided, force render mode
|
|
93
|
-
if (actions && actions.length > 0) {
|
|
94
|
-
render = true;
|
|
95
|
-
}
|
|
96
|
-
// If branding is requested, force render mode
|
|
97
|
-
if (options.branding) {
|
|
98
|
-
render = true;
|
|
99
|
-
}
|
|
100
|
-
// If autoScroll is requested, force render mode
|
|
101
|
-
if (autoScrollOpts) {
|
|
102
|
-
render = true;
|
|
103
|
-
}
|
|
104
|
-
// ---------------------------------------------------------------------------
|
|
105
|
-
// YouTube special case — extract transcript instead of fetching the page HTML
|
|
106
|
-
// ---------------------------------------------------------------------------
|
|
107
|
-
const ytVideoId = parseYouTubeUrl(url);
|
|
108
|
-
if (ytVideoId) {
|
|
109
|
-
const ytStartTime = Date.now();
|
|
110
|
-
try {
|
|
111
|
-
const transcript = await getYouTubeTranscript(url, {
|
|
112
|
-
language: options.language ?? 'en',
|
|
113
|
-
});
|
|
114
|
-
// Build a clean markdown representation of the video + transcript
|
|
115
|
-
const videoInfoLines = [
|
|
116
|
-
`# ${transcript.title}`,
|
|
117
|
-
'',
|
|
118
|
-
`**Channel:** ${transcript.channel}`,
|
|
119
|
-
`**Duration:** ${transcript.duration}`,
|
|
120
|
-
`**Language:** ${transcript.language}`,
|
|
121
|
-
transcript.availableLanguages.length > 1
|
|
122
|
-
? `**Available Languages:** ${transcript.availableLanguages.join(', ')}`
|
|
123
|
-
: '',
|
|
124
|
-
'',
|
|
125
|
-
'## Transcript',
|
|
126
|
-
'',
|
|
127
|
-
transcript.fullText,
|
|
128
|
-
].filter(l => l !== undefined);
|
|
129
|
-
const videoInfoContent = videoInfoLines.join('\n');
|
|
130
|
-
const elapsed = Date.now() - ytStartTime;
|
|
131
|
-
const tokens = estimateTokens(videoInfoContent);
|
|
132
|
-
const fingerprint = createHash('sha256').update(videoInfoContent).digest('hex').slice(0, 16);
|
|
133
|
-
return {
|
|
134
|
-
url: `https://www.youtube.com/watch?v=${ytVideoId}`,
|
|
135
|
-
title: transcript.title,
|
|
136
|
-
content: videoInfoContent,
|
|
137
|
-
metadata: {
|
|
138
|
-
description: `YouTube video by ${transcript.channel}, duration ${transcript.duration}`,
|
|
139
|
-
author: transcript.channel,
|
|
140
|
-
},
|
|
141
|
-
links: [`https://www.youtube.com/watch?v=${ytVideoId}`],
|
|
142
|
-
tokens,
|
|
143
|
-
method: 'simple',
|
|
144
|
-
elapsed,
|
|
145
|
-
contentType: 'youtube',
|
|
146
|
-
quality: 1.0,
|
|
147
|
-
fingerprint,
|
|
148
|
-
extracted: undefined,
|
|
149
|
-
structured: transcript,
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
|
-
catch (_ytError) {
|
|
153
|
-
// If transcript extraction fails (no captions, page changed, etc.),
|
|
154
|
-
// fall through to the normal HTML fetch pipeline below.
|
|
155
|
-
}
|
|
156
|
-
}
|
|
54
|
+
const ctx = createContext(url, options);
|
|
55
|
+
normalizeOptions(ctx);
|
|
56
|
+
const ytResult = await handleYouTube(ctx);
|
|
57
|
+
if (ytResult)
|
|
58
|
+
return ytResult;
|
|
157
59
|
try {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
stealth,
|
|
165
|
-
waitMs: wait,
|
|
166
|
-
userAgent,
|
|
167
|
-
timeoutMs: timeout,
|
|
168
|
-
screenshot,
|
|
169
|
-
screenshotFullPage,
|
|
170
|
-
headers,
|
|
171
|
-
cookies,
|
|
172
|
-
actions,
|
|
173
|
-
keepPageOpen: needsBranding || needsAutoScroll,
|
|
174
|
-
profileDir,
|
|
175
|
-
headed,
|
|
176
|
-
storageState,
|
|
177
|
-
proxy,
|
|
178
|
-
});
|
|
179
|
-
timer.end('fetch');
|
|
180
|
-
// Auto-scroll to load lazy content, then grab fresh HTML
|
|
181
|
-
if (needsAutoScroll && fetchResult.page) {
|
|
182
|
-
try {
|
|
183
|
-
await runAutoScroll(fetchResult.page, autoScrollOpts);
|
|
184
|
-
// Capture refreshed HTML after scrolling
|
|
185
|
-
fetchResult.html = await fetchResult.page.content();
|
|
186
|
-
}
|
|
187
|
-
catch (e) {
|
|
188
|
-
// Non-fatal: auto-scroll failed, continuing with whatever HTML we have
|
|
189
|
-
if (process.env.DEBUG)
|
|
190
|
-
console.debug('[webpeel]', 'auto-scroll failed:', e instanceof Error ? e.message : e);
|
|
191
|
-
}
|
|
192
|
-
finally {
|
|
193
|
-
// Close page unless branding also needs it
|
|
194
|
-
if (!needsBranding) {
|
|
195
|
-
try {
|
|
196
|
-
await fetchResult.page.close().catch(() => { });
|
|
197
|
-
if (fetchResult.browser && !needsBranding) {
|
|
198
|
-
await fetchResult.browser.close().catch(() => { });
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
catch (e) {
|
|
202
|
-
// Non-fatal: page/browser cleanup after auto-scroll
|
|
203
|
-
if (process.env.DEBUG)
|
|
204
|
-
console.debug('[webpeel]', 'page/browser cleanup after auto-scroll:', e instanceof Error ? e.message : e);
|
|
205
|
-
}
|
|
206
|
-
fetchResult.page = undefined;
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
// Detect content type from the response
|
|
211
|
-
const ct = (fetchResult.contentType || '').toLowerCase();
|
|
212
|
-
const urlLower = fetchResult.url.toLowerCase();
|
|
213
|
-
// Check for binary document types (PDF/DOCX)
|
|
214
|
-
const isDocument = isPdfContentType(ct) || isDocxContentType(ct) ||
|
|
215
|
-
urlLower.endsWith('.pdf') || urlLower.endsWith('.docx');
|
|
216
|
-
const hasBuffer = !!fetchResult.buffer;
|
|
217
|
-
const isHTML = !isDocument && (ct.includes('html') || ct.includes('xhtml') || (!ct && fetchResult.html.trimStart().startsWith('<')));
|
|
218
|
-
const isJSON = !isDocument && ct.includes('json');
|
|
219
|
-
const isXML = !isDocument && (ct.includes('xml') || ct.includes('rss') || ct.includes('atom'));
|
|
220
|
-
const isPlainText = !isDocument && (ct.includes('text/plain') || ct.includes('text/markdown') || ct.includes('text/csv') || ct.includes('text/css') || ct.includes('javascript'));
|
|
221
|
-
const detectedType = isDocument ? 'document' : isHTML ? 'html' : isJSON ? 'json' : isXML ? 'xml' : isPlainText ? 'text' : 'html';
|
|
222
|
-
let content;
|
|
223
|
-
let title = '';
|
|
224
|
-
let metadata = {};
|
|
225
|
-
let links = [];
|
|
226
|
-
let quality = 0;
|
|
227
|
-
let prunedPercent;
|
|
228
|
-
if (isDocument && hasBuffer) {
|
|
229
|
-
// Document parsing pipeline (PDF/DOCX)
|
|
230
|
-
const docResult = await extractDocumentToFormat(fetchResult.buffer, {
|
|
231
|
-
url: fetchResult.url,
|
|
232
|
-
contentType: fetchResult.contentType,
|
|
233
|
-
format,
|
|
234
|
-
});
|
|
235
|
-
content = docResult.content;
|
|
236
|
-
title = docResult.metadata.title;
|
|
237
|
-
metadata = docResult.metadata;
|
|
238
|
-
quality = 1.0; // Documents are inherently structured content
|
|
239
|
-
}
|
|
240
|
-
else if (isHTML) {
|
|
241
|
-
// Standard HTML pipeline
|
|
242
|
-
let html = fetchResult.html;
|
|
243
|
-
// Apply include/exclude tags filtering first (before selector)
|
|
244
|
-
if (includeTags || excludeTags) {
|
|
245
|
-
html = filterByTags(html, includeTags, excludeTags);
|
|
246
|
-
}
|
|
247
|
-
if (selector) {
|
|
248
|
-
html = selectContent(html, selector, exclude);
|
|
249
|
-
}
|
|
250
|
-
else if (exclude?.length) {
|
|
251
|
-
// Apply exclude selectors even without a specific selector
|
|
252
|
-
const cheerio = await import('cheerio');
|
|
253
|
-
const $doc = cheerio.load(html);
|
|
254
|
-
exclude.forEach(sel => $doc(sel).remove());
|
|
255
|
-
html = $doc.html() || html;
|
|
256
|
-
}
|
|
257
|
-
// Smart main content detection (unless raw or selector specified)
|
|
258
|
-
let contentHtml = html;
|
|
259
|
-
if (!raw && !selector) {
|
|
260
|
-
const detected = detectMainContent(html);
|
|
261
|
-
if (detected.detected) {
|
|
262
|
-
contentHtml = detected.html;
|
|
263
|
-
}
|
|
264
|
-
}
|
|
265
|
-
const metadataTask = Promise.resolve().then(() => {
|
|
266
|
-
timer.mark('metadata');
|
|
267
|
-
const meta = extractMetadata(html, fetchResult.url);
|
|
268
|
-
const result = {
|
|
269
|
-
title: meta.title,
|
|
270
|
-
metadata: meta.metadata,
|
|
271
|
-
links: extractLinks(html, fetchResult.url),
|
|
272
|
-
};
|
|
273
|
-
timer.end('metadata');
|
|
274
|
-
return result;
|
|
275
|
-
});
|
|
276
|
-
// Content density pruning — runs on HTML before markdown conversion.
|
|
277
|
-
// Removes low-value blocks (sidebars, footers, ads) CSS selectors miss.
|
|
278
|
-
// OFF when fullPage=true or format !== markdown.
|
|
279
|
-
if (format === 'markdown' && !fullPage) {
|
|
280
|
-
timer.mark('prune');
|
|
281
|
-
const pruned = pruneContent(contentHtml, { dynamic: true });
|
|
282
|
-
timer.end('prune');
|
|
283
|
-
contentHtml = pruned.html;
|
|
284
|
-
if (pruned.nodesRemoved > 0) {
|
|
285
|
-
prunedPercent = pruned.reductionPercent;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
const contentTask = Promise.resolve().then(() => {
|
|
289
|
-
timer.mark('convert');
|
|
290
|
-
let converted;
|
|
291
|
-
switch (format) {
|
|
292
|
-
case 'html':
|
|
293
|
-
converted = contentHtml;
|
|
294
|
-
break;
|
|
295
|
-
case 'text':
|
|
296
|
-
converted = htmlToText(contentHtml);
|
|
297
|
-
break;
|
|
298
|
-
case 'markdown':
|
|
299
|
-
default:
|
|
300
|
-
// prune:false — already pruned above; avoid double-pruning in htmlToMarkdown
|
|
301
|
-
converted = htmlToMarkdown(contentHtml, { raw, prune: false });
|
|
302
|
-
break;
|
|
303
|
-
}
|
|
304
|
-
timer.end('convert');
|
|
305
|
-
return converted;
|
|
306
|
-
});
|
|
307
|
-
const [metaResult, convertedContent] = await Promise.all([metadataTask, contentTask]);
|
|
308
|
-
title = metaResult.title;
|
|
309
|
-
metadata = metaResult.metadata;
|
|
310
|
-
links = metaResult.links;
|
|
311
|
-
content = convertedContent;
|
|
312
|
-
quality = calculateQuality(content, fetchResult.html);
|
|
313
|
-
}
|
|
314
|
-
else if (isJSON) {
|
|
315
|
-
// JSON content — format nicely
|
|
316
|
-
try {
|
|
317
|
-
const parsed = JSON.parse(fetchResult.html);
|
|
318
|
-
content = JSON.stringify(parsed, null, 2);
|
|
319
|
-
title = 'JSON Response';
|
|
320
|
-
// Extract any URLs from JSON for links
|
|
321
|
-
const urlRegex = /https?:\/\/[^\s"'`,\]})]+/g;
|
|
322
|
-
const found = content.match(urlRegex) || [];
|
|
323
|
-
links = [...new Set(found)];
|
|
324
|
-
}
|
|
325
|
-
catch (e) {
|
|
326
|
-
// Non-fatal: JSON parse failed, treating as malformed
|
|
327
|
-
if (process.env.DEBUG)
|
|
328
|
-
console.debug('[webpeel]', 'JSON parse failed:', e instanceof Error ? e.message : e);
|
|
329
|
-
content = fetchResult.html;
|
|
330
|
-
title = 'JSON Response (malformed)';
|
|
331
|
-
}
|
|
332
|
-
quality = 1.0; // JSON is structured, always "clean"
|
|
333
|
-
}
|
|
334
|
-
else if (isXML) {
|
|
335
|
-
// XML/RSS/Atom — convert to readable format
|
|
336
|
-
try {
|
|
337
|
-
const $ = (await import('cheerio')).load(fetchResult.html, { xml: true });
|
|
338
|
-
// Check if RSS/Atom feed
|
|
339
|
-
const items = $('item, entry');
|
|
340
|
-
if (items.length > 0) {
|
|
341
|
-
title = $('channel > title, feed > title').first().text() || 'RSS/Atom Feed';
|
|
342
|
-
const feedItems = [];
|
|
343
|
-
items.each((_, el) => {
|
|
344
|
-
const itemTitle = $(el).find('title').first().text();
|
|
345
|
-
const itemLink = $(el).find('link').first().text() || $(el).find('link').first().attr('href') || '';
|
|
346
|
-
const itemDesc = $(el).find('description, summary, content').first().text().slice(0, 200);
|
|
347
|
-
feedItems.push(`## ${itemTitle}\n${itemLink}\n${itemDesc}`);
|
|
348
|
-
if (itemLink)
|
|
349
|
-
links.push(itemLink);
|
|
350
|
-
});
|
|
351
|
-
content = `# ${title}\n\n${feedItems.join('\n\n---\n\n')}`;
|
|
352
|
-
}
|
|
353
|
-
else {
|
|
354
|
-
content = fetchResult.html;
|
|
355
|
-
title = $('title').first().text() || 'XML Document';
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
catch (e) {
|
|
359
|
-
// Non-fatal: XML/RSS parse failed, using raw content
|
|
360
|
-
if (process.env.DEBUG)
|
|
361
|
-
console.debug('[webpeel]', 'XML/RSS parse failed:', e instanceof Error ? e.message : e);
|
|
362
|
-
content = fetchResult.html;
|
|
363
|
-
title = 'XML Document';
|
|
364
|
-
}
|
|
365
|
-
quality = 0.9;
|
|
366
|
-
}
|
|
367
|
-
else {
|
|
368
|
-
// Plain text, CSS, JS, etc — return as-is
|
|
369
|
-
content = fetchResult.html;
|
|
370
|
-
title = fetchResult.url.split('/').pop() || 'Text Document';
|
|
371
|
-
// Extract URLs from plain text
|
|
372
|
-
const urlRegex = /https?:\/\/[^\s"'`,\]})]+/g;
|
|
373
|
-
const found = content.match(urlRegex) || [];
|
|
374
|
-
links = [...new Set(found)];
|
|
375
|
-
quality = 1.0;
|
|
376
|
-
}
|
|
377
|
-
// Readability mode
|
|
378
|
-
let readabilityResult;
|
|
379
|
-
if (options.readable && isHTML && fetchResult.html) {
|
|
380
|
-
timer.mark('readability');
|
|
381
|
-
const readResult = extractReadableContent(fetchResult.html, fetchResult.url);
|
|
382
|
-
timer.end('readability');
|
|
383
|
-
readabilityResult = readResult;
|
|
384
|
-
content = readResult.content;
|
|
385
|
-
metadata = {
|
|
386
|
-
...metadata,
|
|
387
|
-
title: readResult.title || metadata?.title,
|
|
388
|
-
author: readResult.author || undefined,
|
|
389
|
-
publishedDate: readResult.date || undefined,
|
|
390
|
-
};
|
|
391
|
-
title = readResult.title || title;
|
|
392
|
-
}
|
|
393
|
-
// Extract images if requested
|
|
394
|
-
let imagesList;
|
|
395
|
-
if (extractImagesFlag && isHTML) {
|
|
396
|
-
imagesList = extractImages(fetchResult.html, fetchResult.url);
|
|
397
|
-
}
|
|
398
|
-
// Extract structured data if requested
|
|
399
|
-
let extracted;
|
|
400
|
-
if (extract && isHTML) {
|
|
401
|
-
if (extract.llmApiKey && (extract.prompt || extract.schema)) {
|
|
402
|
-
// LLM-powered extraction
|
|
403
|
-
const { extractWithLLM } = await import('./core/extract.js');
|
|
404
|
-
extracted = await extractWithLLM(content, extract);
|
|
405
|
-
}
|
|
406
|
-
else if (extract.selectors || extract.schema) {
|
|
407
|
-
// CSS-based extraction (existing)
|
|
408
|
-
extracted = extractStructured(fetchResult.html, extract);
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
// Truncate to token budget if requested (simple truncation)
|
|
412
|
-
if (maxTokens && maxTokens > 0) {
|
|
413
|
-
content = truncateToTokenBudget(content, maxTokens);
|
|
414
|
-
}
|
|
415
|
-
// Smart budget distillation — applied AFTER maxTokens truncation
|
|
416
|
-
// This intelligently compresses content (strips boilerplate, compresses
|
|
417
|
-
// tables, removes weak paragraphs) rather than blindly cutting.
|
|
418
|
-
if (options.budget && options.budget > 0) {
|
|
419
|
-
const budgetFormat = detectedType === 'json' ? 'json' :
|
|
420
|
-
format === 'text' ? 'text' : 'markdown';
|
|
421
|
-
timer.mark('budget');
|
|
422
|
-
content = distillToBudget(content, options.budget, budgetFormat);
|
|
423
|
-
timer.end('budget');
|
|
424
|
-
}
|
|
425
|
-
// Domain-aware structured extraction (Twitter, Reddit, GitHub, HN)
|
|
426
|
-
// Fires when URL matches a known domain. Replaces content with clean markdown.
|
|
427
|
-
let domainData;
|
|
428
|
-
if (getDomainExtractor(fetchResult.url)) {
|
|
429
|
-
try {
|
|
430
|
-
timer.mark('domainExtract');
|
|
431
|
-
const ddResult = await extractDomainData(fetchResult.html, fetchResult.url);
|
|
432
|
-
timer.end('domainExtract');
|
|
433
|
-
if (ddResult) {
|
|
434
|
-
domainData = ddResult;
|
|
435
|
-
content = ddResult.cleanContent;
|
|
436
|
-
}
|
|
437
|
-
}
|
|
438
|
-
catch (e) {
|
|
439
|
-
// Domain extraction failure is non-fatal; continue with normal content
|
|
440
|
-
if (process.env.DEBUG)
|
|
441
|
-
console.debug('[webpeel]', 'domain extraction failed:', e instanceof Error ? e.message : e);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
// Quick answer (LLM-free)
|
|
445
|
-
let quickAnswerResult;
|
|
446
|
-
if (options.question && content) {
|
|
447
|
-
timer.mark('quickAnswer');
|
|
448
|
-
const qa = runQuickAnswer({
|
|
449
|
-
question: options.question,
|
|
450
|
-
content,
|
|
451
|
-
url: fetchResult.url,
|
|
452
|
-
});
|
|
453
|
-
timer.end('quickAnswer');
|
|
454
|
-
quickAnswerResult = qa;
|
|
455
|
-
}
|
|
456
|
-
// Calculate elapsed time, tokens, and fingerprint
|
|
457
|
-
const elapsed = Date.now() - startTime;
|
|
458
|
-
const tokens = estimateTokens(content);
|
|
459
|
-
const fingerprint = createHash('sha256').update(content).digest('hex').slice(0, 16);
|
|
460
|
-
// Convert screenshot buffer to base64 if present
|
|
461
|
-
const screenshotBase64 = fetchResult.screenshot?.toString('base64');
|
|
462
|
-
// Extract branding if requested (reuses existing browser page when available)
|
|
463
|
-
let brandingProfile;
|
|
464
|
-
if (options.branding && render && fetchResult.page) {
|
|
465
|
-
try {
|
|
466
|
-
const { extractBranding } = await import('./core/branding.js');
|
|
467
|
-
brandingProfile = await extractBranding(fetchResult.page);
|
|
468
|
-
}
|
|
469
|
-
catch (error) {
|
|
470
|
-
console.error('Branding extraction failed:', error);
|
|
471
|
-
}
|
|
472
|
-
finally {
|
|
473
|
-
// Clean up the kept-open page and browser
|
|
474
|
-
try {
|
|
475
|
-
await fetchResult.page.close().catch(() => { });
|
|
476
|
-
if (fetchResult.browser) {
|
|
477
|
-
await fetchResult.browser.close().catch(() => { });
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
catch (e) {
|
|
481
|
-
// Non-fatal: page/browser cleanup after branding extraction
|
|
482
|
-
if (process.env.DEBUG)
|
|
483
|
-
console.debug('[webpeel]', 'page/browser cleanup after branding:', e instanceof Error ? e.message : e);
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
}
|
|
487
|
-
// Track content changes if requested
|
|
488
|
-
let changeResult;
|
|
489
|
-
if (options.changeTracking) {
|
|
490
|
-
try {
|
|
491
|
-
const { trackChange } = await import('./core/change-tracking.js');
|
|
492
|
-
changeResult = await trackChange(fetchResult.url, content, fingerprint);
|
|
493
|
-
}
|
|
494
|
-
catch (error) {
|
|
495
|
-
console.error('Change tracking failed:', error);
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
// Generate AI summary if requested
|
|
499
|
-
let summaryText;
|
|
500
|
-
if (options.summary && options.llm) {
|
|
501
|
-
try {
|
|
502
|
-
const { summarizeContent } = await import('./core/summarize.js');
|
|
503
|
-
const maxLength = typeof options.summary === 'object' && options.summary.maxLength
|
|
504
|
-
? options.summary.maxLength
|
|
505
|
-
: 150;
|
|
506
|
-
summaryText = await summarizeContent(content, {
|
|
507
|
-
apiKey: options.llm.apiKey,
|
|
508
|
-
model: options.llm.model,
|
|
509
|
-
apiBase: options.llm.baseUrl,
|
|
510
|
-
maxWords: maxLength,
|
|
511
|
-
});
|
|
512
|
-
}
|
|
513
|
-
catch (error) {
|
|
514
|
-
console.error('Summary generation failed:', error);
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
|
-
return {
|
|
518
|
-
url: fetchResult.url,
|
|
519
|
-
title,
|
|
520
|
-
content,
|
|
521
|
-
metadata,
|
|
522
|
-
links,
|
|
523
|
-
tokens,
|
|
524
|
-
method: fetchResult.method === 'cached' ? 'simple' : fetchResult.method,
|
|
525
|
-
elapsed,
|
|
526
|
-
screenshot: screenshotBase64,
|
|
527
|
-
contentType: detectedType,
|
|
528
|
-
quality,
|
|
529
|
-
fingerprint,
|
|
530
|
-
extracted,
|
|
531
|
-
branding: brandingProfile,
|
|
532
|
-
changeTracking: changeResult,
|
|
533
|
-
summary: summaryText,
|
|
534
|
-
images: imagesList,
|
|
535
|
-
...(prunedPercent !== undefined ? { prunedPercent } : {}),
|
|
536
|
-
...(domainData !== undefined ? { domainData } : {}),
|
|
537
|
-
...(readabilityResult !== undefined ? { readability: readabilityResult } : {}),
|
|
538
|
-
...(quickAnswerResult !== undefined ? { quickAnswer: quickAnswerResult } : {}),
|
|
539
|
-
timing: timer.toTiming(),
|
|
540
|
-
};
|
|
60
|
+
await fetchContent(ctx);
|
|
61
|
+
detectContentType(ctx);
|
|
62
|
+
await parseContent(ctx);
|
|
63
|
+
await postProcess(ctx);
|
|
64
|
+
await finalize(ctx);
|
|
65
|
+
return buildResult(ctx);
|
|
541
66
|
}
|
|
542
67
|
catch (error) {
|
|
543
68
|
// Clean up browser resources on error
|