webpeel 0.14.3 → 0.14.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +11 -4
- package/dist/cache.js.map +1 -1
- package/dist/cli.bundle.cjs +159248 -0
- package/dist/core/agent.js +12 -8
- package/dist/core/agent.js.map +1 -1
- package/dist/core/application-tracker.js +3 -2
- package/dist/core/application-tracker.js.map +1 -1
- package/dist/core/auto-extract.js +6 -4
- package/dist/core/auto-extract.js.map +1 -1
- package/dist/core/browser-fetch.d.ts +90 -0
- package/dist/core/browser-fetch.d.ts.map +1 -0
- package/dist/core/browser-fetch.js +599 -0
- package/dist/core/browser-fetch.js.map +1 -0
- package/dist/core/browser-pool.d.ts +70 -0
- package/dist/core/browser-pool.d.ts.map +1 -0
- package/dist/core/browser-pool.js +378 -0
- package/dist/core/browser-pool.js.map +1 -0
- package/dist/core/change-tracking.js +3 -2
- package/dist/core/change-tracking.js.map +1 -1
- package/dist/core/diff.js +3 -2
- package/dist/core/diff.js.map +1 -1
- package/dist/core/domain-extractors.js +3 -2
- package/dist/core/domain-extractors.js.map +1 -1
- package/dist/core/extract-inline.js +6 -4
- package/dist/core/extract-inline.js.map +1 -1
- package/dist/core/fetcher.d.ts +9 -118
- package/dist/core/fetcher.d.ts.map +1 -1
- package/dist/core/fetcher.js +10 -1525
- package/dist/core/fetcher.js.map +1 -1
- package/dist/core/http-fetch.d.ts +37 -0
- package/dist/core/http-fetch.d.ts.map +1 -0
- package/dist/core/http-fetch.js +618 -0
- package/dist/core/http-fetch.js.map +1 -0
- package/dist/core/metadata.js +18 -12
- package/dist/core/metadata.js.map +1 -1
- package/dist/core/pipeline.d.ts +104 -0
- package/dist/core/pipeline.d.ts.map +1 -0
- package/dist/core/pipeline.js +623 -0
- package/dist/core/pipeline.js.map +1 -0
- package/dist/core/profiles.js +15 -10
- package/dist/core/profiles.js.map +1 -1
- package/dist/core/quick-answer.d.ts.map +1 -1
- package/dist/core/quick-answer.js +120 -9
- package/dist/core/quick-answer.js.map +1 -1
- package/dist/core/rate-governor.js +3 -2
- package/dist/core/rate-governor.js.map +1 -1
- package/dist/core/research.js +9 -6
- package/dist/core/research.js.map +1 -1
- package/dist/core/search-provider.js +12 -8
- package/dist/core/search-provider.js.map +1 -1
- package/dist/core/youtube.js +3 -2
- package/dist/core/youtube.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -487
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +1 -1
- package/dist/mcp/server.js.map +1 -1
- package/dist/server/middleware/auth.js +3 -2
- package/dist/server/middleware/auth.js.map +1 -1
- package/dist/server/routes/compat.js +3 -2
- package/dist/server/routes/compat.js.map +1 -1
- package/dist/server/routes/fetch.d.ts.map +1 -1
- package/dist/server/routes/fetch.js +44 -4
- package/dist/server/routes/fetch.js.map +1 -1
- package/dist/server/routes/health.js +3 -2
- package/dist/server/routes/health.js.map +1 -1
- package/dist/server/routes/mcp.js +1 -1
- package/dist/server/routes/mcp.js.map +1 -1
- package/dist/server/routes/search.js +6 -4
- package/dist/server/routes/search.js.map +1 -1
- package/dist/server/routes/users.js +3 -2
- package/dist/server/routes/users.js.map +1 -1
- package/package.json +1 -1
package/dist/cache.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAc5C;AAiBD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAc5C;AAiBD;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,GAAG,GAAG,IAAI,CAsB/E;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAerG;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,GAAG,UAAQ,GAAG,MAAM,CAyB9C;AAED;;GAEG;AACH,wBAAgB,UAAU,IAAI;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAgBhF"}
|
package/dist/cache.js
CHANGED
|
@@ -57,7 +57,10 @@ export function getCache(url, options) {
|
|
|
57
57
|
try {
|
|
58
58
|
unlinkSync(filePath);
|
|
59
59
|
}
|
|
60
|
-
catch {
|
|
60
|
+
catch (e) {
|
|
61
|
+
if (process.env.DEBUG)
|
|
62
|
+
console.debug('[webpeel]', 'file unlink failed:', e instanceof Error ? e.message : e);
|
|
63
|
+
}
|
|
61
64
|
return null;
|
|
62
65
|
}
|
|
63
66
|
return entry.result;
|
|
@@ -106,8 +109,9 @@ export function clearCache(all = false) {
|
|
|
106
109
|
}
|
|
107
110
|
}
|
|
108
111
|
}
|
|
109
|
-
catch {
|
|
110
|
-
|
|
112
|
+
catch (e) {
|
|
113
|
+
if (process.env.DEBUG)
|
|
114
|
+
console.debug('[webpeel]', 'cache file parse failed:', e instanceof Error ? e.message : e);
|
|
111
115
|
}
|
|
112
116
|
}
|
|
113
117
|
return cleared;
|
|
@@ -125,7 +129,10 @@ export function cacheStats() {
|
|
|
125
129
|
const stat = statSync(join(CACHE_DIR, file));
|
|
126
130
|
sizeBytes += stat.size;
|
|
127
131
|
}
|
|
128
|
-
catch {
|
|
132
|
+
catch (e) {
|
|
133
|
+
if (process.env.DEBUG)
|
|
134
|
+
console.debug('[webpeel]', 'stat file failed:', e instanceof Error ? e.message : e);
|
|
135
|
+
}
|
|
129
136
|
}
|
|
130
137
|
return { entries: files.length, sizeBytes, dir: CACHE_DIR };
|
|
131
138
|
}
|
package/dist/cache.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC3G,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AAUvD;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,yBAAyB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,IAAI,CAAC;QAC9B,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC;QACnC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACxC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC7C,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAW,EAAE,OAA6B;IAC1D,MAAM,QAAQ,GAAG;QACf,GAAG;QACH,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;QAChC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK;QAClC,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI;QACnC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,UAAU;KACtC,CAAC;IACF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9F,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,OAA6B;IACjE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAExC,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,mCAAmC;YACnC,IAAI,CAAC;gBAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;YAAC,
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../src/cache.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC3G,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AAUvD;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,yBAAyB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,IAAI,CAAC;QAC9B,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC;QACnC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QACxC,KAAK,GAAG,CAAC,CAAC,OAAO,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;QAC7C,OAAO,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,QAAQ,CAAC,GAAW,EAAE,OAA6B;IAC1D,MAAM,QAAQ,GAAG;QACf,GAAG;QACH,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK;QAChC,OAAO,EAAE,OAAO,EAAE,OAAO,IAAI,KAAK;QAClC,QAAQ,EAAE,OAAO,EAAE,QAAQ,IAAI,IAAI;QACnC,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,UAAU;KACtC,CAAC;IACF,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9F,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,OAA6B;IACjE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IAEhD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAEvC,IAAI,CAAC;QACH,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAExC,IAAI,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YACtB,mCAAmC;YACnC,IAAI,CAAC;gBAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAC,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;oBAAE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,qBAAqB,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/G,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW,EAAE,MAAW,EAAE,KAAa,EAAE,OAA6B;IAC7F,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAe;QACxB,GAAG;QACH,MAAM;QACN,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE;QACpB,KAAK;QACL,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;KACvD,CAAC;IAEF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,GAAG,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AACvE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,GAAG,GAAG,KAAK;IACpC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,CAAC,CAAC;IAErC,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC;YACH,IAAI,GAAG,EAAE,CAAC;gBACR,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACrB,OAAO,EAAE,CAAC;YACZ,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,GAAe,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC9C,UAAU,CAAC,QAAQ,CAAC,CAAC;oBACrB,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,0BAA0B,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU;IACxB,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;IAEhF,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACtE,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;YAC7C,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC;QACzB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,mBAAmB,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7G,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC"}
|