website-scrap-engine 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAE3C,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAO9C,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAQ7C,MAAM,OAAO,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,UAAU;IAOrB,YACS,QAAgB,EAChB,YAAoB,EACpB,UAAmC,EACnC,UAAkB,CAAC,CAAC,EACpB,UAAyB,oBAAoB;QAJ7C,aAAQ,GAAR,QAAQ,CAAQ;QAChB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAyB;QACnC,YAAO,GAAP,OAAO,CAAa;QACpB,YAAO,GAAP,OAAO,CAAsC;QAX7C,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAE3C,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAO9C,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAQ7C,MAAM,OAAO,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,UAAU;IAOrB,YACS,QAAgB,EAChB,YAAoB,EACpB,UAAmC,EACnC,UAAkB,CAAC,CAAC,EACpB,UAAyB,oBAAoB;QAJ7C,aAAQ,GAAR,QAAQ,CAAQ;QAChB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAyB;QACnC,YAAO,GAAP,OAAO,CAAa;QACpB,YAAO,GAAP,OAAO,CAAsC;QAX7C,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAY,CAAC,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAC/B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,aAAa,CAAC,IAAgB,EAAE,GAAU;QACxC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,SAAS,CAAC,IAAgB,EAAE,OAAsB;QAChD,IAAI,OAAO,CAAC,IAAI,KAAK,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAA2B,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,CAAC,IAAgB,EAAE,OAAyB;;QACjD,IAAI,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,CAAA,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,0CAAE,OAAO,CAAC;QACvC,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,EAAE,CAAC;YACpB,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;QAC7E,CAAC;aAAM,CAAC;YACN,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,IAAgB,EAAE,OAAsB;QAC/C,EAAE,IAAI,CAAC,IAAI,CAAC;QACZ,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpC,MAAM,OAAO,GAA+B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,UAAU,CACR,QAAW,EACX,YAA+C;QAC/C,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,IAAI,GAA8B;gBACtC,MAAM,EAAE,EAAE,IAAI,CAAC,aAAa;gBAC5B,OAAO;gBACP,MAAM;gBACN,IAAI,EAAE,QAAQ;gBACd,YAAY;aACb,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAC9B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACV,OACE,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;gBAC/C,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EACxC,CAAC;gBACD,MAAM,IAAI,GACR,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC;oBACH,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;wBACtB,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;qBAChB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBACtB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAsB,CAAC;gBAC1D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBACf,SAAS;gBACX,CAAC;gBACD,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACnB,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,qCAAqC;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YACxD,qCAAqC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "website-scrap-engine",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.5",
|
|
4
4
|
"description": "Configurable website scraper in typescript",
|
|
5
5
|
"main": "lib",
|
|
6
6
|
"types": "lib",
|
|
@@ -24,20 +24,20 @@
|
|
|
24
24
|
"got": "^13.0.0",
|
|
25
25
|
"log4js": "^6.9.1",
|
|
26
26
|
"mkdirp": "^3.0.1",
|
|
27
|
-
"p-queue": "^8.1.
|
|
28
|
-
"srcset": "^5.0.
|
|
27
|
+
"p-queue": "^8.1.1",
|
|
28
|
+
"srcset": "^5.0.3",
|
|
29
29
|
"urijs": "^1.19.11"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
|
-
"@jest/globals": "^30.
|
|
33
|
-
"@types/node": "^
|
|
34
|
-
"@types/urijs": "^1.19.
|
|
35
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
36
|
-
"@typescript-eslint/parser": "^8.
|
|
37
|
-
"eslint": "^9.
|
|
38
|
-
"jest": "^30.0
|
|
39
|
-
"ts-jest": "^29.4.
|
|
40
|
-
"typescript": "^5.9.
|
|
32
|
+
"@jest/globals": "^30.1.1",
|
|
33
|
+
"@types/node": "^25.0.10",
|
|
34
|
+
"@types/urijs": "^1.19.26",
|
|
35
|
+
"@typescript-eslint/eslint-plugin": "^8.53.1",
|
|
36
|
+
"@typescript-eslint/parser": "^8.53.0",
|
|
37
|
+
"eslint": "^9.39.2",
|
|
38
|
+
"jest": "^30.2.0",
|
|
39
|
+
"ts-jest": "^29.4.6",
|
|
40
|
+
"typescript": "^5.9.3"
|
|
41
41
|
},
|
|
42
42
|
"files": [
|
|
43
43
|
".editorconfig",
|
|
@@ -55,7 +55,7 @@ export class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMessage> {
|
|
|
55
55
|
this.workers[i].worker.addListener('message',
|
|
56
56
|
msg => this.onMessage(this.workers[i], msg));
|
|
57
57
|
this.workers[i].worker.addListener('error',
|
|
58
|
-
err => this.workerOnError(this.workers[i], err));
|
|
58
|
+
err => this.workerOnError(this.workers[i], err as Error));
|
|
59
59
|
ready.push(new Promise(resolve =>
|
|
60
60
|
this.workers[i].worker.addListener('online',resolve)));
|
|
61
61
|
}
|