website-scrap-engine 0.7.1 → 0.8.0

package/lib/downloader/single.js

@@ -1,9 +1,6 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SingleThreadDownloader = void 0;
-const main_1 = require("./main");
-const logger_1 = require("../logger/logger");
-class SingleThreadDownloader extends main_1.AbstractDownloader {
+import { AbstractDownloader } from './main.js';
+import { skip } from '../logger/logger.js';
+export class SingleThreadDownloader extends AbstractDownloader {
     constructor(pathToOptions, overrideOptions) {
         super(pathToOptions, overrideOptions);
         this.pathToOptions = pathToOptions;
@@ -11,7 +8,7 @@ class SingleThreadDownloader extends main_1.AbstractDownloader {
             this.init = this.addInitialResource(this.options.initialUrl);
         }
         else {
-            this.init = this.pipeline.init(this.pipeline, this);
+            this.init = this._initOptions.then(() => this.pipeline.init(this.pipeline, this));
         }
     }
     async downloadAndProcessResource(res) {
@@ -19,7 +16,7 @@ class SingleThreadDownloader extends main_1.AbstractDownloader {
         try {
             r = await this.pipeline.download(res);
             if (!r) {
-                logger_1.skip.debug('discarded after download', res.url, res.rawUrl, res.refUrl);
+                skip.debug('discarded after download', res.url, res.rawUrl, res.refUrl);
                 return;
             }
         }
@@ -41,10 +38,10 @@ class SingleThreadDownloader extends main_1.AbstractDownloader {
         try {
             const processedResource = await this.pipeline.processAfterDownload(r, submit);
             if (!processedResource) {
-                logger_1.skip.warn('skipped downloaded resource', r.url, r.refUrl);
+                skip.warn('skipped downloaded resource', r.url, r.refUrl);
             }
             else if (await this.pipeline.saveToDisk(processedResource)) {
-                logger_1.skip.warn('downloaded resource not saved', r.url, r.refUrl);
+                skip.warn('downloaded resource not saved', r.url, r.refUrl);
             }
             if (processedResource && processedResource.redirectedUrl &&
                 processedResource.redirectedUrl !== processedResource.url) {
@@ -62,5 +59,4 @@ class SingleThreadDownloader extends main_1.AbstractDownloader {
         return super.onIdle();
     }
 }
-exports.SingleThreadDownloader = SingleThreadDownloader;
 //# sourceMappingURL=single.js.map

package/lib/downloader/single.js.map

@@ -1 +1 @@
-{"version":3,"file":"single.js","sourceRoot":"","sources":["../../src/downloader/single.ts"],"names":[],"mappings":";;;AAAA,iCAA0C;AAG1C,6CAAsC;AAGtC,MAAa,sBAAuB,SAAQ,yBAAkB;IAG5D,YAAmB,aAAqB,EACtC,eAA4E;QAC5E,KAAK,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QAFrB,kBAAa,GAAb,aAAa,CAAQ;QAGtC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE;YAC3B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;SAC9D;aAAM;YACL,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SACrD;IACH,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,GAAa;QAC5C,IAAI,CAA0B,CAAC;QAC/B,IAAI;YACF,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,CAAC,EAAE;gBACN,aAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACxE,OAAO;aACR;SACF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,sBAAsB,EAAE,GAAG,CAAC,CAAC;YACjD,OAAO;SACR;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACzC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC1C;aACF;iBAAM;gBACL,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;aACvC;QACH,CAAC,CAAC;QACF,IAAI;YACF,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtD,IAAI,CAAC,iBAAiB,EAAE;gBACtB,aAAI,CAAC,IAAI,CAAC,6BAA6B,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;aAC3D;iBAAM,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;gBAC5D,aAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;aAC7D;YACD,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;gBACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE;gBAC3D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;aACrD;SACF;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,CAAC;SAC1C;IACH,CAAC;IAED,MAAM;QACJ,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE;YACtC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;SAC7C;QACD,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;IACxB,CAAC;CACF;AA3DD,wDA2DC"}
+{"version":3,"file":"single.js","sourceRoot":"","sources":["../../src/downloader/single.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,kBAAkB,EAAC,MAAM,WAAW,CAAC;AAG7C,OAAO,EAAC,IAAI,EAAC,MAAM,qBAAqB,CAAC;AAMzC,MAAM,OAAO,sBAAuB,SAAQ,kBAAkB;IAG5D,YAAmB,aAAqB,EACtC,eAA4E;QAC5E,KAAK,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;QAFrB,kBAAa,GAAb,aAAa,CAAQ;QAGtC,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IAED,KAAK,CAAC,0BAA0B,CAAC,GAAa;QAC5C,IAAI,CAA0B,CAAC;QAC/B,IAAI,CAAC;YACH,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACP,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACxE,OAAO;YACT,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,sBAAsB,EAAE,GAAG,CAAC,CAAC;YACjD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEhC,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC;YACxC,CAAC;QACH,CAAC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,iBAAiB,GACrB,MAAM,IAAI,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtD,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACvB,IAAI,CAAC,IAAI,CAAC,6BAA6B,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YAC5D,CAAC;iBAAM,IAAI,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC7D,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;gBACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE,CAAC;gBAC5D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM;QACJ,IAAI,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;IACxB,CAAC;CACF"}

package/lib/downloader/types.d.ts

@@ -1,7 +1,6 @@
-/// <reference types="node" />
-import type { MessagePort } from 'worker_threads';
-import type { DownloadOptions } from '../options';
-import type { RawResource } from '../resource';
+import type { MessagePort } from 'node:worker_threads';
+import type { DownloadOptions } from '../options.js';
+import type { RawResource } from '../resource.js';
 export interface DownloaderStats {
     firstPeriodCount: number;
     lastPeriodTotalCount: number;
@@ -53,3 +52,4 @@ export interface DownloadWorkerMessage extends WorkerMessage<RawResource[]> {
      */
     redirectedUrl?: string;
 }
+//# sourceMappingURL=types.d.ts.map

package/lib/downloader/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/downloader/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAEhD,MAAM,WAAW,eAAe;IAC9B,gBAAgB,EAAE,MAAM,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAElC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B;;OAEG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B;;OAEG;IACH,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO;IAEtD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,GAC7C,CAAC,CAAC,KAAK,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAE;IACzC,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,sBAAsB,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,OAAO,CAC3E,SAAQ,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,CAAC,CAAC;IACR,YAAY,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC,CAAC;CACjD;AAED,oBAAY,iBAAiB;IAC3B,GAAG,IAAA;IACH,QAAQ,IAAA;CACT;AAED,MAAM,WAAW,aAAa,CAAC,CAAC,GAAG,OAAO;IACxC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,iBAAiB,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC;IACR,KAAK,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,IAAI,CAAC;CAChC;AAED,MAAM,WAAW,qBAAsB,SAAQ,aAAa,CAAC,WAAW,EAAE,CAAC;IACzE;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB"}

package/lib/downloader/types.js

@@ -1,9 +1,6 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WorkerMessageType = void 0;
-var WorkerMessageType;
+export var WorkerMessageType;
 (function (WorkerMessageType) {
     WorkerMessageType[WorkerMessageType["Log"] = 0] = "Log";
     WorkerMessageType[WorkerMessageType["Complete"] = 1] = "Complete";
-})(WorkerMessageType = exports.WorkerMessageType || (exports.WorkerMessageType = {}));
+})(WorkerMessageType || (WorkerMessageType = {}));
 //# sourceMappingURL=types.js.map

package/lib/downloader/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/downloader/types.ts"],"names":[],"mappings":";;;AAkDA,IAAY,iBAGX;AAHD,WAAY,iBAAiB;IAC3B,uDAAG,CAAA;IACH,iEAAQ,CAAA;AACV,CAAC,EAHW,iBAAiB,GAAjB,yBAAiB,KAAjB,yBAAiB,QAG5B"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/downloader/types.ts"],"names":[],"mappings":"AAkDA,MAAM,CAAN,IAAY,iBAGX;AAHD,WAAY,iBAAiB;IAC3B,uDAAG,CAAA;IACH,iEAAQ,CAAA;AACV,CAAC,EAHW,iBAAiB,KAAjB,iBAAiB,QAG5B"}

package/lib/downloader/worker-pool.d.ts

@@ -1,10 +1,8 @@
-/// <reference types="node" />
-/// <reference types="node" />
-import type { MessagePort, WorkerOptions } from 'worker_threads';
-import type { URL } from 'url';
-import { Worker } from 'worker_threads';
-import type { LogWorkerMessage } from './worker-type';
-import { PendingPromise, PendingPromiseWithBody, WorkerMessage } from './types';
+import type { MessagePort, WorkerOptions } from 'node:worker_threads';
+import { Worker } from 'node:worker_threads';
+import type { URL } from 'node:url';
+import type { LogWorkerMessage } from './worker-type.js';
+import type { PendingPromise, PendingPromiseWithBody, WorkerMessage } from './types.js';
 export interface WorkerInfo {
     readonly id: number;
     load: number;
@@ -36,3 +34,4 @@ export declare class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMes
     nextTask(): void;
     dispose(): Promise<number[]>;
 }
+//# sourceMappingURL=worker-pool.d.ts.map

package/lib/downloader/worker-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAE,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAC3C,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,UAAU,CAAC;AAElC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EACV,cAAc,EACd,sBAAsB,EACtB,aAAa,EACd,MAAM,YAAY,CAAC;AAGpB,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,cAAe,YAAW,UAAU;IAI5B,MAAM,EAAE,MAAM;IAHjC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,SAAK;gBAEU,MAAM,EAAE,MAAM;CAGlC;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC3D;AAOD,qBAAa,UAAU,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,SAAS,aAAa,GAAG,aAAa;IAQjE,QAAQ,EAAE,MAAM;IAGhB,OAAO,EAAE,MAAM;IAVxB,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,CAAM;IACpC,QAAQ,CAAC,YAAY,EAAE,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAM;IACxD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAM;IAC3D,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,aAAa,SAAK;gBAGT,QAAQ,EAAE,MAAM,EACvB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC5B,OAAO,GAAE,MAAW,EAC3B,OAAO,GAAE,aAAoC;IAgB/C,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,GAAG,IAAI;IAIjD,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI;IAOzD,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAa1D,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI;IASxD,UAAU,CACR,QAAQ,EAAE,CAAC,EACX,YAAY,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAc9D,QAAQ,IAAI,IAAI;IAsCV,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAWnC"}

package/lib/downloader/worker-pool.js

@@ -1,44 +1,17 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WorkerPool = exports.WorkerInfoImpl = void 0;
-const worker_threads_1 = require("worker_threads");
-const logger = __importStar(require("../logger/logger"));
-const types_1 = require("./types");
-class WorkerInfoImpl {
+import { Worker } from 'node:worker_threads';
+import * as logger from '../logger/logger.js';
+import { WorkerMessageType } from './types.js';
+export class WorkerInfoImpl {
     constructor(worker) {
         this.worker = worker;
         this.load = 0;
         this.id = worker.threadId;
     }
 }
-exports.WorkerInfoImpl = WorkerInfoImpl;
 function defaultWorkerFactory(filename, options) {
-    return new worker_threads_1.Worker(filename, options);
+    return new Worker(filename, options);
 }
-class WorkerPool {
+export class WorkerPool {
     constructor(coreSize, workerScript, workerData, maxLoad = -1, factory = defaultWorkerFactory) {
         this.coreSize = coreSize;
         this.maxLoad = maxLoad;
@@ -59,7 +32,7 @@ class WorkerPool {
         logger.error.error('worker error', info.id, err);
     }
     onMessage(info, message) {
-        if (message.type === types_1.WorkerMessageType.Complete) {
+        if (message.type === WorkerMessageType.Complete) {
             this.complete(info, message);
         }
         this.takeLog(info, message);
@@ -145,5 +118,4 @@ class WorkerPool {
         return numbers;
     }
 }
-exports.WorkerPool = WorkerPool;
 //# sourceMappingURL=worker-pool.js.map

package/lib/downloader/worker-pool.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,mDAAsC;AACtC,yDAA2C;AAE3C,mCAKiB;AAQjB,MAAa,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAPD,wCAOC;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,uBAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAa,UAAU;IAOrB,YACS,QAAgB,EACvB,YAAoB,EACpB,UAAmC,EAC5B,UAAkB,CAAC,CAAC,EAC3B,UAAyB,oBAAoB;QAJtC,aAAQ,GAAR,QAAQ,CAAQ;QAGhB,YAAO,GAAP,OAAO,CAAa;QAVpB,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAC/B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1D;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,aAAa,CAAC,IAAgB,EAAE,GAAU;QACxC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,SAAS,CAAC,IAAgB,EAAE,OAAsB;QAChD,IAAI,OAAO,CAAC,IAAI,KAAK,yBAAiB,CAAC,QAAQ,EAAE;YAC/C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;SAC9B;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAA2B,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,CAAC,IAAgB,EAAE,OAAyB;;QACjD,IAAI,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,CAAA,EAAE;YAClB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,OAAO;SACR;QACD,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,0CAAE,OAAO,CAAC;QACvC,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,EAAE;YACnB,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;SAC5E;aAAM;YACL,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,CAAC,CAAC;SAChE;IACH,CAAC;IAED,QAAQ,CAAC,IAAgB,EAAE,OAAsB;QAC/C,EAAE,IAAI,CAAC,IAAI,CAAC;QACZ,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpC,MAAM,OAAO,GAA+B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,UAAU,CACR,QAAW,EACX,YAA+C;QAC/C,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,IAAI,GAA8B;gBACtC,MAAM,EAAE,EAAE,IAAI,CAAC,aAAa;gBAC5B,OAAO;gBACP,MAAM;gBACN,IAAI,EAAE,QAAQ;gBACd,YAAY;aACb,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YAC7B,OAAO;SACR;QACD,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAC9B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAClE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACV,OACE,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;gBAC/C,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EACxC;gBACA,MAAM,IAAI,GACR,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,EAAE;oBACT,MAAM;iBACP;gBACD,IAAI;oBACF,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;wBACtB,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;qBAChB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBACtB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAsB,CAAC;iBACzD;gBAAC,OAAO,CAAC,EAAE;oBACV,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBACf,SAAS;iBACV;gBACD,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB;YACD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;gBAC7B,MAAM;aACP;SACF;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE;YACtC,qCAAqC;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YACxD,qCAAqC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;SAClC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AA9HD,gCA8HC"}
+{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAE3C,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAO9C,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAQ7C,MAAM,OAAO,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,UAAU;IAOrB,YACS,QAAgB,EACvB,YAAoB,EACpB,UAAmC,EAC5B,UAAkB,CAAC,CAAC,EAC3B,UAAyB,oBAAoB;QAJtC,aAAQ,GAAR,QAAQ,CAAQ;QAGhB,YAAO,GAAP,OAAO,CAAa;QAVpB,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAC/B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,aAAa,CAAC,IAAgB,EAAE,GAAU;QACxC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,SAAS,CAAC,IAAgB,EAAE,OAAsB;QAChD,IAAI,OAAO,CAAC,IAAI,KAAK,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAA2B,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,CAAC,IAAgB,EAAE,OAAyB;;QACjD,IAAI,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,CAAA,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,0CAAE,OAAO,CAAC;QACvC,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,EAAE,CAAC;YACpB,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;QAC7E,CAAC;aAAM,CAAC;YACN,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,IAAgB,EAAE,OAAsB;QAC/C,EAAE,IAAI,CAAC,IAAI,CAAC;QACZ,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpC,MAAM,OAAO,GAA+B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,UAAU,CACR,QAAW,EACX,YAA+C;QAC/C,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,IAAI,GAA8B;gBACtC,MAAM,EAAE,EAAE,IAAI,CAAC,aAAa;gBAC5B,OAAO;gBACP,MAAM;gBACN,IAAI,EAAE,QAAQ;gBACd,YAAY;aACb,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAC9B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACV,OACE,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;gBAC/C,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EACxC,CAAC;gBACD,MAAM,IAAI,GACR,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM;gBACR,CAAC;gBACD,IAAI,CAAC;oBACH,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;wBACtB,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;qBAChB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBACtB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAsB,CAAC;gBAC1D,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBACf,SAAS;gBACX,CAAC;gBACD,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACnB,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,qCAAqC;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YACxD,qCAAqC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}

package/lib/downloader/worker-type.d.ts

@@ -1,6 +1,6 @@
-import type { logLevels } from '../logger/logger-worker';
-import type * as logger from '../logger/logger';
-import { WorkerMessage, WorkerMessageType } from './types';
+import type { logLevels } from '../logger/logger-worker.js';
+import type * as logger from '../logger/logger.js';
+import type { WorkerMessage, WorkerMessageType } from './types.js';
 export interface WorkerLog<T = unknown> {
     logger: keyof typeof logger;
     level: typeof logLevels[number];
@@ -13,3 +13,4 @@ export interface WorkerTaskMessage<T> {
     readonly taskId: number;
     body: T;
 }
+//# sourceMappingURL=worker-type.d.ts.map

package/lib/downloader/worker-type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker-type.d.ts","sourceRoot":"","sources":["../../src/downloader/worker-type.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,KAAK,MAAM,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAC,aAAa,EAAE,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAEjE,MAAM,WAAW,SAAS,CAAC,CAAC,GAAG,OAAO;IACpC,MAAM,EAAE,MAAM,OAAO,MAAM,CAAC;IAC5B,KAAK,EAAE,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;IAChC,OAAO,EAAE,CAAC,EAAE,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO,CAAE,SAAQ,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAChF,IAAI,EAAE,iBAAiB,CAAC,GAAG,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC;CACT"}

package/lib/downloader/worker-type.js

@@ -1,3 +1,2 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
+export {};
 //# sourceMappingURL=worker-type.js.map

package/lib/downloader/worker.d.ts

@@ -1 +1,2 @@
 export {};
+//# sourceMappingURL=worker.d.ts.map

package/lib/downloader/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":""}

package/lib/downloader/worker.js

@@ -1,43 +1,46 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const worker_threads_1 = require("worker_threads");
-const options_1 = require("../options");
-const resource_1 = require("../resource");
-const logger_1 = require("../logger/logger");
-const util_1 = require("../util");
-const types_1 = require("./types");
-const pipeline_executor_impl_1 = require("./pipeline-executor-impl");
-const { pathToOptions, overrideOptions } = worker_threads_1.workerData;
-const options = 
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-(0, options_1.mergeOverrideOptions)((0, util_1.importDefaultFromPath)(pathToOptions), overrideOptions);
-const pipeline = new pipeline_executor_impl_1.PipelineExecutorImpl(options, options.req, options);
-options.configureLogger(options.localRoot, options.logSubDir || '');
-const init = pipeline.init(pipeline);
-worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.addListener('message', async (msg) => {
+import { parentPort, workerData } from 'node:worker_threads';
+import { mergeOverrideOptions } from '../options.js';
+import { normalizeResource, prepareResourceForClone } from '../resource.js';
+import { skip } from '../logger/logger.js';
+import { importDefaultFromPath } from '../util.js';
+import { WorkerMessageType } from './types.js';
+import { PipelineExecutorImpl } from './pipeline-executor-impl.js';
+const { pathToOptions, overrideOptions } = workerData;
+const asyncOptions = importDefaultFromPath(pathToOptions);
+const asyncPipeline = asyncOptions.then(options => {
+    options = mergeOverrideOptions(options, overrideOptions);
+    const pipeline = new PipelineExecutorImpl(options, options.req, options);
+    options.configureLogger(options.localRoot, options.logSubDir || '');
+    const init = pipeline.init(pipeline);
+    if (init && init.then) {
+        return init.then(() => pipeline);
+    }
+    return pipeline;
+});
+parentPort === null || parentPort === void 0 ? void 0 : parentPort.addListener('message', async (msg) => {
     const collectedResource = [];
     let error;
     let redirectedUrl;
     try {
-        await init;
+        const pipeline = await asyncPipeline;
         const res = msg.body;
-        const downloadResource = (0, resource_1.normalizeResource)(res);
+        const downloadResource = normalizeResource(res);
         const submit = (resources) => {
             if (Array.isArray(resources)) {
                 for (let i = 0; i < resources.length; i++) {
-                    collectedResource.push((0, resource_1.prepareResourceForClone)(resources[i]));
+                    collectedResource.push(prepareResourceForClone(resources[i]));
                 }
             }
             else {
-                collectedResource.push((0, resource_1.prepareResourceForClone)(resources));
+                collectedResource.push(prepareResourceForClone(resources));
             }
         };
         const processedResource = await pipeline.processAfterDownload(downloadResource, submit);
         if (!processedResource) {
-            logger_1.skip.warn('skipped downloaded resource', downloadResource.url, downloadResource.refUrl);
+            skip.warn('skipped downloaded resource', downloadResource.url, downloadResource.refUrl);
         }
         else if (await pipeline.saveToDisk(processedResource)) {
-            logger_1.skip.warn('downloaded resource not saved', downloadResource.url, downloadResource.refUrl);
+            skip.warn('downloaded resource not saved', downloadResource.url, downloadResource.refUrl);
         }
         if (processedResource && processedResource.redirectedUrl &&
             processedResource.redirectedUrl !== processedResource.url) {
@@ -45,18 +48,40 @@ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ?
         }
     }
     catch (e) {
-        // TODO: handle if object could not be cloned here
-        error = e;
+        // handle if object could not be cloned here
+        // https://github.com/website-local/website-scrap-engine/issues/340
+        try {
+            // should always be
+            if (typeof structuredClone === 'function') {
+                error = structuredClone(e);
+            }
+            else {
+                // this is the old behavior before this
+                error = e;
+            }
+        }
+        catch (_a) {
+            // can not clone, so no need to get the full error here
+            if (e && typeof e === 'object') {
+                const clone = {};
+                for (const k in e) {
+                    clone[k] = String(e[k]);
+                }
+            }
+            else {
+                error = String(e);
+            }
+        }
     }
     finally {
         const message = {
             taskId: msg.taskId,
-            type: types_1.WorkerMessageType.Complete,
+            type: WorkerMessageType.Complete,
             body: collectedResource,
             error,
             redirectedUrl
         };
-        worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ? void 0 : worker_threads_1.parentPort.postMessage(message);
+        parentPort === null || parentPort === void 0 ? void 0 : parentPort.postMessage(message);
     }
 });
 //# sourceMappingURL=worker.js.map

package/lib/downloader/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":";;AAAA,mDAAsD;AACtD,wCAIoB;AAEpB,0CAKqB;AACrB,6CAAsC;AACtC,kCAA8C;AAC9C,mCAAiE;AACjE,qEAA8D;AAK9D,MAAM,EAAC,aAAa,EAAE,eAAe,EAAC,GAGlC,2BAAU,CAAC;AAEf,MAAM,OAAO;AACX,8DAA8D;AAC9D,IAAA,8BAAoB,EAAC,IAAA,4BAAqB,EAAC,aAAa,CAAC,EAAE,eAAe,CAAC,CAAC;AAE9E,MAAM,QAAQ,GACZ,IAAI,6CAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAE1D,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;AAEpE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAErC,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,GAAmC,EAAE,EAAE;IAC/E,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAC5C,IAAI,KAA6B,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI;QACF,MAAM,IAAI,CAAC;QACX,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;QACrB,MAAM,gBAAgB,GAAqB,IAAA,4BAAiB,EAAC,GAAG,CAAqB,CAAC;QACtF,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACzC,iBAAiB,CAAC,IAAI,CAAC,IAAA,kCAAuB,EAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC/D;aACF;iBAAM;gBACL,iBAAiB,CAAC,IAAI,CAAC,IAAA,kCAAuB,EAAC,SAAS,CAAC,CAAC,CAAC;aAC5D;QACH,CAAC,CAAC;QACF,MAAM,iBAAiB,GACrB,MAAM,QAAQ,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE;YACtB,aAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;SAClD;aAAM,IAAI,MAAM,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;YACvD,aAAI,CAAC,IAAI,CAAC,+BAA+B,EACvC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;SAClD;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;YACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE;YAC3D,aAAa,GAAG,iBAAiB,CAAC,aAAa,CAAC;SACjD;KACF;IAAC,OAAO,CAAC,EAAE;QACV,kDAAkD;QAClD,KAAK,GAAG,CAAC,CAAC;KACX;YAAS;QACR,MAAM,OAAO,GAA0B;YACrC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,IAAI,EAAE,yBAAiB,CAAC,QAAQ;YAChC,IAAI,EAAE,iBAAiB;YACvB,KAAK;YACL,aAAa;SACd,CAAC;QACF,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,OAAO,CAAC,CAAC;KAClC;AAEH,CAAC,CAAC,CAAC"}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,UAAU,EAAC,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAMnD,OAAO,EAAC,iBAAiB,EAAE,uBAAuB,EAAC,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAC,IAAI,EAAC,MAAM,qBAAqB,CAAC;AACzC,OAAO,EAAC,qBAAqB,EAAC,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAKjE,MAAM,EAAC,aAAa,EAAE,eAAe,EAAC,GAGlC,UAAU,CAAC;AAEf,MAAM,YAAY,GAA6B,qBAAqB,CAAC,aAAa,CAAC,CAAC;AAEpF,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;IAChD,OAAO,GAAG,oBAAoB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAEzD,MAAM,QAAQ,GACZ,IAAI,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,IAAI,IAAK,IAAsB,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,GAAmC,EAAE,EAAE;IAC/E,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAC5C,IAAI,KAA6B,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;QACrB,MAAM,gBAAgB,GAAqB,iBAAiB,CAAC,GAAG,CAAqB,CAAC;QACtF,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,iBAAiB,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,iBAAiB,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC;QACF,MAAM,iBAAiB,GACrB,MAAM,QAAQ,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;aAAM,IAAI,MAAM,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,IAAI,CAAC,+BAA+B,EACvC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;YACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE,CAAC;YAC5D,aAAa,GAAG,iBAAiB,CAAC,aAAa,CAAC;QAClD,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,4CAA4C;QAC5C,mEAAmE;QACnE,IAAI,CAAC;YACH,mBAAmB;YACnB,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,uCAAuC;gBACvC,KAAK,GAAG,CAAC,CAAC;YACZ,CAAC;QACH,CAAC;QAAC,WAAM,CAAC;YACP,uDAAuD;YACvD,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAA2B,EAAE,CAAC;gBACzC,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAE,CAA6B,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,GAA0B;YACrC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,IAAI,EAAE,iBAAiB,CAAC,QAAQ;YAChC,IAAI,EAAE,iBAAiB;YACvB,KAAK;YACL,aAAa;SACd,CAAC;QACF,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;AAEH,CAAC,CAAC,CAAC"}

package/lib/index.d.ts

@@ -1,8 +1,9 @@
-export * as logger from './logger/logger';
-export * as downloader from './downloader/index';
-export * as lifeCycle from './life-cycle/index';
-export * as io from './io';
-export * as options from './options';
-export * as resource from './resource';
-export { SourceDefinition } from './sources';
-export * as util from './util';
+export * as logger from './logger/logger.js';
+export * as downloader from './downloader/index.js';
+export * as lifeCycle from './life-cycle/index.js';
+export * as io from './io.js';
+export * as options from './options.js';
+export * as resource from './resource.js';
+export type { SourceDefinition } from './sources.js';
+export * as util from './util.js';
+//# sourceMappingURL=index.d.ts.map

package/lib/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,SAAS,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AACxC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAC1C,YAAY,EAAC,gBAAgB,EAAC,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC"}

package/lib/index.js

@@ -1,34 +1,8 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.util = exports.resource = exports.options = exports.io = exports.lifeCycle = exports.downloader = exports.logger = void 0;
-exports.logger = __importStar(require("./logger/logger"));
-exports.downloader = __importStar(require("./downloader/index"));
-exports.lifeCycle = __importStar(require("./life-cycle/index"));
-exports.io = __importStar(require("./io"));
-exports.options = __importStar(require("./options"));
-exports.resource = __importStar(require("./resource"));
-exports.util = __importStar(require("./util"));
+export * as logger from './logger/logger.js';
+export * as downloader from './downloader/index.js';
+export * as lifeCycle from './life-cycle/index.js';
+export * as io from './io.js';
+export * as options from './options.js';
+export * as resource from './resource.js';
+export * as util from './util.js';
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,0DAA0C;AAC1C,iEAAiD;AACjD,gEAAgD;AAChD,2CAA2B;AAC3B,qDAAqC;AACrC,uDAAuC;AAEvC,+CAA+B"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,oBAAoB,CAAC;AAC7C,OAAO,KAAK,UAAU,MAAM,uBAAuB,CAAC;AACpD,OAAO,KAAK,SAAS,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,OAAO,MAAM,cAAc,CAAC;AACxC,OAAO,KAAK,QAAQ,MAAM,eAAe,CAAC;AAE1C,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC"}

package/lib/io.d.ts

@@ -1,3 +1,4 @@
-import type { ResourceBody, ResourceEncoding } from './resource';
+import type { ResourceBody, ResourceEncoding } from './resource.js';
 export declare const mkdirRetry: (dir: string, retry?: number) => Promise<void>;
 export declare const writeFile: (filePath: string, data: ResourceBody, encoding: ResourceEncoding, mtime?: number | void | undefined, atime?: number | void | undefined) => Promise<void>;
+//# sourceMappingURL=io.d.ts.map

package/lib/io.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"io.d.ts","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,eAAe,CAAC;AAGlE,eAAO,MAAM,UAAU,QAAe,MAAM,qBAAc,OAAO,CAAC,IAAI,CAqBrE,CAAC;AAEF,eAAO,MAAM,SAAS,aACV,MAAM,QACV,YAAY,YACR,gBAAgB,2EAGzB,OAAO,CAAC,IAAI,CAoCd,CAAC"}

package/lib/io.js

@@ -1,27 +1,21 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.writeFile = exports.mkdirRetry = void 0;
-const fs_1 = __importDefault(require("fs"));
-const path_1 = require("path");
-const mkdirp_1 = require("mkdirp");
-const logger_1 = require("./logger/logger");
-const mkdirRetry = async (dir, retry = 3) => {
+import fs from 'node:fs';
+import { dirname } from 'node:path';
+import { mkdirp } from 'mkdirp';
+import { error as errorLogger, mkdir as mkdirLogger } from './logger/logger.js';
+export const mkdirRetry = async (dir, retry = 3) => {
     let error;
     for (let i = 0; i < retry; i++) {
         error = undefined;
         try {
-            await (0, mkdirp_1.mkdirp)(dir);
+            await mkdirp(dir);
         }
         catch (e) {
             error = e;
             if (i > 0) {
-                logger_1.mkdir.debug('mkdir', dir, 'fail', i, 'times', e);
+                mkdirLogger.debug('mkdir', dir, 'fail', i, 'times', e);
             }
             else {
-                logger_1.mkdir.trace('mkdir', dir, 'fail', i, 'times', e);
+                mkdirLogger.trace('mkdir', dir, 'fail', i, 'times', e);
             }
             continue;
         }
@@ -32,14 +26,13 @@ const mkdirRetry = async (dir, retry = 3) => {
         throw error;
     }
 };
-exports.mkdirRetry = mkdirRetry;
-const writeFile = async (filePath, data, encoding, mtime, atime) => {
-    const dir = (0, path_1.dirname)(filePath);
-    if (!fs_1.default.existsSync(dir)) {
-        await (0, exports.mkdirRetry)(dir);
+export const writeFile = async (filePath, data, encoding, mtime, atime) => {
+    const dir = dirname(filePath);
+    if (!fs.existsSync(dir)) {
+        await mkdirRetry(dir);
     }
     let fileData;
-    let options;
+    let options = void 0;
     if (typeof data === 'string') {
         fileData = data;
         options = { encoding };
@@ -58,10 +51,10 @@ const writeFile = async (filePath, data, encoding, mtime, atime) => {
         throw new TypeError('Type of data not supported.');
     }
     if (options) {
-        await fs_1.default.promises.writeFile(filePath, fileData, options);
+        await fs.promises.writeFile(filePath, fileData, options);
     }
     else {
-        await fs_1.default.promises.writeFile(filePath, fileData);
+        await fs.promises.writeFile(filePath, fileData);
     }
     // void and NaN check
     if (mtime) {
@@ -69,12 +62,11 @@ const writeFile = async (filePath, data, encoding, mtime, atime) => {
             atime = mtime;
         }
         try {
-            await fs_1.default.promises.utimes(filePath, atime, mtime);
+            await fs.promises.utimes(filePath, atime, mtime);
         }
         catch (e) {
-            logger_1.error.warn('skipping utimes ' + filePath, e);
+            errorLogger.warn('skipping utimes ' + filePath, e);
         }
     }
 };
-exports.writeFile = writeFile;
 //# sourceMappingURL=io.js.map

package/lib/io.js.map

@@ -1 +1 @@
-{"version":3,"file":"io.js","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":";;;;;;AACA,4CAAoB;AACpB,+BAA6B;AAC7B,mCAA8B;AAE9B,4CAA2E;AAEpE,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAAiB,EAAE;IACxE,IAAI,KAAqB,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9B,KAAK,GAAG,SAAS,CAAC;QAClB,IAAI;YACF,MAAM,IAAA,eAAM,EAAC,GAAG,CAAC,CAAC;SACnB;QAAC,OAAO,CAAC,EAAE;YACV,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,EAAE;gBACT,cAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;aACxD;iBAAM;gBACL,cAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;aACxD;YACD,SAAS;SACV;QACD,KAAK,GAAG,SAAS,CAAC;QAClB,OAAO;KACR;IACD,IAAI,KAAK,EAAE;QACT,MAAM,KAAK,CAAC;KACb;AACH,CAAC,CAAC;AArBW,QAAA,UAAU,cAqBrB;AAEK,MAAM,SAAS,GAAG,KAAK,EAC5B,QAAgB,EAChB,IAAkB,EAClB,QAA0B,EAC1B,KAAqB,EACrB,KAAqB,EACN,EAAE;IACjB,MAAM,GAAG,GAAW,IAAA,cAAO,EAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QACvB,MAAM,IAAA,kBAAU,EAAC,GAAG,CAAC,CAAC;KACvB;IACD,IAAI,QAA6B,CAAC;IAClC,IAAI,OAAqC,CAAC;IAC1C,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC5B,QAAQ,GAAG,IAAI,CAAC;QAChB,OAAO,GAAG,EAAC,QAAQ,EAAC,CAAC;KACtB;SAAM,IAAI,IAAI,YAAY,WAAW,EAAE;QACtC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;SAAM,IAAI,IAAI,YAAY,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC9D,QAAQ,GAAG,IAAI,CAAC;KACjB;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACnC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;KACvE;SAAM;QACL,oBAAoB;QACpB,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;KACpD;IACD,IAAI,OAAO,EAAE;QACX,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;KAC1D;SAAM;QACL,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;KACjD;IACD,qBAAqB;IACrB,IAAI,KAAK,EAAE;QACT,IAAI,CAAC,KAAK,EAAE;YACV,KAAK,GAAG,KAAK,CAAC;SACf;QACD,IAAI;YACF,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACV,cAAW,CAAC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;SACpD;KACF;AACH,CAAC,CAAC;AA1CW,QAAA,SAAS,aA0CpB"}
+{"version":3,"file":"io.js","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAC,OAAO,EAAC,MAAM,WAAW,CAAC;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AAE9B,OAAO,EAAC,KAAK,IAAI,WAAW,EAAE,KAAK,IAAI,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAE9E,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAAiB,EAAE;IACxE,IAAI,KAAqB,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,KAAK,GAAG,SAAS,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACV,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;YACzD,CAAC;YACD,SAAS;QACX,CAAC;QACD,KAAK,GAAG,SAAS,CAAC;QAClB,OAAO;IACT,CAAC;IACD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,QAAgB,EAChB,IAAkB,EAClB,QAA0B,EAC1B,KAAqB,EACrB,KAAqB,EACN,EAAE;IACjB,MAAM,GAAG,GAAW,OAAO,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IACD,IAAI,QAA6B,CAAC;IAClC,IAAI,OAAO,GAAiC,KAAK,CAAC,CAAC;IACnD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,QAAQ,GAAG,IAAI,CAAC;QAChB,OAAO,GAAG,EAAC,QAAQ,EAAC,CAAC;IACvB,CAAC;SAAM,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QACvC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;SAAM,IAAI,IAAI,YAAY,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/D,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACxE,CAAC;SAAM,CAAC;QACN,oBAAoB;QACpB,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;SAAM,CAAC;QACN,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAClD,CAAC;IACD,qBAAqB;IACrB,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,KAAK,CAAC;QAChB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,WAAW,CAAC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;AACH,CAAC,CAAC"}

package/lib/life-cycle/adapters.d.ts

@@ -1,8 +1,9 @@
-import { Resource, ResourceType } from '../resource';
-import type { AsyncResult, DownloadResource, LinkRedirectFunc, ProcessResourceAfterDownloadFunc, ProcessResourceBeforeDownloadFunc, SubmitResourceFunc } from './types';
-import type { StaticDownloadOptions } from '../options';
-import type { PipelineExecutor } from './pipeline-executor';
-import type { Cheerio, CheerioStatic } from '../types';
+import type { Resource } from '../resource.js';
+import { ResourceType } from '../resource.js';
+import type { AsyncResult, DownloadResource, LinkRedirectFunc, ProcessResourceAfterDownloadFunc, ProcessResourceBeforeDownloadFunc, SubmitResourceFunc } from './types.js';
+import type { StaticDownloadOptions } from '../options.js';
+import type { PipelineExecutor } from './pipeline-executor.js';
+import type { Cheerio, CheerioStatic } from '../types.js';
 export interface SkipProcessFunc {
     (url: string, element: Cheerio | null, parent: Resource | null): boolean;
 }
@@ -36,3 +37,4 @@ export interface AsyncHtmlProcessFunc {
     }): AsyncResult<CheerioStatic>;
 }
 export declare const processHtmlAsync: (fn: AsyncHtmlProcessFunc) => ProcessResourceAfterDownloadFunc;
+//# sourceMappingURL=adapters.d.ts.map

package/lib/life-cycle/adapters.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,QAAQ,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,gCAAgC,EAChC,iCAAiC,EACjC,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAC,OAAO,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAExD,MAAM,WAAW,eAAe;IAC9B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;CAC1E;AAED,eAAO,MAAM,WAAW,OAAQ,eAAe,KAAG,gBACoB,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC;CAC1B;AAED,eAAO,MAAM,YAAY,OAAQ,gBAAgB,KAAG,iCAMjD,CAAC;AAGJ,MAAM,WAAW,sBAAsB;IACrC,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;CACtF;AAED,eAAO,MAAM,UAAU,OAAQ,sBAAsB,KAAG,iCAIrD,CAAC;AAEJ,MAAM,WAAW,mBAAmB;IAClC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,eAAO,MAAM,eAAe,OAAQ,mBAAmB,KAAG,iCAUvD,CAAC;AAEJ,eAAO,MAAM,cAAc,OAAQ,mBAAmB,KAAG,gCAMtD,CAAC;AAEJ,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAclC;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,QAAQ,GAAG;QAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;KAAE,GAAG,aAAa,CAAC;CAChF;AAED,eAAO,MAAM,SAAS,QACf,gBAAgB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;CAAE,WAC1C,qBAAqB,KAC7B,aAOF,CAAC;AAEF,eAAO,MAAM,WAAW,OAAQ,eAAe,KAAG,gCAS/C,CAAC;AAGJ,MAAM,WAAW,oBAAoB;IACnC,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,QAAQ,GAAG;QAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;KAAE,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;CAC7F;AAED,eAAO,MAAM,gBAAgB,OAAQ,oBAAoB,KAAG,gCASzD,CAAC"}