website-scrap-engine 0.7.2 → 0.8.0

package/lib/sources.js

@@ -1,12 +1,9 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.sources = void 0;
-const resource_1 = require("./resource");
+import { ResourceType } from './resource.js';
 // https://github.com/website-scraper/node-website-scraper
 // /blob/66f5113475843ae86f12ea9e5d2ebcfade9f056e/lib/config/defaults.js
-exports.sources = [
-    { selector: 'style', type: resource_1.ResourceType.CssInline },
-    { selector: '[style]', attr: 'style', type: resource_1.ResourceType.CssInline },
+export const sources = [
+    { selector: 'style', type: ResourceType.CssInline },
+    { selector: '[style]', attr: 'style', type: ResourceType.CssInline },
     { selector: 'img', attr: 'src' },
     { selector: 'img', attr: 'srcset' },
     { selector: 'input', attr: 'src' },
@@ -14,7 +11,7 @@ exports.sources = [
     { selector: 'embed', attr: 'src' },
     { selector: 'param[name="movie"]', attr: 'value' },
     { selector: 'script', attr: 'src' },
-    { selector: 'link[rel="stylesheet"]', attr: 'href', type: resource_1.ResourceType.Css },
+    { selector: 'link[rel="stylesheet"]', attr: 'href', type: ResourceType.Css },
     { selector: 'link[rel*="icon"]', attr: 'href' },
     { selector: 'link[rel*="preload"]', attr: 'href' },
     // prefetch links not included by default
@@ -38,9 +35,9 @@ exports.sources = [
     { selector: 'audio', attr: 'src' },
     { selector: 'audio source', attr: 'src' },
     { selector: 'audio track', attr: 'src' },
-    { selector: 'frame', attr: 'src', type: resource_1.ResourceType.Html },
-    { selector: 'iframe', attr: 'src', type: resource_1.ResourceType.Html },
-    { selector: 'a', attr: 'href', type: resource_1.ResourceType.Html },
+    { selector: 'frame', attr: 'src', type: ResourceType.Html },
+    { selector: 'iframe', attr: 'src', type: ResourceType.Html },
+    { selector: 'a', attr: 'href', type: ResourceType.Html },
     // https://github.com/website-scraper/node-website-scraper/pull/408
     { selector: '[background]', attr: 'background' },
 ].map((obj) => {
@@ -48,7 +45,7 @@ exports.sources = [
         obj.selector += `[${obj.attr}]`;
     }
     if (!obj.type) {
-        obj.type = resource_1.ResourceType.Binary;
+        obj.type = ResourceType.Binary;
     }
     return obj;
 });

package/lib/sources.js.map

@@ -1 +1 @@
-{"version":3,"file":"sources.js","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":";;;AAAA,yCAAwC;AAQxC,0DAA0D;AAC1D,wEAAwE;AAC3D,QAAA,OAAO,GAAuB;IACzC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAY,CAAC,SAAS,EAAC;IACjD,EAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAY,CAAC,SAAS,EAAC;IAClE,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAC;IAC9B,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAC;IACjC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAC;IAClC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAC;IAChD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAC;IACjC,EAAC,QAAQ,EAAE,wBAAwB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAY,CAAC,GAAG,EAAC;IAC1E,EAAC,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAC;IAC7C,EAAC,QAAQ,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,EAAC;IAChD,yCAAyC;IACzC,qDAAqD;IACrD,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,YAAY,EAAC;IACrD,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAC;IACvC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC5C,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAC;IACnC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IACzD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IAC1D,EAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IACtD,mEAAmE;IACnE,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAC;CAC/C,CAAC,GAAG,CAAC,CAAC,GAA8B,EAAE,EAAE;IACvC,IAAI,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE;QAC/D,GAAG,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC;KACjC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;QACb,GAAG,CAAC,IAAI,GAAG,uBAAY,CAAC,MAAM,CAAC;KAChC;IACD,OAAO,GAAuB,CAAC;AACjC,CAAC,CAAC,CAAC"}
+{"version":3,"file":"sources.js","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,eAAe,CAAC;AAQ3C,0DAA0D;AAC1D,wEAAwE;AACxE,MAAM,CAAC,MAAM,OAAO,GAAuB;IACzC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,SAAS,EAAC;IACjD,EAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,SAAS,EAAC;IAClE,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAC;IAC9B,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAC;IACjC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAC;IAClC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAC;IAChD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAC;IACjC,EAAC,QAAQ,EAAE,wBAAwB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,GAAG,EAAC;IAC1E,EAAC,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAC;IAC7C,EAAC,QAAQ,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,EAAC;IAChD,yCAAyC;IACzC,qDAAqD;IACrD,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,YAAY,EAAC;IACrD,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAC;IACvC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC5C,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAC;IACnC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC;IACzD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC;IAC1D,EAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAC;IACtD,mEAAmE;IACnE,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAC;CAC/C,CAAC,GAAG,CAAC,CAAC,GAA8B,EAAE,EAAE;IACvC,IAAI,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QAChE,GAAG,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC;IAClC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,GAAG,CAAC,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC;IACjC,CAAC;IACD,OAAO,GAAuB,CAAC;AACjC,CAAC,CAAC,CAAC"}

package/lib/types.d.ts

@@ -3,3 +3,4 @@ export type CheerioStatic = ReturnType<typeof load>;
 export type Cheerio = ReturnType<CheerioStatic>;
 export type CheerioOptionsInterface = NonNullable<Parameters<typeof load>[1]>;
 export type CheerioElement = Cheerio[number];
+//# sourceMappingURL=types.d.ts.map

package/lib/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAGlC,MAAM,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC;AACpD,MAAM,MAAM,OAAO,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC;AAChD,MAAM,MAAM,uBAAuB,GAAG,WAAW,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9E,MAAM,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC"}

package/lib/types.js

@@ -1,3 +1,2 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
+export {};
 //# sourceMappingURL=types.js.map

package/lib/util.d.ts

@@ -1,10 +1,10 @@
-import type { ResourceBody, ResourceEncoding } from './resource';
+import type { ResourceBody, ResourceEncoding } from './resource.js';
 export declare const sleep: (ms: number) => Promise<void>;
 export declare const escapePath: (str: string) => string;
-export declare const isSiteMap: (url?: string) => boolean | '' | void;
+export declare const isSiteMap: (url?: string) => boolean | "" | void;
 export declare const arrayToMap: (array: (string | number)[], freeze?: boolean) => Record<string | number, number>;
 export declare const toString: (body: ResourceBody, encoding: ResourceEncoding) => string;
-export declare const importDefaultFromPath: (path: string) => any;
+export declare const importDefaultFromPath: <T>(path: string) => Promise<T>;
 export declare const orderUrlSearch: (search: string) => string;
 export declare const simpleHashString: (str: string) => string;
 export declare const hasOwnProperty: (v: PropertyKey) => boolean;
@@ -19,3 +19,4 @@ export declare const weakAssign: <T, U>(target: T, source: U) => T & U;
  * @param url
  */
 export declare const isUrlHttp: (url: string) => boolean;
+//# sourceMappingURL=util.d.ts.map

package/lib/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,eAAe,CAAC;AAIlE,eAAO,MAAM,KAAK,OAAQ,MAAM,KAAG,OAAO,CAAC,IAAI,CACN,CAAC;AAE1C,eAAO,MAAM,UAAU,QAAS,MAAM,KAAG,MACD,CAAC;AAEzC,eAAO,MAAM,SAAS,SAAU,MAAM,KAAG,OAAO,GAAG,EAAE,GAAG,IAEY,CAAC;AAErE,eAAO,MAAM,UAAU,UAAW,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,WAAW,OAAO,KACrE,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,MAAM,CAM/B,CAAC;AAEF,eAAO,MAAM,QAAQ,SAAU,YAAY,YAAY,gBAAgB,KAAG,MAezE,CAAC;AAEF,eAAO,MAAM,qBAAqB,GAAI,CAAC,QAAQ,MAAM,KAAG,OAAO,CAAC,CAAC,CAIhE,CAAC;AAEF,eAAO,MAAM,cAAc,WAAY,MAAM,KAAG,MAmB/C,CAAC;AAEF,eAAO,MAAM,gBAAgB,QAAS,MAAM,KAAG,MAQzB,CAAC;AAEvB,eAAO,MAAM,cAAc,6BAAkC,CAAC;AAE9D;;;;GAIG;AACH,eAAO,MAAM,UAAU,GAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,KAAG,CAAC,GAAG,CAW3D,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,OACiB,CAAC"}

package/lib/util.js

@@ -1,25 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isUrlHttp = exports.weakAssign = exports.hasOwnProperty = exports.simpleHashString = exports.orderUrlSearch = exports.importDefaultFromPath = exports.toString = exports.arrayToMap = exports.isSiteMap = exports.escapePath = exports.sleep = void 0;
-const crypto_1 = require("crypto");
+import { createHash } from 'node:crypto';
 const forbiddenChar = /[:*?"<>|&]|%3A|%2A|%3F|%22|%3C|%3E|%7C|%26/ig;
-const sleep = (ms) => new Promise(r => setTimeout(r, ms | 0));
-exports.sleep = sleep;
-const escapePath = (str) => str && str.replace(forbiddenChar, '_');
-exports.escapePath = escapePath;
-const isSiteMap = (url) => url &&
+export const sleep = (ms) => new Promise(r => setTimeout(r, ms | 0));
+export const escapePath = (str) => str && str.replace(forbiddenChar, '_');
+export const isSiteMap = (url) => url &&
     url.includes('/sitemaps/') &&
     (url.endsWith('sitemap.xml') || url.endsWith('sitemap_other.xml'));
-exports.isSiteMap = isSiteMap;
-const arrayToMap = (array, freeze) => {
+export const arrayToMap = (array, freeze) => {
     const obj = {};
     for (const item of array) {
         obj[item] = 1;
     }
     return freeze ? Object.freeze(obj) : obj;
 };
-exports.arrayToMap = arrayToMap;
-const toString = (body, encoding) => {
+export const toString = (body, encoding) => {
     let stringValue;
     if (Buffer.isBuffer(body)) {
         stringValue = body.toString(encoding || 'utf8');
@@ -38,18 +31,12 @@ const toString = (body, encoding) => {
     }
     return stringValue;
 };
-exports.toString = toString;
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const importDefaultFromPath = (path) => {
-    // eslint-disable-next-line @typescript-eslint/no-var-requires,@typescript-eslint/no-explicit-any
-    const mod = require(path);
-    if (mod && mod.__esModule && mod.default) {
-        return mod.default;
-    }
-    return mod;
+export const importDefaultFromPath = (path) => {
+    return import(path).then(mod => {
+        return mod.default || mod;
+    });
 };
-exports.importDefaultFromPath = importDefaultFromPath;
-const orderUrlSearch = (search) => {
+export const orderUrlSearch = (search) => {
     const parts = (search[0] === '?' ? search.slice(1) : search)
         .split('&');
     const searchKeys = [], searchMap = {};
@@ -69,8 +56,7 @@ const orderUrlSearch = (search) => {
         .map(k => { var _a; return (_a = searchMap[k]) === null || _a === void 0 ? void 0 : _a.map(v => k + '=' + v).join('&'); })
         .join('&');
 };
-exports.orderUrlSearch = orderUrlSearch;
-const simpleHashString = (str) => (0, crypto_1.createHash)('sha256')
+export const simpleHashString = (str) => createHash('sha256')
     .update(str)
     .digest()
     .toString('base64')
@@ -78,32 +64,29 @@ const simpleHashString = (str) => (0, crypto_1.createHash)('sha256')
     .replace(/\+/g, '-')
     .replace(/\//g, '_')
     .replace(/=/g, '');
-exports.simpleHashString = simpleHashString;
-exports.hasOwnProperty = Object.prototype.hasOwnProperty;
+export const hasOwnProperty = Object.prototype.hasOwnProperty;
 /**
  * Merge values from source to target only if key not exists in target
  * Note that using this function against incompatible type or null | undefined
  * may lead to typescript parser errors.
  */
-const weakAssign = (target, source) => {
+export const weakAssign = (target, source) => {
     if (!target)
         return Object.assign({}, source);
     if (!source)
         return target;
     for (const key in source) {
-        if (exports.hasOwnProperty.call(source, key) &&
-            !exports.hasOwnProperty.call(target, key)) {
+        if (hasOwnProperty.call(source, key) &&
+            !hasOwnProperty.call(target, key)) {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             Reflect.set(target, key, source[key]);
         }
     }
     return target;
 };
-exports.weakAssign = weakAssign;
 /**
  * Test if the given url is http url
  * @param url
  */
-const isUrlHttp = (url) => url.startsWith('http://') || url.startsWith('https://');
-exports.isUrlHttp = isUrlHttp;
+export const isUrlHttp = (url) => url.startsWith('http://') || url.startsWith('https://');
 //# sourceMappingURL=util.js.map

package/lib/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;AAAA,mCAAkC;AAGlC,MAAM,aAAa,GAAG,8CAA8C,CAAC;AAE9D,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CACjD,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;AAD7B,QAAA,KAAK,SACwB;AAEnC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAChD,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;AAD5B,QAAA,UAAU,cACkB;AAElC,MAAM,SAAS,GAAG,CAAC,GAAY,EAAuB,EAAE,CAAC,GAAG;IACjE,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;IAC1B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAFxD,QAAA,SAAS,aAE+C;AAE9D,MAAM,UAAU,GAAG,CAAC,KAA0B,EAAE,MAAgB,EACrC,EAAE;IAClC,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;QACxB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;KACf;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC3C,CAAC,CAAC;AAPW,QAAA,UAAU,cAOrB;AAEK,MAAM,QAAQ,GAAG,CAAC,IAAkB,EAAE,QAA0B,EAAU,EAAE;IACjF,IAAI,WAAmB,CAAC;IACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;QACzB,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;KACjD;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACnC,uCAAuC;QACvC,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;aACrE,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;KACjC;SAAM,IAAI,IAAI,YAAY,WAAW,EAAE;QACtC,uCAAuC;QACvC,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;KAC9D;SAAM;QACL,WAAW,GAAG,IAAI,CAAC;KACpB;IACD,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAfW,QAAA,QAAQ,YAenB;AAEF,8DAA8D;AACvD,MAAM,qBAAqB,GAAG,CAAC,IAAY,EAAO,EAAE;IACzD,iGAAiG;IACjG,MAAM,GAAG,GAAQ,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,OAAO,EAAE;QACxC,OAAO,GAAG,CAAC,OAAO,CAAC;KACpB;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAPW,QAAA,qBAAqB,yBAOhC;AAEK,MAAM,cAAc,GAAG,CAAC,MAAc,EAAU,EAAE;IACvD,MAAM,KAAK,GAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;SACnE,KAAK,CAAC,GAAG,CAAC,CAAC;IACd,MAAM,UAAU,GAAa,EAAE,EAC7B,SAAS,GAA6B,EAAE,CAAC;IAC3C,IAAI,WAAqB,EAAE,SAAiB,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACrC,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;YAC1D,SAAS,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;SAClD;aAAM;YACL,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3B,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;SAChD;KACF;IACD,OAAO,GAAG,GAAG,UAAU;SACpB,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,CAAC,EAAE,WAAC,OAAA,MAAA,SAAS,CAAC,CAAC,CAAC,0CAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA,EAAA,CAAC;SACvD,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC,CAAC;AAnBW,QAAA,cAAc,kBAmBzB;AAEK,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAU,EAAE,CACtD,IAAA,mBAAU,EAAC,QAAQ,CAAC;KACjB,MAAM,CAAC,GAAG,CAAC;KACX,MAAM,EAAE;KACR,QAAQ,CAAC,QAAQ,CAAC;IACnB,qBAAqB;KACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;KACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;KACnB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AARV,QAAA,gBAAgB,oBAQN;AAEV,QAAA,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC;AAE9D;;;;GAIG;AACI,MAAM,UAAU,GAAG,CAAO,MAAS,EAAE,MAAS,EAAS,EAAE;IAC9D,IAAI,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAU,CAAC;IACvD,IAAI,CAAC,MAAM;QAAE,OAAO,MAAe,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE;QACxB,IAAI,sBAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC;YAClC,CAAC,sBAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;YACnC,8DAA8D;YAC9D,OAAO,CAAC,GAAG,CAAC,MAAa,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;SAC9C;KACF;IACD,OAAO,MAAe,CAAC;AACzB,CAAC,CAAC;AAXW,QAAA,UAAU,cAWrB;AAEF;;;GAGG;AACI,MAAM,SAAS,GAAG,CAAC,GAAW,EAAW,EAAE,CAChD,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AAD7C,QAAA,SAAS,aACoC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,aAAa,CAAC;AAGvC,MAAM,aAAa,GAAG,8CAA8C,CAAC;AAErE,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CACjD,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;AAE1C,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAChD,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;AAEzC,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAY,EAAuB,EAAE,CAAC,GAAG;IACjE,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;IAC1B,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC,CAAC;AAErE,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,KAA0B,EAAE,MAAgB,EACrC,EAAE;IAClC,MAAM,GAAG,GAAoC,EAAE,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC3C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAkB,EAAE,QAA0B,EAAU,EAAE;IACjF,IAAI,WAAmB,CAAC;IACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;IAClD,CAAC;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,uCAAuC;QACvC,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC;aACrE,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;IAClC,CAAC;SAAM,IAAI,IAAI,YAAY,WAAW,EAAE,CAAC;QACvC,uCAAuC;QACvC,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,MAAM,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAI,IAAY,EAAc,EAAE;IACnE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QAC7B,OAAO,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC;IAC5B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,MAAc,EAAU,EAAE;IACvD,MAAM,KAAK,GAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;SACnE,KAAK,CAAC,GAAG,CAAC,CAAC;IACd,MAAM,UAAU,GAAa,EAAE,EAC7B,SAAS,GAA6B,EAAE,CAAC;IAC3C,IAAI,WAAqB,EAAE,SAAiB,CAAC;IAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,SAAS,CAAC,SAAS,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,SAAS,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC3B,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,GAAG,UAAU;SACpB,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,CAAC,EAAE,WAAC,OAAA,MAAA,SAAS,CAAC,CAAC,CAAC,0CAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAA,EAAA,CAAC;SACvD,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAW,EAAU,EAAE,CACtD,UAAU,CAAC,QAAQ,CAAC;KACjB,MAAM,CAAC,GAAG,CAAC;KACX,MAAM,EAAE;KACR,QAAQ,CAAC,QAAQ,CAAC;IACnB,qBAAqB;KACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;KACnB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;KACnB,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAEvB,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC;AAE9D;;;;GAIG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,CAAO,MAAS,EAAE,MAAS,EAAS,EAAE;IAC9D,IAAI,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAU,CAAC;IACvD,IAAI,CAAC,MAAM;QAAE,OAAO,MAAe,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC;YAClC,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACpC,8DAA8D;YAC9D,OAAO,CAAC,GAAG,CAAC,MAAa,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,MAAe,CAAC;AACzB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAW,EAAE,CAChD,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC"}

package/package.json

@@ -1,45 +1,43 @@
 {
   "name": "website-scrap-engine",
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Configurable website scraper in typescript",
   "main": "lib",
   "types": "lib",
+  "type": "module",
   "engines": {
-    "node": ">=12.16.0"
+    "node": ">=18.17.0"
   },
   "scripts": {
-    "clean": "rimraf lib/*",
+    "clean": "node -e \"require('fs').rmSync('lib',{force:true,recursive:true})\"",
     "tsc": "tsc",
     "lint": "eslint --fix src test",
-    "test": "npm run lint && jest",
+    "test": "npm run lint && node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "build": "npm run lint && npm run tsc && npm run copy",
     "copy": "node copy-src.js",
     "prepack": "npm run clean && npm run build",
     "postshrinkwrap": "node package-lock-resolved.js"
   },
-  "jest": {
-    "preset": "ts-jest"
-  },
   "dependencies": {
-    "cheerio": "^1.0.0-rc.12",
-    "css-url-parser": "^1.1.3",
-    "got": "^11.8.6",
+    "cheerio": "^1.0.0",
+    "css-url-parser": "^1.1.4",
+    "got": "^13.0.0",
     "log4js": "^6.9.1",
     "mkdirp": "^3.0.1",
-    "p-queue": "^6.6.2",
-    "srcset": "^4.0.0",
+    "p-queue": "^8.1.0",
+    "srcset": "^5.0.1",
     "urijs": "^1.19.11"
   },
   "devDependencies": {
-    "@types/jest": "^28.1.1",
-    "@types/node": "^22.1.0",
+    "@jest/globals": "^29.7.0",
+    "@types/node": "^22.12.0",
     "@types/urijs": "^1.19.25",
-    "@typescript-eslint/eslint-plugin": "^5.62.0",
-    "@typescript-eslint/parser": "^5.62.0",
-    "eslint": "^8.57.0",
-    "jest": "^28.1.3",
-    "ts-jest": "^28.0.8",
-    "typescript": "^5.0.4"
+    "@typescript-eslint/eslint-plugin": "^8.22.0",
+    "@typescript-eslint/parser": "^8.22.0",
+    "eslint": "^9.19.0",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.7.3"
   },
   "files": [
     ".editorconfig",

package/src/downloader/adjust-concurrency.ts

@@ -1,5 +1,5 @@
-import {adjustConcurrency as logger} from '../logger/logger';
-import type {DownloaderWithMeta} from './types';
+import {adjustConcurrency as logger} from '../logger/logger.js';
+import type {DownloaderWithMeta} from './types.js';
 
 export function adjust(downloader: DownloaderWithMeta): void {
   const {meta} = downloader;

package/src/downloader/index.ts

@@ -1,8 +1,8 @@
-export {adjust} from './adjust-concurrency';
-export {AbstractDownloader} from './main';
-export {MultiThreadDownloader} from './multi';
-export {PipelineExecutorImpl} from './pipeline-executor-impl';
-export {SingleThreadDownloader} from './single';
-export * as types from './types';
-export {WorkerPool} from './worker-pool';
-export * as workerType from './worker-type';
+export {adjust} from './adjust-concurrency.js';
+export {AbstractDownloader} from './main.js';
+export {MultiThreadDownloader} from './multi.js';
+export {PipelineExecutorImpl} from './pipeline-executor-impl.js';
+export {SingleThreadDownloader} from './single.js';
+export * as types from './types.js';
+export {WorkerPool} from './worker-pool.js';
+export * as workerType from './worker-type.js';

package/src/downloader/main.ts

@@ -1,26 +1,23 @@
 import PQueue from 'p-queue';
 import type {HTTPError} from 'got';
 import URI from 'urijs';
-import {
-  DownloadOptions,
-  mergeOverrideOptions,
-  StaticDownloadOptions
-} from '../options';
-import {
-  normalizeResource,
-  RawResource,
-  Resource,
-  ResourceType
-} from '../resource';
-import {error, notFound, skip} from '../logger/logger';
-import {importDefaultFromPath} from '../util';
-import type {DownloaderStats, DownloaderWithMeta} from './types';
-import {PipelineExecutorImpl} from './pipeline-executor-impl';
+import type {DownloadOptions, StaticDownloadOptions} from '../options.js';
+import {mergeOverrideOptions} from '../options.js';
+import type {RawResource, Resource} from '../resource.js';
+import {normalizeResource, ResourceType} from '../resource.js';
+import {error, notFound, skip} from '../logger/logger.js';
+import {importDefaultFromPath} from '../util.js';
+import type {DownloaderStats, DownloaderWithMeta} from './types.js';
+import {PipelineExecutorImpl} from './pipeline-executor-impl.js';
 
 export abstract class AbstractDownloader implements DownloaderWithMeta {
   readonly queue: PQueue;
-  readonly pipeline: PipelineExecutorImpl;
-  readonly options: DownloadOptions;
+  readonly _asyncOptions: Promise<DownloadOptions>;
+  readonly _overrideOptions?: Partial<StaticDownloadOptions> & { pathToWorker?: string };
+  _options?: DownloadOptions;
+  _isInit: boolean;
+  _pipeline?: PipelineExecutorImpl;
+  _initOptions: Promise<void>;
   readonly downloadedUrl: Set<string> = new Set<string>();
   readonly queuedUrl: Set<string> = new Set<string>();
   readonly meta: DownloaderStats = {
@@ -33,10 +30,31 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
 
   protected constructor(public pathToOptions: string,
     overrideOptions?: Partial<StaticDownloadOptions> & { pathToWorker?: string }) {
-    this.options = mergeOverrideOptions(importDefaultFromPath(pathToOptions), overrideOptions);
-    this.queue = new PQueue({concurrency: this.options.concurrency});
-    this.pipeline = new PipelineExecutorImpl(this.options, this.options.req, this.options);
-    this.options.configureLogger(this.options.localRoot, this.options.logSubDir || '');
+    this._asyncOptions = importDefaultFromPath(pathToOptions);
+    this._overrideOptions = overrideOptions;
+    this.queue = new PQueue();
+    this._isInit = false;
+    this._initOptions = this._asyncOptions.then(options => {
+      options = mergeOverrideOptions(options, this._overrideOptions);
+      this._options = options;
+      this._pipeline = new PipelineExecutorImpl(options, options.req, options);
+      options.configureLogger(options.localRoot, options.logSubDir || '');
+      this._isInit = true;
+    });
+  }
+
+  get options(): DownloadOptions {
+    if (this._options) {
+      return this._options;
+    }
+    throw new TypeError('AbstractDownloader: not initialized');
+  }
+
+  get pipeline(): PipelineExecutorImpl {
+    if (this._pipeline) {
+      return this._pipeline;
+    }
+    throw new TypeError('AbstractDownloader: not initialized');
   }
 
   get concurrency(): number {
@@ -56,20 +74,22 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
   }
 
   async addInitialResource(urlArr: string[]): Promise<void> {
-    await this.pipeline.init(this.pipeline, this);
+    await this._initOptions;
+    const pipeline = this.pipeline;
+    await pipeline.init(pipeline, this);
     // noinspection DuplicatedCode
     for (let i = 0, l = urlArr.length; i < l; i++) {
       let url: string | void = urlArr[i];
-      url = await this.pipeline.linkRedirect(url, null, null);
+      url = await pipeline.linkRedirect(url, null, null);
       if (!url) continue;
-      const type: ResourceType | void = await this.pipeline.detectResourceType(
+      const type: ResourceType | void = await pipeline.detectResourceType(
         url, ResourceType.Html, null, null);
       if (!type) continue;
-      let r: Resource | void = await this.pipeline.createResource(
+      let r: Resource | void = await pipeline.createResource(
         type, 0, url, url,
         undefined, undefined, undefined, type);
       if (!r) continue;
-      r = await this.pipeline.processBeforeDownload(r, null, null);
+      r = await pipeline.processBeforeDownload(r, null, null);
       if (!r) continue;
       if (!r.shouldBeDiscardedFromDownload) {
         this.addProcessedResource(r);
@@ -138,7 +158,9 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
         () => this.options.adjustConcurrencyFunc?.(this),
         this.options.adjustConcurrencyPeriod || 60000);
     }
-    this.queue.start();
+    this._initOptions.then(() => {
+      this.queue.start();
+    });
   }
 
   stop(): void {
@@ -155,7 +177,7 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
   async dispose(): Promise<void> {
     this.stop();
     this.queue.clear();
-    await this.pipeline.dispose(this.pipeline, this);
+    await this.pipeline?.dispose(this.pipeline, this);
   }
 
 }

package/src/downloader/multi.ts

@@ -1,11 +1,12 @@
-import path from 'path';
-import {WorkerPool, WorkerFactory} from './worker-pool';
-import type {RawResource, Resource} from '../resource';
-import type {DownloadWorkerMessage} from './types';
-import type {StaticDownloadOptions} from '../options';
-import type {DownloadResource} from '../life-cycle/types';
-import {skip} from '../logger/logger';
-import {AbstractDownloader} from './main';
+import path from 'node:path';
+import type {WorkerFactory} from './worker-pool.js';
+import {WorkerPool} from './worker-pool.js';
+import type {RawResource, Resource} from '../resource.js';
+import type {DownloadWorkerMessage} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {DownloadResource} from '../life-cycle/types.js';
+import {skip} from '../logger/logger.js';
+import {AbstractDownloader} from './main.js';
 
 export interface MultiThreadDownloaderOptions extends StaticDownloadOptions {
   pathToWorker?: string;
@@ -46,14 +47,14 @@ export class MultiThreadDownloader extends AbstractDownloader {
     if (this.options.initialUrl) {
       this.init = this.addInitialResource(this.options.initialUrl);
     } else {
-      this.init = this.pipeline.init(this.pipeline, this);
+      this.init = this._initOptions.then(() => this.pipeline.init(this.pipeline, this));
     }
   }
 
   async downloadAndProcessResource(res: Resource): Promise<boolean | void> {
     let r: DownloadResource | void;
     try {
-      r = await this.pipeline.download(res);
+      r = await this.pipeline!.download(res);
       if (!r) {
         skip.debug('discarded after download', res.url, res.rawUrl, res.refUrl);
         return;

package/src/downloader/pipeline-executor-impl.ts

@@ -1,21 +1,21 @@
-import type {StaticDownloadOptions} from '../options';
+import type {StaticDownloadOptions} from '../options.js';
 import type {
   CreateResourceArgument,
   Resource,
   ResourceEncoding,
   ResourceType
-} from '../resource';
+} from '../resource.js';
 import type {
   DownloadResource,
   ProcessingLifeCycle,
   RequestOptions,
   SubmitResourceFunc
-} from '../life-cycle/types';
+} from '../life-cycle/types.js';
 // noinspection ES6PreferShortImport
-import type {PipelineExecutor} from '../life-cycle/pipeline-executor';
-import type {Cheerio} from '../types';
-import type {DownloaderWithMeta} from './types';
-import type {WorkerInfo} from './worker-pool';
+import type {PipelineExecutor} from '../life-cycle/pipeline-executor.js';
+import type {Cheerio} from '../types.js';
+import type {DownloaderWithMeta} from './types.js';
+import type {WorkerInfo} from './worker-pool.js';
 
 /**
  * Pipeline executor

package/src/downloader/single.ts

@@ -1,8 +1,11 @@
-import {AbstractDownloader} from './main';
-import type {Resource} from '../resource';
-import type {StaticDownloadOptions} from '../options';
-import {skip} from '../logger/logger';
-import type {DownloadResource, SubmitResourceFunc} from '../life-cycle/types';
+import {AbstractDownloader} from './main.js';
+import type {Resource} from '../resource.js';
+import type {StaticDownloadOptions} from '../options.js';
+import {skip} from '../logger/logger.js';
+import type {
+  DownloadResource,
+  SubmitResourceFunc
+} from '../life-cycle/types.js';
 
 export class SingleThreadDownloader extends AbstractDownloader {
   readonly init: Promise<void>;
@@ -13,7 +16,7 @@ export class SingleThreadDownloader extends AbstractDownloader {
     if (this.options.initialUrl) {
       this.init = this.addInitialResource(this.options.initialUrl);
     } else {
-      this.init = this.pipeline.init(this.pipeline, this);
+      this.init = this._initOptions.then(() => this.pipeline.init(this.pipeline, this));
     }
   }
 

package/src/downloader/types.ts

@@ -1,6 +1,6 @@
-import type {MessagePort} from 'worker_threads';
-import type {DownloadOptions} from '../options';
-import type {RawResource} from '../resource';
+import type {MessagePort} from 'node:worker_threads';
+import type {DownloadOptions} from '../options.js';
+import type {RawResource} from '../resource.js';
 
 export interface DownloaderStats {
   firstPeriodCount: number;

package/src/downloader/worker-pool.ts

@@ -1,14 +1,14 @@
-import type {MessagePort, WorkerOptions} from 'worker_threads';
-import type {URL} from 'url';
-import {Worker} from 'worker_threads';
-import * as logger from '../logger/logger';
-import type {LogWorkerMessage} from './worker-type';
-import {
+import type {MessagePort, WorkerOptions} from 'node:worker_threads';
+import {Worker} from 'node:worker_threads';
+import type {URL} from 'node:url';
+import * as logger from '../logger/logger.js';
+import type {LogWorkerMessage} from './worker-type.js';
+import type {
   PendingPromise,
   PendingPromiseWithBody,
-  WorkerMessage,
-  WorkerMessageType
-} from './types';
+  WorkerMessage
+} from './types.js';
+import {WorkerMessageType} from './types.js';
 
 export interface WorkerInfo {
   readonly id: number;

package/src/downloader/worker-type.ts

@@ -1,6 +1,6 @@
-import type {logLevels} from '../logger/logger-worker';
-import type * as logger from '../logger/logger';
-import {WorkerMessage, WorkerMessageType} from './types';
+import type {logLevels} from '../logger/logger-worker.js';
+import type * as logger from '../logger/logger.js';
+import type {WorkerMessage, WorkerMessageType} from './types.js';
 
 export interface WorkerLog<T = unknown> {
   logger: keyof typeof logger;