website-scrap-engine 0.7.2 → 0.8.1

package/src/life-cycle/process-source-map.ts

@@ -1,7 +1,8 @@
-import type {DownloadResource, SubmitResourceFunc} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {Resource, ResourceEncoding, ResourceType} from '../resource';
-import type {PipelineExecutor} from './pipeline-executor';
+import type {DownloadResource, SubmitResourceFunc} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {Resource, ResourceEncoding} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
 
 // https://developer.mozilla.org/docs/Web/HTTP/Headers/SourceMap
 export const SOURCE_MAP_HEADER = 'SourceMap'.toLowerCase();

package/src/life-cycle/process-svg.ts

@@ -1,12 +1,13 @@
-import type {SourceDefinition} from '../sources';
-import type {DownloadResource, SubmitResourceFunc} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {Resource, ResourceType} from '../resource';
-import {error, skip} from '../logger/logger';
-import type {PipelineExecutor} from './pipeline-executor';
-import {parseHtml} from './adapters';
-import {getResourceBodyFromHtml} from './save-html-to-disk';
-import type {Cheerio, CheerioStatic} from '../types';
+import type {SourceDefinition} from '../sources.js';
+import type {DownloadResource, SubmitResourceFunc} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import {error, skip} from '../logger/logger.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import {parseHtml} from './adapters.js';
+import {getResourceBodyFromHtml} from './save-html-to-disk.js';
+import type {Cheerio, CheerioStatic} from '../types.js';
 
 const svgSelectors: SourceDefinition[] = [
   {selector: '*[xlink\\:href]', attr: 'xlink:href', type: ResourceType.Binary},

package/src/life-cycle/read-or-copy-local-resource.ts

@@ -1,9 +1,11 @@
-import * as path from 'path';
-import {promises, Stats} from 'fs';
-import {Resource, ResourceType} from '../resource';
-import type {DownloadResource, RequestOptions} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {error as errorLogger} from '../logger/logger';
+import * as path from 'node:path';
+import type {Stats} from 'node:fs';
+import {promises} from 'node:fs';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import type {DownloadResource, RequestOptions} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import {error as errorLogger} from '../logger/logger.js';
 
 const FILE_PREFIX = 'file://';
 
@@ -27,7 +29,7 @@ export async function readOrCopyLocalResource(
     return;
   }
   // index.html handling
-  let stats: Stats | void;
+  let stats: Stats | void = void 0;
   if (res.type === ResourceType.Html) {
     stats = await promises.stat(fileSrcPath);
     if (stats.isDirectory()) {

package/src/life-cycle/save-html-to-disk.ts

@@ -1,16 +1,12 @@
-import path from 'path';
+import path from 'node:path';
 import URI from 'urijs';
-import type {DownloadResource} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {
-  ResourceBody,
-  ResourceEncoding,
-  ResourceType,
-  urlOfSavePath
-} from '../resource';
-import {escapePath} from '../util';
-import {writeFile} from '../io';
-import type {PipelineExecutor} from './pipeline-executor';
+import type {DownloadResource} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {ResourceBody, ResourceEncoding} from '../resource.js';
+import {ResourceType, urlOfSavePath} from '../resource.js';
+import {escapePath} from '../util.js';
+import {writeFile} from '../io.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
 
 export function getResourceBodyFromHtml(
   res: DownloadResource & { type: ResourceType.Html },
@@ -46,7 +42,7 @@ export async function saveHtmlToDisk(
   }
   const localRoot: string = res.localRoot ?? options.localRoot;
   // https://github.com/website-local/website-scrap-engine/issues/174
-  let mtime: number | void;
+  let mtime: number | void = void 0;
   if (options.preferRemoteLastModifiedTime && res.meta?.headers?.['last-modified']) {
     mtime = Date.parse(res.meta.headers?.['last-modified']);
   }

package/src/life-cycle/save-resource-to-disk.ts

@@ -1,8 +1,8 @@
-import path from 'path';
-import type {DownloadResource} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {writeFile} from '../io';
-import type {PipelineExecutor} from './pipeline-executor';
+import path from 'node:path';
+import type {DownloadResource} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import {writeFile} from '../io.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
 
 export async function saveResourceToDisk(
   res: DownloadResource,
@@ -10,7 +10,7 @@ export async function saveResourceToDisk(
   pipeline: PipelineExecutor): Promise<DownloadResource | void> {
   const localRoot: string = res.localRoot ?? options.localRoot;
   // https://github.com/website-local/website-scrap-engine/issues/174
-  let mtime: number | void;
+  let mtime: number | void = void 0;
   if (options.preferRemoteLastModifiedTime && res.meta?.headers?.['last-modified']) {
     mtime = Date.parse(res.meta.headers?.['last-modified']);
   }

package/src/life-cycle/types.ts

@@ -1,16 +1,16 @@
-import type {Options as GotOptions} from 'got/dist/source/as-promise';
+import type {OptionsInit as GotOptions} from 'got';
 import type {
   createResource,
   GenerateSavePathFn,
   Resource,
   ResourceBody,
   ResourceType
-} from '../resource';
-import type {StaticDownloadOptions} from '../options';
-import type {PipelineExecutor} from './pipeline-executor';
-import type {Cheerio} from '../types';
-import type {DownloaderWithMeta} from '../downloader/types';
-import type {WorkerInfo} from '../downloader/worker-pool';
+} from '../resource.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import type {Cheerio} from '../types.js';
+import type {DownloaderWithMeta} from '../downloader/types.js';
+import type {WorkerInfo} from '../downloader/worker-pool.js';
 
 export type AsyncResult<T> = T | Promise<T>;
 

package/src/logger/config-logger.ts

@@ -1,8 +1,10 @@
-import {configure, Log4js} from 'log4js';
-import * as path from 'path';
+import type {Log4js} from 'log4js';
+// https://github.com/jestjs/jest/issues/11563
+import log4js from 'log4js';
+import * as path from 'node:path';
 
 export const configureLogger = (localRoot: string, subDir: string): Log4js =>
-  configure({
+  log4js.configure({
     appenders: {
       'retry': {
         type: 'file',

package/src/logger/logger-worker.ts

@@ -1,7 +1,11 @@
-import {getLogger, Logger} from 'log4js';
-import {parentPort} from 'worker_threads';
-import type {LogWorkerMessage, WorkerLog} from '../downloader/worker-type';
-import {WorkerMessageType} from '../downloader/types';
+import type {Logger} from 'log4js';
+// https://github.com/jestjs/jest/issues/11563
+import log4js from 'log4js';
+import {parentPort} from 'node:worker_threads';
+import type {LogWorkerMessage, WorkerLog} from '../downloader/worker-type.js';
+import {WorkerMessageType} from '../downloader/types.js';
+
+const getLogger = log4js.getLogger;
 
 export const logLevels = [
   'trace', 'debug', 'info', 'warn', 'error', 'fatal', 'mark'

package/src/logger/logger.ts

@@ -1,9 +1,11 @@
-import {getLogger as getMainLogger, Logger} from 'log4js';
-import {isMainThread} from 'worker_threads';
-import {getWorkerLogger} from './logger-worker';
+import type {Logger} from 'log4js';
+// https://github.com/jestjs/jest/issues/11563
+import log4js from 'log4js';
+import {isMainThread} from 'node:worker_threads';
+import {getWorkerLogger} from './logger-worker.js';
 
 const getLogger: typeof getWorkerLogger =
-  isMainThread ? getMainLogger : getWorkerLogger;
+  isMainThread ? log4js.getLogger : getWorkerLogger;
 
 export const notFound: Logger = getLogger('notFound');
 export const retry: Logger = getLogger('retry');

package/src/options.ts

@@ -1,22 +1,18 @@
-import got from 'got';
-import type {
-  RetryFunction,
-  RetryObject,
-  TimeoutError
-} from 'got/dist/source/as-promise/types';
-import type {RequestError} from 'got/dist/source/core';
-import {createResource, ResourceEncoding, ResourceType} from './resource';
-import type {ProcessingLifeCycle, RequestOptions} from './life-cycle/types';
+import type {RequestError, RetryFunction, RetryObject, TimeoutError} from 'got';
+import got, {Options} from 'got';
+import type {ResourceEncoding, ResourceType} from './resource.js';
+import {createResource} from './resource.js';
+import type {ProcessingLifeCycle, RequestOptions} from './life-cycle/types.js';
 // noinspection ES6PreferShortImport
-import {beforeRetryHook} from './life-cycle/download-resource';
-import {error} from './logger/logger';
+import {beforeRetryHook} from './life-cycle/download-resource.js';
+import {error} from './logger/logger.js';
 // noinspection ES6PreferShortImport
-import {adjust} from './downloader/adjust-concurrency';
-import {configureLogger} from './logger/config-logger';
-import type {DownloaderWithMeta} from './downloader/types';
-import {weakAssign} from './util';
-import type {SourceDefinition} from './sources';
-import type {CheerioOptionsInterface} from './types';
+import {adjust} from './downloader/adjust-concurrency.js';
+import {configureLogger} from './logger/config-logger.js';
+import type {DownloaderWithMeta} from './downloader/types.js';
+import {weakAssign} from './util.js';
+import type {SourceDefinition} from './sources.js';
+import type {CheerioOptionsInterface} from './types.js';
 
 /**
  * Extra options for custom life cycle
@@ -24,6 +20,7 @@ import type {CheerioOptionsInterface} from './types';
 export interface StaticDownloadMeta
   extends Record<string, string | number | boolean| void> {
   detectIncompleteHtml?: '</html>' | '</body>' | string;
+  warnForNonHtml?: boolean;
 }
 
 /**
@@ -379,8 +376,8 @@ export function mergeOverrideOptions(
     overrideOptions.meta = Object.assign(opt.meta, overrideOptions.meta);
   }
   if (opt.req && overrideOptions.req) {
-    overrideOptions.req =
-      got.mergeOptions(opt.req, overrideOptions.req);
+    const options = got.defaults.options;
+    overrideOptions.req = new Options(opt.req, overrideOptions.req, options);
   }
   return checkDownloadOptions(Object.assign(opt, overrideOptions));
 }

package/src/resource.ts

@@ -1,9 +1,14 @@
 import URI from 'urijs';
-import type {IncomingHttpHeaders} from 'http';
-import * as path from 'path';
-import {escapePath, isUrlHttp, orderUrlSearch, simpleHashString} from './util';
-import type {CheerioStatic} from './types';
-import {error as log} from './logger/logger';
+import type {IncomingHttpHeaders} from 'node:http';
+import * as path from 'node:path';
+import {
+  escapePath,
+  isUrlHttp,
+  orderUrlSearch,
+  simpleHashString
+} from './util.js';
+import type {CheerioStatic} from './types.js';
+import {error as log} from './logger/logger.js';
 
 export enum ResourceType {
   /**

package/src/sources.ts

@@ -1,4 +1,4 @@
-import {ResourceType} from './resource';
+import {ResourceType} from './resource.js';
 
 export interface SourceDefinition {
   selector: string;

package/src/util.ts

@@ -1,5 +1,5 @@
-import {createHash} from 'crypto';
-import type {ResourceBody, ResourceEncoding} from './resource';
+import {createHash} from 'node:crypto';
+import type {ResourceBody, ResourceEncoding} from './resource.js';
 
 const forbiddenChar = /[:*?"<>|&]|%3A|%2A|%3F|%22|%3C|%3E|%7C|%26/ig;
 
@@ -39,14 +39,10 @@ export const toString = (body: ResourceBody, encoding: ResourceEncoding): string
   return stringValue;
 };
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const importDefaultFromPath = (path: string): any => {
-  // eslint-disable-next-line @typescript-eslint/no-var-requires,@typescript-eslint/no-explicit-any
-  const mod: any = require(path);
-  if (mod && mod.__esModule && mod.default) {
-    return mod.default;
-  }
-  return mod;
+export const importDefaultFromPath = <T>(path: string): Promise<T> => {
+  return import(path).then(mod => {
+    return mod.default || mod;
+  });
 };
 
 export const orderUrlSearch = (search: string): string => {

package/tsconfig.json

@@ -1,6 +1,7 @@
 {
   "compilerOptions": {
-    "module": "commonjs",
+    "module": "node16",
+    "moduleResolution": "node16",
     "target": "es2018",
     "sourceMap": true,
     "newLine": "lf",
@@ -8,7 +9,10 @@
     "declaration": true,
     "esModuleInterop": true,
     "removeComments": false,
-    "strict": true
+    "strict": true,
+    "declarationMap": true,
+    "allowJs": true,
+    "verbatimModuleSyntax": true
   },
   "include": [
     "src"