website-scrap-engine 0.8.6 → 0.9.0

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "website-scrap-engine",
-  "version": "0.8.6",
+  "version": "0.9.0",
   "description": "Configurable website scraper in typescript",
-  "main": "lib",
-  "types": "lib",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
   "type": "module",
   "engines": {
     "node": ">=18.17.0"
@@ -17,13 +17,12 @@
     "copy": "node copy-src.js",
     "prepack": "npm run clean && npm run build",
     "postshrinkwrap": "node package-lock-resolved.js",
-    "postinstall": "npm run postshrinkwrap && node -e \"require('fs').copyFileSync('fake-undici.d.ts', 'node_modules/cheerio/node_modules/undici/index.d.ts')\""
+    "postinstall": "node -e \"var f='fake-undici.d.ts',t='node_modules/cheerio/node_modules/undici/index.d.ts';try{require('fs').copyFileSync(f,t)}catch(e){}\""
   },
   "dependencies": {
     "cheerio": "^1.2.0",
     "css-url-parser": "^1.1.4",
     "got": "^13.0.0",
-    "log4js": "^6.9.1",
     "mkdirp": "^3.0.1",
     "p-queue": "^8.1.1",
     "srcset": "^5.0.3",
@@ -33,13 +32,16 @@
     "@jest/globals": "^30.1.1",
     "@types/node": "^25.5.0",
     "@types/urijs": "^1.19.26",
-    "@typescript-eslint/eslint-plugin": "^8.57.1",
-    "@typescript-eslint/parser": "^8.57.1",
+    "@typescript-eslint/eslint-plugin": "^8.58.0",
+    "@typescript-eslint/parser": "^8.58.0",
     "eslint": "^9.39.2",
     "jest": "^30.3.0",
-    "ts-jest": "^29.4.6",
+    "ts-jest": "^29.4.9",
     "typescript": "^5.9.3"
   },
+  "optionalDependencies": {
+    "log4js": "^6.9.1"
+  },
   "overrides": {
     "undici": "npm:@favware/skip-dependency@1.2.2"
   },

package/src/downloader/main.ts

@@ -1,14 +1,16 @@
 import PQueue from 'p-queue';
-import type {HTTPError} from 'got';
 import URI from 'urijs';
 import type {DownloadOptions, StaticDownloadOptions} from '../options.js';
 import {mergeOverrideOptions} from '../options.js';
 import type {RawResource, Resource} from '../resource.js';
 import {normalizeResource, ResourceType} from '../resource.js';
-import {error, notFound, skip} from '../logger/logger.js';
+import {skip} from '../logger/logger.js';
+import {setLogger} from '../logger/logger.js';
+import {createDefaultLogger} from '../logger/default-logger.js';
 import {importDefaultFromPath} from '../util.js';
 import type {DownloaderStats, DownloaderWithMeta} from './types.js';
 import {PipelineExecutorImpl} from './pipeline-executor-impl.js';
+import type {InitSubmitFunc} from '../life-cycle/types.js';
 
 export abstract class AbstractDownloader implements DownloaderWithMeta {
   readonly queue: PQueue;
@@ -41,7 +43,7 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
       // https://github.com/website-local/website-scrap-engine/issues/1113
       this.queue.concurrency = options.concurrency;
       this._pipeline = new PipelineExecutorImpl(options, options.req, options);
-      options.configureLogger(options.localRoot, options.logSubDir || '');
+      setLogger((options.createLogger ?? createDefaultLogger)(options));
       return this._internalInit(options).then(() => {
         this._isInit = true;
       });
@@ -89,9 +91,12 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
       await this._initOptions;
     }
     const pipeline = this.pipeline;
-    await pipeline.init(pipeline, this);
+    const submit: InitSubmitFunc = (url: string) => {
+      urlArr.push(url);
+    };
+    await pipeline.init(pipeline, this, submit);
     // noinspection DuplicatedCode
-    for (let i = 0, l = urlArr.length; i < l; i++) {
+    for (let i = 0; i < urlArr.length; i++) {
       let url: string | void = urlArr[i];
       url = await pipeline.linkRedirect(url, null, null);
       if (!url) continue;
@@ -114,6 +119,7 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
     // noinspection DuplicatedCode
     if (res.depth > this.options.maxDepth) {
       skip.info('skipped max depth', res.url, res.refUrl, res.depth);
+      this.pipeline.notifyStatusChange(res, 'dispose');
       return false;
     }
     let url: string;
@@ -146,15 +152,10 @@ export abstract class AbstractDownloader implements DownloaderWithMeta {
   }
 
   handleError(err: Error | unknown | null, cause: string, resource: RawResource): void {
-    // force cast in case of typescript 4.4
-    if (err && (err as {name?: string}).name === 'HTTPError' &&
-      (err as HTTPError)?.response?.statusCode === 404) {
-      notFound.error(resource.url, resource.downloadLink, resource.refUrl);
-    } else if (err) {
-      error.error(cause, resource.url, resource.downloadLink, resource.refUrl, err);
-    } else {
-      error.error(cause, resource.url, resource.downloadLink, resource.refUrl);
-    }
+    resource.meta = resource.meta || {};
+    resource.meta['error'] = err;
+    resource.meta['errorCause'] = cause;
+    this.pipeline.notifyStatusChange(resource, 'error');
   }
 
 

package/src/downloader/multi.ts

@@ -5,7 +5,6 @@ import type {RawResource, Resource} from '../resource.js';
 import type {DownloadWorkerMessage} from './types.js';
 import type {DownloadOptions, StaticDownloadOptions} from '../options.js';
 import type {DownloadResource} from '../life-cycle/types.js';
-import {skip} from '../logger/logger.js';
 import {AbstractDownloader} from './main.js';
 
 export interface MultiThreadDownloaderOptions extends StaticDownloadOptions {
@@ -52,7 +51,7 @@ export class MultiThreadDownloader extends AbstractDownloader {
     if (this.options.initialUrl) {
       return this.addInitialResource(this.options.initialUrl);
     } else {
-      return this._initOptions.then(() => this.pipeline.init(this.pipeline, this));
+      return this.addInitialResource([]);
     }
   }
 
@@ -68,7 +67,7 @@ export class MultiThreadDownloader extends AbstractDownloader {
     try {
       r = await this.pipeline!.download(res);
       if (!r) {
-        skip.debug('discarded after download', res.url, res.rawUrl, res.refUrl);
+        await this.pipeline.notifyStatusChange(res, 'download');
         return;
       }
     } catch (e) {
@@ -94,8 +93,7 @@ export class MultiThreadDownloader extends AbstractDownloader {
     }
     this.downloadedUrl.add(res.url);
     if (!msg) {
-      skip.info('discarded in post-processing',
-        res.url, res.rawUrl, res.refUrl);
+      await this.pipeline.notifyStatusChange(res, 'processAfterDownload');
       return;
     }
     if (msg.error) {

package/src/downloader/pipeline-executor-impl.ts

@@ -1,14 +1,22 @@
+import path from 'node:path';
+import {existsSync, statSync} from 'node:fs';
+import type {Stats} from 'node:fs';
 import type {StaticDownloadOptions} from '../options.js';
 import type {
   CreateResourceArgument,
+  RawResource,
   Resource,
   ResourceEncoding,
   ResourceType
 } from '../resource.js';
 import type {
   DownloadResource,
+  ExistingResourceAction,
+  ExistingResourceStage,
+  InitSubmitFunc,
   ProcessingLifeCycle,
   RequestOptions,
+  ResourceStatus,
   SubmitResourceFunc
 } from '../life-cycle/types.js';
 // noinspection ES6PreferShortImport
@@ -28,11 +36,12 @@ export class PipelineExecutorImpl implements PipelineExecutor {
 
   async init(
     pipeline: PipelineExecutor,
-    downloader?: DownloaderWithMeta
+    downloader?: DownloaderWithMeta,
+    submit?: InitSubmitFunc
   ): Promise<void> {
     if (!this.lifeCycle.init) return;
     for (const init of this.lifeCycle.init) {
-      await init(pipeline, downloader);
+      await init(pipeline, downloader, submit);
     }
   }
 
@@ -156,6 +165,22 @@ export class PipelineExecutorImpl implements PipelineExecutor {
     if (!options) {
       options = this.options;
     }
+    if (this.lifeCycle.existingResource) {
+      const action = this._checkExistingResource(res, 'download');
+      if (action === 'skip') {
+        res.shouldBeDiscardedFromDownload = true;
+        return undefined;
+      }
+      if (action === 'ifModifiedSince') {
+        const mtime = this._getExistingFileMtime(res);
+        if (mtime) {
+          requestOptions = Object.assign({}, requestOptions);
+          requestOptions.headers = Object.assign({}, requestOptions.headers, {
+            'if-modified-since': mtime
+          });
+        }
+      }
+    }
     let downloadedResource: DownloadResource | Resource | void = res;
     for (const download of this.lifeCycle.download) {
       if ((downloadedResource = await download(
@@ -204,6 +229,29 @@ export class PipelineExecutorImpl implements PipelineExecutor {
     if (!options) {
       options = this.options;
     }
+    if (this.lifeCycle.existingResource) {
+      const action = this._checkExistingResource(res, 'saveToDisk');
+      if (action === 'skip' || action === 'skipSave') {
+        return undefined;
+      }
+      if (action === 'ifModifiedSince') {
+        const remoteLastMod = res.meta?.headers?.['last-modified'];
+        if (remoteLastMod) {
+          const localPath = path.join(
+            res.localRoot ?? this.options.localRoot,
+            decodeURI(res.savePath)
+          );
+          try {
+            const localMtime = statSync(localPath).mtime;
+            if (new Date(remoteLastMod as string) <= localMtime) {
+              return undefined;
+            }
+          } catch {
+            // file removed between check and stat, proceed with save
+          }
+        }
+      }
+    }
     let downloadedResource: DownloadResource | void = res;
     for (const saveToDisk of this.lifeCycle.saveToDisk) {
       if ((downloadedResource = await saveToDisk(
@@ -229,4 +277,52 @@ export class PipelineExecutorImpl implements PipelineExecutor {
     }
   }
 
+  async notifyStatusChange(
+    res: Resource | RawResource,
+    status: ResourceStatus
+  ): Promise<void> {
+    if (!this.lifeCycle.statusChange?.length) return;
+    for (const listener of this.lifeCycle.statusChange) {
+      try {
+        const r = listener(res, status, this.options, this);
+        if (r) await r;
+      } catch {
+        // swallow
+      }
+    }
+  }
+
+  private _checkExistingResource(
+    res: Resource, stage: ExistingResourceStage
+  ): ExistingResourceAction | void {
+    const localPath = path.join(
+      res.localRoot ?? this.options.localRoot,
+      decodeURI(res.savePath)
+    );
+    if (!existsSync(localPath)) return undefined;
+    let stat: Stats;
+    try {
+      stat = statSync(localPath);
+    } catch {
+      // TOCTOU: file deleted between existsSync and statSync
+      return undefined;
+    }
+    if (!stat.isFile()) return undefined;
+    return this.lifeCycle.existingResource!({
+      res, stage, localPath, stat, options: this.options
+    });
+  }
+
+  private _getExistingFileMtime(res: Resource): string | undefined {
+    const localPath = path.join(
+      res.localRoot ?? this.options.localRoot,
+      decodeURI(res.savePath)
+    );
+    try {
+      return statSync(localPath).mtime.toUTCString();
+    } catch {
+      return undefined;
+    }
+  }
+
 }

package/src/downloader/single.ts

@@ -1,7 +1,6 @@
 import {AbstractDownloader} from './main.js';
 import type {Resource} from '../resource.js';
 import type {DownloadOptions, StaticDownloadOptions} from '../options.js';
-import {skip} from '../logger/logger.js';
 import type {
   DownloadResource,
   SubmitResourceFunc
@@ -20,7 +19,7 @@ export class SingleThreadDownloader extends AbstractDownloader {
     if (options.initialUrl) {
       return this.addInitialResource(options.initialUrl);
     } else {
-      return this.pipeline.init(this.pipeline, this);
+      return this.addInitialResource([]);
     }
   }
 
@@ -29,7 +28,7 @@ export class SingleThreadDownloader extends AbstractDownloader {
     try {
       r = await this.pipeline.download(res);
       if (!r) {
-        skip.debug('discarded after download', res.url, res.rawUrl, res.refUrl);
+        await this.pipeline.notifyStatusChange(res, 'download');
         return;
       }
     } catch (e) {
@@ -51,9 +50,9 @@ export class SingleThreadDownloader extends AbstractDownloader {
       const processedResource: DownloadResource | void =
         await this.pipeline.processAfterDownload(r, submit);
       if (!processedResource) {
-        skip.warn('skipped downloaded resource', r.url, r.refUrl);
+        await this.pipeline.notifyStatusChange(r, 'processAfterDownload');
       } else if (await this.pipeline.saveToDisk(processedResource)) {
-        skip.warn('downloaded resource not saved', r.url, r.refUrl);
+        await this.pipeline.notifyStatusChange(r, 'saveToDisk');
       }
       if (processedResource && processedResource.redirectedUrl &&
         processedResource.redirectedUrl !== processedResource.url) {

package/src/downloader/worker-pool.ts

@@ -1,7 +1,7 @@
 import type {MessagePort, WorkerOptions} from 'node:worker_threads';
 import {Worker} from 'node:worker_threads';
 import type {URL} from 'node:url';
-import * as logger from '../logger/logger.js';
+import {error as errorLogger, getLogger} from '../logger/logger.js';
 import type {LogWorkerMessage} from './worker-type.js';
 import type {
   PendingPromise,
@@ -63,7 +63,7 @@ export class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMessage> {
   }
 
   workerOnError(info: WorkerInfo, err: Error): void {
-    logger.error.error('worker error', info.id, err);
+    errorLogger.error('worker error', info.id, err);
   }
 
   onMessage(info: WorkerInfo, message: WorkerMessage): void {
@@ -76,14 +76,20 @@ export class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMessage> {
 
   takeLog(info: WorkerInfo, message: LogWorkerMessage): void {
     if (!message?.body) {
-      logger.error.warn('Invalid formatted log', info.id);
+      errorLogger.warn('Invalid formatted log', info.id);
       return;
     }
-    const content = message?.body?.content;
+    const level = message.body.level;
+    const logType = message.body.logType;
+    if (!level || !logType) {
+      return;
+    }
+    const log = getLogger();
+    const content = message.body.content;
     if (content?.length) {
-      logger?.[message.body.logger]?.[message.body.level]?.(info.id, ...content);
+      log[level](logType, info.id, ...content);
     } else {
-      logger?.[message.body.logger]?.[message.body.level]?.(info.id);
+      log[level](logType, info.id);
     }
   }
 

package/src/downloader/worker-type.ts

@@ -1,9 +1,9 @@
 import type {logLevels} from '../logger/logger-worker.js';
-import type * as logger from '../logger/logger.js';
+import type {LogType} from '../logger/types.js';
 import type {WorkerMessage, WorkerMessageType} from './types.js';
 
 export interface WorkerLog<T = unknown> {
-  logger: keyof typeof logger;
+  logType: LogType;
   level: typeof logLevels[number];
   content: T[];
 }

package/src/downloader/worker.ts

@@ -7,7 +7,6 @@ import type {
 } from '../life-cycle/types.js';
 import type {RawResource, Resource} from '../resource.js';
 import {normalizeResource, prepareResourceForClone} from '../resource.js';
-import {skip} from '../logger/logger.js';
 import {importDefaultFromPath} from '../util.js';
 import type {DownloadWorkerMessage} from './types.js';
 import {WorkerMessageType} from './types.js';
@@ -29,8 +28,6 @@ const asyncPipeline = asyncOptions.then(options => {
   const pipeline: PipelineExecutor =
     new PipelineExecutorImpl(options, options.req, options);
 
-  options.configureLogger(options.localRoot, options.logSubDir || '');
-
   const init = pipeline.init(pipeline);
   if (init && (init as Promise<void>).then) {
     return init.then(() => pipeline);
@@ -58,11 +55,9 @@ parentPort?.addListener('message', async (msg: WorkerTaskMessage<RawResource>) =
     const processedResource: DownloadResource | void =
       await pipeline.processAfterDownload(downloadResource, submit);
     if (!processedResource) {
-      skip.warn('skipped downloaded resource',
-        downloadResource.url, downloadResource.refUrl);
+      await pipeline.notifyStatusChange(downloadResource, 'processAfterDownload');
     } else if (await pipeline.saveToDisk(processedResource)) {
-      skip.warn('downloaded resource not saved',
-        downloadResource.url, downloadResource.refUrl);
+      await pipeline.notifyStatusChange(downloadResource, 'saveToDisk');
     }
 
     if (processedResource && processedResource.redirectedUrl &&

package/src/index.ts

@@ -1,4 +1,6 @@
 export * as logger from './logger/logger.js';
+export type {Logger, LogType, CategoryLogger} from './logger/types.js';
+export {createDefaultLogger} from './logger/default-logger.js';
 export * as downloader from './downloader/index.js';
 export * as lifeCycle from './life-cycle/index.js';
 export * as io from './io.js';

package/src/life-cycle/adapters.ts

@@ -4,6 +4,7 @@ import {ResourceType} from '../resource.js';
 import type {
   AsyncResult,
   DownloadResource,
+  ExistingResourceFunc,
   LinkRedirectFunc,
   ProcessResourceAfterDownloadFunc,
   ProcessResourceBeforeDownloadFunc,
@@ -132,3 +133,15 @@ export const processHtmlAsync = (fn: AsyncHtmlProcessFunc): ProcessResourceAfter
     return res;
   };
 
+/** Skip download if local file already exists */
+export const skipExisting = (): ExistingResourceFunc =>
+  ({stage}) => stage === 'download' ? 'skip' : 'overwrite';
+
+/** Re-download only if remote is newer (If-Modified-Since) */
+export const preferNewerRemote = (): ExistingResourceFunc =>
+  () => 'ifModifiedSince';
+
+/** Always overwrite (current default behavior, explicit) */
+export const alwaysOverwrite = (): ExistingResourceFunc =>
+  () => 'overwrite';
+

package/src/life-cycle/default-life-cycle.ts

@@ -13,6 +13,7 @@ import {saveResourceToDisk} from './save-resource-to-disk.js';
 import {processRedirectedUrl} from './adapters.js';
 import {downloadStreamingResource} from './download-streaming-resource.js';
 import {readOrCopyLocalResource} from './read-or-copy-local-resource.js';
+import {defaultStatusListener} from './default-status-listener.js';
 
 /**
  * Get a copy of default life cycle
@@ -37,6 +38,7 @@ export const defaultLifeCycle = (): ProcessingLifeCycle => ({
     processSiteMap
   ],
   saveToDisk: [saveHtmlToDisk, saveResourceToDisk],
-  dispose: []
+  dispose: [],
+  statusChange: [defaultStatusListener]
 });
 

package/src/life-cycle/default-status-listener.ts

@@ -0,0 +1,40 @@
+import type {RawResource, Resource} from '../resource.js';
+import type {ResourceStatus} from './types.js';
+import {error, notFound, skip} from '../logger/logger.js';
+
+export const defaultStatusListener = (
+  res: Resource | RawResource,
+  status: ResourceStatus
+): void => {
+  switch (status) {
+  case 'processBeforeDownload':
+    break;
+  case 'createResource':
+    break;
+  case 'download':
+    skip.debug('discarded after download', res.url, res.rawUrl, res.refUrl);
+    break;
+  case 'processAfterDownload':
+    skip.warn('skipped downloaded resource', res.url, res.refUrl);
+    break;
+  case 'saveToDisk':
+    skip.warn('downloaded resource not saved', res.url, res.refUrl);
+    break;
+  case 'error':
+    if (res.meta?.['errorCause']) {
+      const err = res.meta['error'];
+      const cause = res.meta['errorCause'] as string;
+      if (err && (err as {name?: string}).name === 'HTTPError' &&
+        (err as {response?: {statusCode?: number}})?.response?.statusCode === 404) {
+        notFound.error(res.url, res.downloadLink, res.refUrl);
+      } else if (err) {
+        error.error(cause, res.url, res.downloadLink, res.refUrl, err);
+      } else {
+        error.error(cause, res.url, res.downloadLink, res.refUrl);
+      }
+    }
+    break;
+  case 'dispose':
+    break;
+  }
+};

package/src/life-cycle/download-resource.ts

@@ -1,5 +1,5 @@
 import type {BeforeRetryHook, OptionsInit, RequestError, Response} from 'got';
-import got, {TimeoutError} from 'got';
+import got, {HTTPError, TimeoutError} from 'got';
 import type {DownloadResource, RequestOptions} from './types.js';
 import type {Resource} from '../resource.js';
 import {generateSavePath, ResourceType} from '../resource.js';
@@ -117,8 +117,16 @@ export async function requestForResource(
   }
   logger.request.info(res.url, downloadLink, res.refUrl,
     res.encoding, res.type);
-  const response: Response<string | Buffer> | void =
-    await getRetry(downloadLink, reqOptions);
+  let response: Response<string | Buffer> | void;
+  try {
+    response = await getRetry(downloadLink, reqOptions);
+  } catch (e) {
+    if (e instanceof HTTPError &&
+      (e as HTTPError).response.statusCode === 304) {
+      return undefined;
+    }
+    throw e;
+  }
   if (!response) {
     const resource = res as Resource;
     delete resource.downloadStartTimestamp;
@@ -195,7 +203,8 @@ export async function downloadResource(
           }
         }
         if (nonHtml) {
-          logger.error.warn('Detected non-html content type',
+          logger.error.warn('Detected non-html content type for resource typed as',
+            downloadedResource.type,
             downloadedResource.downloadLink, downloadedResource.rawUrl, contentType);
         }
       }

package/src/life-cycle/index.ts

@@ -1,5 +1,6 @@
 export * as adapter from './adapters.js';
 export {defaultLifeCycle} from './default-life-cycle.js';
+export {defaultStatusListener} from './default-status-listener.js';
 export {detectResourceType} from './detect-resource-type.js';
 export {
   beforeRetryHook, getRetry, requestForResource, downloadResource

package/src/life-cycle/pipeline-executor.ts

@@ -1,9 +1,11 @@
-import type {Resource, ResourceEncoding, ResourceType} from '../resource.js';
+import type {RawResource, Resource, ResourceEncoding, ResourceType} from '../resource.js';
 import type {StaticDownloadOptions} from '../options.js';
 import type {
   AsyncResult,
   DownloadResource,
+  InitSubmitFunc,
   RequestOptions,
+  ResourceStatus,
   SubmitResourceFunc
 } from './types.js';
 import type {Cheerio} from '../types.js';
@@ -16,7 +18,8 @@ export interface PipelineExecutor {
    */
   init(
     pipeline: PipelineExecutor,
-    downloader?: DownloaderWithMeta
+    downloader?: DownloaderWithMeta,
+    submit?: InitSubmitFunc
   ): AsyncResult<void>;
 
   /**
@@ -99,4 +102,15 @@ export interface PipelineExecutor {
     workerExitCode?: number
   ): AsyncResult<void>;
 
+  /**
+   * Notify status change listeners.
+   *
+   * All listeners always run (void return does not short-circuit),
+   * and thrown errors are swallowed.
+   */
+  notifyStatusChange(
+    res: Resource | RawResource,
+    status: ResourceStatus
+  ): AsyncResult<void>;
+
 }