website-scrap-engine 0.7.2 → 0.8.1

package/src/downloader/worker.ts

@@ -1,46 +1,49 @@
-import {parentPort, workerData} from 'worker_threads';
-import {
-  DownloadOptions,
-  mergeOverrideOptions,
-  StaticDownloadOptions
-} from '../options';
-import type {DownloadResource, SubmitResourceFunc} from '../life-cycle/types';
-import {
-  normalizeResource,
-  prepareResourceForClone,
-  RawResource,
-  Resource
-} from '../resource';
-import {skip} from '../logger/logger';
-import {importDefaultFromPath} from '../util';
-import {DownloadWorkerMessage, WorkerMessageType} from './types';
-import {PipelineExecutorImpl} from './pipeline-executor-impl';
+import {parentPort, workerData} from 'node:worker_threads';
+import type {DownloadOptions, StaticDownloadOptions} from '../options.js';
+import {mergeOverrideOptions} from '../options.js';
+import type {
+  DownloadResource,
+  SubmitResourceFunc
+} from '../life-cycle/types.js';
+import type {RawResource, Resource} from '../resource.js';
+import {normalizeResource, prepareResourceForClone} from '../resource.js';
+import {skip} from '../logger/logger.js';
+import {importDefaultFromPath} from '../util.js';
+import type {DownloadWorkerMessage} from './types.js';
+import {WorkerMessageType} from './types.js';
+import {PipelineExecutorImpl} from './pipeline-executor-impl.js';
 // noinspection ES6PreferShortImport
-import type {PipelineExecutor} from '../life-cycle/pipeline-executor';
-import type {WorkerTaskMessage} from './worker-type';
+import type {PipelineExecutor} from '../life-cycle/pipeline-executor.js';
+import type {WorkerTaskMessage} from './worker-type.js';
 
 const {pathToOptions, overrideOptions}: {
   pathToOptions: string,
   overrideOptions?: Partial<StaticDownloadOptions>
 } = workerData;
 
-const options: DownloadOptions =
-  // eslint-disable-next-line @typescript-eslint/no-var-requires
-  mergeOverrideOptions(importDefaultFromPath(pathToOptions), overrideOptions);
+const asyncOptions: Promise<DownloadOptions> = importDefaultFromPath(pathToOptions);
 
-const pipeline: PipelineExecutor =
-  new PipelineExecutorImpl(options, options.req, options);
+const asyncPipeline = asyncOptions.then(options => {
+  options = mergeOverrideOptions(options, overrideOptions);
 
-options.configureLogger(options.localRoot, options.logSubDir || '');
+  const pipeline: PipelineExecutor =
+    new PipelineExecutorImpl(options, options.req, options);
 
-const init = pipeline.init(pipeline);
+  options.configureLogger(options.localRoot, options.logSubDir || '');
+
+  const init = pipeline.init(pipeline);
+  if (init && (init as Promise<void>).then) {
+    return init.then(() => pipeline);
+  }
+  return pipeline;
+});
 
 parentPort?.addListener('message', async (msg: WorkerTaskMessage<RawResource>) => {
   const collectedResource: RawResource[] = [];
   let error: Error | unknown | void;
   let redirectedUrl: string | undefined;
   try {
-    await init;
+    const pipeline = await asyncPipeline;
     const res = msg.body;
     const downloadResource: DownloadResource = normalizeResource(res) as DownloadResource;
     const submit: SubmitResourceFunc = (resources: Resource | Resource[]) => {
@@ -67,8 +70,27 @@ parentPort?.addListener('message', async (msg: WorkerTaskMessage<RawResource>) =
       redirectedUrl = processedResource.redirectedUrl;
     }
   } catch (e) {
-    // TODO: handle if object could not be cloned here
-    error = e;
+    // handle if object could not be cloned here
+    // https://github.com/website-local/website-scrap-engine/issues/340
+    try {
+      // should always be
+      if (typeof structuredClone === 'function') {
+        error = structuredClone(e);
+      } else {
+        // this is the old behavior before this
+        error = e;
+      }
+    } catch {
+      // can not clone, so no need to get the full error here
+      if (e && typeof e === 'object') {
+        const clone: Record<string, string> = {};
+        for (const k in e) {
+          clone[k] = String((e as Record<string, unknown>)[k]);
+        }
+      } else {
+        error = String(e);
+      }
+    }
   } finally {
     const message: DownloadWorkerMessage = {
       taskId: msg.taskId,

package/src/index.ts

@@ -1,8 +1,8 @@
-export * as logger from './logger/logger';
-export * as downloader from './downloader/index';
-export * as lifeCycle from './life-cycle/index';
-export * as io from './io';
-export * as options from './options';
-export * as resource from './resource';
-export {SourceDefinition} from './sources';
-export * as util from './util';
+export * as logger from './logger/logger.js';
+export * as downloader from './downloader/index.js';
+export * as lifeCycle from './life-cycle/index.js';
+export * as io from './io.js';
+export * as options from './options.js';
+export * as resource from './resource.js';
+export type {SourceDefinition} from './sources.js';
+export * as util from './util.js';

package/src/io.ts

@@ -1,9 +1,9 @@
-import type {ObjectEncodingOptions} from 'fs';
-import fs from 'fs';
-import {dirname} from 'path';
+import type {ObjectEncodingOptions} from 'node:fs';
+import fs from 'node:fs';
+import {dirname} from 'node:path';
 import {mkdirp} from 'mkdirp';
-import type {ResourceBody, ResourceEncoding} from './resource';
-import {error as errorLogger, mkdir as mkdirLogger} from './logger/logger';
+import type {ResourceBody, ResourceEncoding} from './resource.js';
+import {error as errorLogger, mkdir as mkdirLogger} from './logger/logger.js';
 
 export const mkdirRetry = async (dir: string, retry = 3): Promise<void> => {
   let error: unknown | void;
@@ -40,7 +40,7 @@ export const writeFile = async (
     await mkdirRetry(dir);
   }
   let fileData: Uint8Array | string;
-  let options: ObjectEncodingOptions | void;
+  let options: ObjectEncodingOptions | void = void 0;
   if (typeof data === 'string') {
     fileData = data;
     options = {encoding};

package/src/life-cycle/adapters.ts

@@ -1,5 +1,6 @@
 import {load} from 'cheerio';
-import {Resource, ResourceEncoding, ResourceType} from '../resource';
+import type {Resource, ResourceEncoding} from '../resource.js';
+import {ResourceType} from '../resource.js';
 import type {
   AsyncResult,
   DownloadResource,
@@ -7,11 +8,11 @@ import type {
   ProcessResourceAfterDownloadFunc,
   ProcessResourceBeforeDownloadFunc,
   SubmitResourceFunc
-} from './types';
-import {toString} from '../util';
-import type {StaticDownloadOptions} from '../options';
-import type {PipelineExecutor} from './pipeline-executor';
-import type {Cheerio, CheerioStatic} from '../types';
+} from './types.js';
+import {toString} from '../util.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import type {Cheerio, CheerioStatic} from '../types.js';
 
 export interface SkipProcessFunc {
   (url: string, element: Cheerio | null, parent: Resource | null): boolean;

package/src/life-cycle/css-url-parser.d.ts

@@ -7,5 +7,5 @@
 declare function parseCssUrls(cssText: string): string[];
 
 declare module 'css-url-parser' {
-  export = parseCssUrls;
+  export default parseCssUrls;
 }

package/src/life-cycle/default-life-cycle.ts

@@ -1,18 +1,18 @@
-import type {ProcessingLifeCycle} from './types';
-import {skipLinks} from './skip-links';
-import {detectResourceType} from './detect-resource-type';
-import {createResource} from '../resource';
-import {downloadResource} from './download-resource';
-import {processHtml} from './process-html';
-import {processHtmlMetaRefresh} from './process-html-meta';
-import {processCss} from './process-css';
-import {processSiteMap} from './process-site-map';
-import {processSvg} from './process-svg';
-import {saveHtmlToDisk} from './save-html-to-disk';
-import {saveResourceToDisk} from './save-resource-to-disk';
-import {processRedirectedUrl} from './adapters';
-import {downloadStreamingResource} from './download-streaming-resource';
-import {readOrCopyLocalResource} from './read-or-copy-local-resource';
+import type {ProcessingLifeCycle} from './types.js';
+import {skipLinks} from './skip-links.js';
+import {detectResourceType} from './detect-resource-type.js';
+import {createResource} from '../resource.js';
+import {downloadResource} from './download-resource.js';
+import {processHtml} from './process-html.js';
+import {processHtmlMetaRefresh} from './process-html-meta.js';
+import {processCss} from './process-css.js';
+import {processSiteMap} from './process-site-map.js';
+import {processSvg} from './process-svg.js';
+import {saveHtmlToDisk} from './save-html-to-disk.js';
+import {saveResourceToDisk} from './save-resource-to-disk.js';
+import {processRedirectedUrl} from './adapters.js';
+import {downloadStreamingResource} from './download-streaming-resource.js';
+import {readOrCopyLocalResource} from './read-or-copy-local-resource.js';
 
 /**
  * Get a copy of default life cycle

package/src/life-cycle/detect-resource-type.ts

@@ -1,5 +1,5 @@
-import {ResourceType} from '../resource';
-import {arrayToMap, isSiteMap} from '../util';
+import {ResourceType} from '../resource.js';
+import {arrayToMap, isSiteMap} from '../util.js';
 
 // immutable
 export const binaryExtension = arrayToMap([

package/src/life-cycle/download-resource.ts

@@ -1,24 +1,22 @@
-import got, {
-  BeforeRetryHook,
-  NormalizedOptions,
-  Options,
-  RequestError,
-  TimeoutError
-} from 'got';
-import type {Response} from 'got/dist/source/as-promise';
-import type {DownloadResource, RequestOptions} from './types';
-import {generateSavePath, Resource, ResourceType} from '../resource';
-import type {StaticDownloadOptions} from '../options';
-import * as logger from '../logger/logger';
-import {isUrlHttp, sleep} from '../util';
+import type {BeforeRetryHook, OptionsInit, RequestError, Response} from 'got';
+import got, {TimeoutError} from 'got';
+import type {DownloadResource, RequestOptions} from './types.js';
+import type {Resource} from '../resource.js';
+import {generateSavePath, ResourceType} from '../resource.js';
+import type {StaticDownloadOptions} from '../options.js';
+import * as logger from '../logger/logger.js';
+import {isUrlHttp, sleep} from '../util.js';
 import URI from 'urijs';
 
 /** Take logs before retry */
 export const beforeRetryHook: BeforeRetryHook = (
-  options: NormalizedOptions,
-  error: RequestError | undefined,
+  error: RequestError,
   retryCount: number | undefined
 ) => {
+  const options = error.options;
+  if (!options) {
+    return;
+  }
   if (!error) {
     logger.retry.warn(retryCount, String(options.url));
     return;
@@ -49,15 +47,15 @@ export interface DownloadError extends Partial<Error> {
  */
 export async function getRetry(
   url: string,
-  options: Options
+  options: OptionsInit
 ): Promise<Response<Buffer | string> | void> {
-  let res: Response<Buffer | string> | void;
-  let err: DownloadError | void, optionsClone: Options;
+  let res: Response<Buffer | string> | void = void 0;
+  let err: DownloadError | void = void 0, optionsClone: OptionsInit;
   for (let i = 0; i < 25; i++) {
     err = void 0;
     try {
       optionsClone = Object.assign({}, options);
-      res = (await got(url, optionsClone)) as typeof res;
+      res = (await got(url, optionsClone)) as Response<Buffer | string>;
       if (!res || !res.body || !res.body.length) {
         logger.retry.warn(i, url, 'manually retry on empty response or body',
           res && res.body);
@@ -102,7 +100,7 @@ export async function requestForResource(
   options?: StaticDownloadOptions
 ): Promise<DownloadResource | Resource | void> {
   const downloadLink: string = encodeURI(decodeURI(res.downloadLink));
-  const reqOptions: Options = Object.assign({}, requestOptions);
+  const reqOptions: OptionsInit = Object.assign({}, requestOptions);
   reqOptions.responseType = 'buffer';
   if (res.refUrl && res.refUrl !== downloadLink) {
     const headers = Object.assign({}, reqOptions.headers);
@@ -167,6 +165,33 @@ export async function downloadResource(
     return downloadedResource;
   }
   if (downloadedResource.type === ResourceType.Html) {
+    if (options.meta.warnForNonHtml) {
+      const headers = downloadedResource.meta.headers;
+      if (headers) {
+        const contentType =
+          headers['content-type'] || headers['Content-Type'];
+        let nonHtml = false;
+        if (typeof contentType === 'string') {
+          nonHtml = !contentType.includes('/html') &&
+            !contentType.includes('/xml') &&
+            !contentType.includes('application/xhtml+xml');
+        } else if (Array.isArray(contentType)) {
+          nonHtml = true;
+          for (const header of contentType) {
+            if (!header.includes('/html') &&
+              !header.includes('/xml') &&
+              !header.includes('application/xhtml+xml')) {
+              nonHtml = false;
+              break;
+            }
+          }
+        }
+        if (nonHtml) {
+          logger.error.warn('Detected non-html content type',
+            downloadedResource.downloadLink, downloadedResource.rawUrl, contentType);
+        }
+      }
+    }
     if (options.meta.detectIncompleteHtml &&
       (typeof downloadedResource.body === 'string' ||
         Buffer.isBuffer(downloadedResource.body))) {

package/src/life-cycle/download-streaming-resource.ts

@@ -1,21 +1,23 @@
-import got, {HTTPError, RequestError} from 'got';
-import type {Response} from 'got/dist/source/core';
-import path from 'path';
-import {constants, createWriteStream, promises as fs, WriteStream} from 'fs';
-import {Resource, ResourceType} from '../resource';
+import path from 'node:path';
+import type {WriteStream} from 'node:fs';
+import {constants, createWriteStream, promises as fs} from 'node:fs';
+import {pipeline} from 'node:stream';
+import {promisify} from 'node:util';
+import type {RequestError, Response} from 'got';
+import got, {HTTPError} from 'got';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
 import type {
   AsyncResult,
   DownloadResource,
   DownloadResourceFunc,
   RequestOptions
-} from './types';
-import {mkdirRetry} from '../io';
-import {pipeline} from 'stream';
-import {promisify} from 'util';
-import {error as errorLogger, retry as retryLogger} from '../logger/logger';
-import type {StaticDownloadOptions} from '../options';
-import type {PipelineExecutor} from './pipeline-executor';
-import {isUrlHttp} from '../util';
+} from './types.js';
+import {mkdirRetry} from '../io.js';
+import {error as errorLogger, retry as retryLogger} from '../logger/logger.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import {isUrlHttp} from '../util.js';
 
 const promisifyPipeline = promisify(pipeline);
 
@@ -108,7 +110,7 @@ export async function streamingDownloadToFile(
           rangeStart = undefined;
         }
 
-        if (response.request.aborted) {
+        if (response.request.isAborted) {
           // Canceled while downloading
           //- will throw a `CancelError` or `TimeoutError` error
           return;
@@ -129,9 +131,9 @@ export async function streamingDownloadToFile(
           return;
         }
 
-        if (request._isAboutToError) {
-          return;
-        }
+        // if (request._isAboutToError) {
+        //   return;
+        // }
 
         resolve(response);
       });
@@ -222,7 +224,7 @@ export async function optionallySetLastModifiedTime(
   res: Resource, options: StaticDownloadOptions
 ): Promise<void> {
   // https://github.com/website-local/website-scrap-engine/issues/174
-  let mtime: number | void;
+  let mtime: number | void = void 0;
   if (options.preferRemoteLastModifiedTime && res.meta?.headers?.['last-modified']) {
     mtime = Date.parse(res.meta.headers?.['last-modified']);
   }

package/src/life-cycle/index.ts

@@ -1,21 +1,21 @@
-export * as adapter from './adapters';
-export {defaultLifeCycle} from './default-life-cycle';
-export {detectResourceType} from './detect-resource-type';
+export * as adapter from './adapters.js';
+export {defaultLifeCycle} from './default-life-cycle.js';
+export {detectResourceType} from './detect-resource-type.js';
 export {
   beforeRetryHook, getRetry, requestForResource, downloadResource
-} from './download-resource';
+} from './download-resource.js';
 export {
   streamingDownloadToFile,
   downloadStreamingResource,
   downloadStreamingResourceWithHook
-} from './download-streaming-resource';
-export {PipelineExecutor} from './pipeline-executor';
-export {processCssText, processCss} from './process-css';
-export {processHtml} from './process-html';
-export {processHtmlMetaRefresh} from './process-html-meta';
-export {processSiteMap} from './process-site-map';
-export {processSvg} from './process-svg';
-export {getResourceBodyFromHtml, saveHtmlToDisk} from './save-html-to-disk';
-export {saveResourceToDisk} from './save-resource-to-disk';
-export {skipLinks} from './skip-links';
-export * as types from './types';
+} from './download-streaming-resource.js';
+export type {PipelineExecutor} from './pipeline-executor.js';
+export {processCssText, processCss} from './process-css.js';
+export {processHtml} from './process-html.js';
+export {processHtmlMetaRefresh} from './process-html-meta.js';
+export {processSiteMap} from './process-site-map.js';
+export {processSvg} from './process-svg.js';
+export {getResourceBodyFromHtml, saveHtmlToDisk} from './save-html-to-disk.js';
+export {saveResourceToDisk} from './save-resource-to-disk.js';
+export {skipLinks} from './skip-links.js';
+export * as types from './types.js';

package/src/life-cycle/pipeline-executor.ts

@@ -1,14 +1,14 @@
-import type {Resource, ResourceEncoding, ResourceType} from '../resource';
-import type {StaticDownloadOptions} from '../options';
+import type {Resource, ResourceEncoding, ResourceType} from '../resource.js';
+import type {StaticDownloadOptions} from '../options.js';
 import type {
   AsyncResult,
   DownloadResource,
   RequestOptions,
   SubmitResourceFunc
-} from './types';
-import type {Cheerio} from '../types';
-import type {DownloaderWithMeta} from '../downloader/types';
-import type {WorkerInfo} from '../downloader/worker-pool';
+} from './types.js';
+import type {Cheerio} from '../types.js';
+import type {DownloaderWithMeta} from '../downloader/types.js';
+import type {WorkerInfo} from '../downloader/worker-pool.js';
 
 export interface PipelineExecutor {
   /**

package/src/life-cycle/process-css.ts

@@ -1,9 +1,10 @@
 import parseCssUrls from 'css-url-parser';
-import type {DownloadResource, SubmitResourceFunc} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {Resource, ResourceType} from '../resource';
-import {toString} from '../util';
-import type {PipelineExecutor} from './pipeline-executor';
+import type {DownloadResource, SubmitResourceFunc} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import {toString} from '../util.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
 
 export async function processCssText(
   cssText: string,

package/src/life-cycle/process-html-meta.ts

@@ -1,9 +1,10 @@
-import type {DownloadResource, SubmitResourceFunc} from './types';
-import type {StaticDownloadOptions} from '../options';
-import type {PipelineExecutor} from './pipeline-executor';
-import {Resource, ResourceType} from '../resource';
-import {parseHtml} from './adapters';
-import {skip} from '../logger/logger';
+import type {DownloadResource, SubmitResourceFunc} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import {parseHtml} from './adapters.js';
+import {skip} from '../logger/logger.js';
 
 /**
  * Originally create by https://github.com/stevenvachon at

package/src/life-cycle/process-html.ts

@@ -1,34 +1,31 @@
-import srcset, {SrcSetDefinition} from 'srcset';
-import {sources as defaultSources} from '../sources';
-import type {DownloadResource, SubmitResourceFunc} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {Resource, ResourceType} from '../resource';
-import {processCssText} from './process-css';
-import {error, skip} from '../logger/logger';
-import type {PipelineExecutor} from './pipeline-executor';
-import {parseHtml} from './adapters';
-import type {Cheerio, CheerioStatic} from '../types';
+import type {SrcSetDefinition} from 'srcset';
+import {parseSrcset, stringifySrcset} from 'srcset';
+import {load} from 'cheerio';
+import {sources as defaultSources} from '../sources.js';
+import type {DownloadResource, SubmitResourceFunc} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import {processCssText} from './process-css.js';
+import {error, skip} from '../logger/logger.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import {parseHtml} from './adapters.js';
+import type {Cheerio, CheerioStatic} from '../types.js';
 
-export async function processHtml(
-  res: DownloadResource,
-  submit: SubmitResourceFunc,
-  options: StaticDownloadOptions,
-  pipeline: PipelineExecutor): Promise<DownloadResource | void> {
-  if (res.type !== ResourceType.Html) {
-    return res;
-  }
-  const refUrl: string = res.redirectedUrl || res.url;
-  const savePath = refUrl === res.url ? res.savePath : undefined;
-  // useless since processRedirectedUrl enabled by default
-  // refUrl = await pipeline.linkRedirect(refUrl, null, res) || refUrl;
+type Writeable<T> = { -readonly [P in keyof T]: T[P] };
+type WriteableSrcSet = Writeable<SrcSetDefinition>;
 
-  const depth: number = res.depth + 1;
-  // resources from inline css
-  const resources: Resource[] = [];
-  let doc: CheerioStatic | void = res.meta.doc;
-  if (!doc) {
-    res.meta.doc = doc = parseHtml(res, options);
-  }
+async function processHtmlDoc(
+  options: StaticDownloadOptions,
+  doc: CheerioStatic,
+  res: DownloadResource,
+  pipeline: PipelineExecutor,
+  depth: number,
+  resources: Resource[],
+  refUrl: string,
+  savePath: string | undefined,
+  submit: SubmitResourceFunc
+) {
   const sources: typeof defaultSources = options.sources || defaultSources;
   for (const {selector, attr, type} of sources) {
     const elements: Cheerio = doc(selector);
@@ -54,7 +51,7 @@ export async function processHtml(
       let links: string[], replaceValue: string | SrcSetDefinition[];
       if (attr === 'srcset') {
         try {
-          replaceValue = srcset.parse(attrValue);
+          replaceValue = parseSrcset(attrValue);
         } catch (e) {
           error.info('skipping invalid srcset', attrValue, e);
           // should invalid srcset being removed?
@@ -104,7 +101,10 @@ export async function processHtml(
           submit(resource);
         }
         if (attr === 'srcset') {
-          (replaceValue as SrcSetDefinition[])[linkIndex].url = resource.replacePath;
+          // 20241005: It's ok to do this
+          // I've looked into the source code of srcset 5.0.1
+          // and there is nothing preventing the return value to change
+          (replaceValue as WriteableSrcSet[])[linkIndex].url = resource.replacePath;
         } else {
           replaceValue = resource.replacePath;
           // historical workaround here
@@ -114,7 +114,7 @@ export async function processHtml(
         }
       }
       if (attr === 'srcset') {
-        elem.attr(attr, srcset.stringify(replaceValue as SrcSetDefinition[]));
+        elem.attr(attr, stringifySrcset(replaceValue as SrcSetDefinition[]));
       } else if (attr) {
         elem.attr(attr, replaceValue as string);
       } else {
@@ -122,6 +122,48 @@ export async function processHtml(
       }
     }
   }
+  const iframeSrcDocs = doc('iframe[srcdoc]');
+
+  for (let index = 0; index < iframeSrcDocs.length; index++) {
+    const elem = iframeSrcDocs.eq(index);
+    const attrValue: string | void = elem.attr('srcdoc');
+    if (!attrValue) {
+      continue;
+    }
+    try {
+      const iframeDoc = load(attrValue);
+      await processHtmlDoc(options, iframeDoc, res, pipeline, depth, resources, refUrl, savePath, submit);
+      const html = options.cheerioSerialize ?
+        iframeDoc.html(options.cheerioSerialize) : iframeDoc.html();
+      elem.attr('srcdoc', html);
+    } catch (e) {
+      error.info('can not parse iframe srcdoc', res.url, res.rawUrl, e);
+    }
+  }
+}
+
+export async function processHtml(
+  res: DownloadResource,
+  submit: SubmitResourceFunc,
+  options: StaticDownloadOptions,
+  pipeline: PipelineExecutor
+): Promise<DownloadResource | void> {
+  if (res.type !== ResourceType.Html) {
+    return res;
+  }
+  const refUrl: string = res.redirectedUrl || res.url;
+  const savePath = refUrl === res.url ? res.savePath : undefined;
+  // useless since processRedirectedUrl enabled by default
+  // refUrl = await pipeline.linkRedirect(refUrl, null, res) || refUrl;
+
+  const depth: number = res.depth + 1;
+  let doc: CheerioStatic | void = res.meta.doc;
+  if (!doc) {
+    res.meta.doc = doc = parseHtml(res, options);
+  }
+  // resources from inline css
+  const resources: Resource[] = [];
+  await processHtmlDoc(options, doc, res, pipeline, depth, resources, refUrl, savePath, submit);
   if (resources.length) {
     submit(resources);
   }

package/src/life-cycle/process-site-map.ts

@@ -1,10 +1,11 @@
 import {load} from 'cheerio';
-import type {DownloadResource, SubmitResourceFunc} from './types';
-import type {StaticDownloadOptions} from '../options';
-import {Resource, ResourceType} from '../resource';
-import {toString} from '../util';
-import type {PipelineExecutor} from './pipeline-executor';
-import type {CheerioStatic} from '../types';
+import type {DownloadResource, SubmitResourceFunc} from './types.js';
+import type {StaticDownloadOptions} from '../options.js';
+import type {Resource} from '../resource.js';
+import {ResourceType} from '../resource.js';
+import {toString} from '../util.js';
+import type {PipelineExecutor} from './pipeline-executor.js';
+import type {CheerioStatic} from '../types.js';
 
 export async function processSiteMap(
   res: DownloadResource,