website-scrap-engine 0.4.0 → 0.6.0

package/src/life-cycle/download-resource.ts

@@ -7,10 +7,11 @@ import got, {
 } from 'got';
 import type {Response} from 'got/dist/source/as-promise';
 import type {DownloadResource, RequestOptions} from './types';
-import {Resource, ResourceType} from '../resource';
+import {generateSavePath, Resource, ResourceType} from '../resource';
 import type {StaticDownloadOptions} from '../options';
 import * as logger from '../logger/logger';
 import {isUrlHttp, sleep} from '../util';
+import URI from 'urijs';
 
 /** Take logs before retry */
 export const beforeRetryHook: BeforeRetryHook = (
@@ -33,6 +34,11 @@ export const beforeRetryHook: BeforeRetryHook = (
   }
 };
 
+export interface DownloadError extends Partial<Error> {
+  retryLimitExceeded?: boolean;
+  code?: string;
+  event?: string;
+}
 
 /**
  * workaround for retry premature close on node 12
@@ -46,7 +52,7 @@ export async function getRetry(
   options: Options
 ): Promise<Response<Buffer | string> | void> {
   let res: Response<Buffer | string> | void;
-  let err: Error | void, optionsClone: Options;
+  let err: DownloadError | void, optionsClone: Options;
   for (let i = 0; i < 25; i++) {
     err = void 0;
     try {
@@ -59,22 +65,23 @@ export async function getRetry(
       }
       break;
     } catch (e) {
-      err = e;
-      if (e && e.message === 'premature close') {
+      // force cast for typescript 4.4
+      err = e as DownloadError | void;
+      if (err && err.message === 'premature close') {
         logger.retry.warn(i, url, 'manually retry on premature close',
-          e.name, e.code, e.event, e.message);
+          err.name, err.code, err.event, err.message);
         await sleep(i * 200);
         continue;
       }
       // these events might be accidentally unhandled
-      if (e && !e.retryLimitExceeded &&
-        (e.name === 'RequestError' || e.name === 'TimeoutError') &&
+      if (err && !err.retryLimitExceeded &&
+        (err.name === 'RequestError' || err.name === 'TimeoutError') &&
         // RequestError: Cannot read property 'request' of undefined
         // at Object.exports.default (got\dist\source\core\utils\timed-out.js:56:23)
         // error.code === undefined
-        (e.code === 'ETIMEDOUT' || e.code === undefined)) {
-        logger.retry.warn(i, url, `manually retry on ${e.event} timeout`,
-          e.name, e.code, e.message);
+        (err.code === 'ETIMEDOUT' || err.code === undefined)) {
+        logger.retry.warn(i, url, `manually retry on ${err.event} timeout`,
+          err.name, err.code, err.message);
         await sleep(i * 300);
         continue;
       }
@@ -91,7 +98,8 @@ export async function getRetry(
 
 export async function requestForResource(
   res: Resource & { downloadStartTimestamp: number },
-  requestOptions: RequestOptions
+  requestOptions: RequestOptions,
+  options?: StaticDownloadOptions
 ): Promise<DownloadResource | Resource | void> {
   const downloadLink: string = encodeURI(decodeURI(res.downloadLink));
   const reqOptions: Options = Object.assign({}, requestOptions);
@@ -122,6 +130,15 @@ export async function requestForResource(
   res.finishTimestamp = Date.now();
   res.downloadTime = res.finishTimestamp - res.downloadStartTimestamp;
   res.redirectedUrl = response.url;
+  // https://github.com/website-local/website-scrap-engine/issues/385
+  // 2011/11/15
+  if (res.redirectedUrl !== res.url) {
+    res.redirectedSavePath = generateSavePath(
+      URI(res.redirectedUrl),
+      res.type === ResourceType.Html,
+      !options?.deduplicateStripSearch,
+      options?.localSrcRoot);
+  }
   res.body = response.body;
   return res;
 }
@@ -145,7 +162,7 @@ export async function downloadResource(
     res.waitTime = res.downloadStartTimestamp - res.createTimestamp;
   }
   let downloadedResource: DownloadResource | Resource | void = await requestForResource(
-    res as (Resource & { downloadStartTimestamp: number }), requestOptions);
+    res as (Resource & { downloadStartTimestamp: number }), requestOptions, options);
   if (!downloadedResource || !downloadedResource.body) {
     return downloadedResource;
   }

package/src/life-cycle/download-streaming-resource.ts

@@ -53,7 +53,8 @@ export async function streamingDownloadToFile(
   try {
     await fs.access(savePath, constants.W_OK);
   } catch (e) {
-    if (e?.code === 'ENOENT') {
+    // force cast for typescript 4.4
+    if (e && (e as {code?: string | void}).code === 'ENOENT') {
       await mkdirRetry(path.dirname(savePath));
     } else {
       throw e;
@@ -303,7 +304,7 @@ export interface StreamingDownloadErrorHook {
    * @param options
    * @param pipeline
    */
-  (e: Error, res: Resource, requestOptions: RequestOptions,
+  (e: Error | unknown, res: Resource, requestOptions: RequestOptions,
    options: StaticDownloadOptions,
    pipeline: PipelineExecutor): AsyncResult<void>;
 }

package/src/life-cycle/process-site-map.ts

@@ -1,4 +1,4 @@
-import cheerio from 'cheerio';
+import {load} from 'cheerio';
 import type {DownloadResource, SubmitResourceFunc} from './types';
 import type {StaticDownloadOptions} from '../options';
 import {Resource, ResourceType} from '../resource';
@@ -14,7 +14,7 @@ export async function processSiteMap(
   if (res.type !== ResourceType.SiteMap) {
     return res;
   }
-  const $: CheerioStatic = cheerio.load(toString(res.body,
+  const $: CheerioStatic = load(toString(res.body,
     res.encoding || options.encoding[ResourceType.SiteMap] || 'utf8'));
   const urlSet: Set<string> = new Set();
   const depth: number = res.depth + 1;

package/src/life-cycle/types.ts

@@ -1,6 +1,7 @@
 import type {Options as GotOptions} from 'got/dist/source/as-promise';
 import type {
   createResource,
+  GenerateSavePathFn,
   Resource,
   ResourceBody,
   ResourceType
@@ -176,6 +177,7 @@ export interface ProcessingLifeCycle {
   init: InitLifeCycleFunc[];
   linkRedirect: LinkRedirectFunc[];
   detectResourceType: DetectResourceTypeFunc[];
+  generateSavePath?: GenerateSavePathFn | void;
   createResource: typeof createResource;
   /**
    * link in parent resource would be replaced after this

package/src/logger/logger-worker.ts

@@ -4,7 +4,7 @@ import type {LogWorkerMessage, WorkerLog} from '../downloader/worker-type';
 import {WorkerMessageType} from '../downloader/types';
 
 export const logLevels = [
-  'log', 'trace', 'debug', 'info', 'warn', 'error', 'fatal', 'mark'
+  'trace', 'debug', 'info', 'warn', 'error', 'fatal', 'mark'
 ] as const;
 
 export function getWorkerLogger(category: WorkerLog['logger']): Logger {

package/src/resource.ts

@@ -285,6 +285,10 @@ export interface CreateResourceArgument {
    * https://github.com/website-local/website-scrap-engine/issues/107
    */
   skipReplacePathError?: boolean;
+  /**
+   * Set this to use a custom implementation of {@link generateSavePath}
+   */
+  generateSavePathFn?: GenerateSavePathFn | void;
 }
 
 /**
@@ -293,6 +297,7 @@ export interface CreateResourceArgument {
  * @param isHtml should the savePath endsWith .html
  * @param keepSearch keep url search params in file name
  * @param localSrcRoot local source path to download from
+ * @return string must return non-empty string
  */
 export function generateSavePath(
   uri: URI,
@@ -361,6 +366,8 @@ export function generateSavePath(
   return savePath;
 }
 
+export type GenerateSavePathFn = typeof generateSavePath;
+
 export const urlOfSavePath = (savePath: string): string => {
   if (savePath.includes('\\')) {
     return `file:///${savePath.replace(/\\/g, '/')}`;
@@ -486,6 +493,7 @@ export function resolveFileUrl(
  * @param encoding {@link CreateResourceArgument.encoding}
  * @param keepSearch {@link CreateResourceArgument.keepSearch}
  * @param skipReplacePathError {@link CreateResourceArgument.skipReplacePathError}
+ * @param generateSavePathFn {@link CreateResourceArgument.generateSavePathFn}
  * @return the resource
  */
 export function createResource({
@@ -499,7 +507,8 @@ export function createResource({
   localSrcRoot,
   encoding,
   keepSearch,
-  skipReplacePathError
+  skipReplacePathError,
+  generateSavePathFn
 }: CreateResourceArgument): Resource {
   const rawUrl: string = url;
   const refUri: URI = URI(refUrl);
@@ -541,11 +550,13 @@ export function createResource({
     downloadLink = uri.clone().hash('').toString();
   }
 
+  const implGenerateSavePath = generateSavePathFn || generateSavePath;
+
   // make savePath and replaceUri
-  const savePath = replacePathHasError ? rawUrl : generateSavePath(
+  const savePath = replacePathHasError ? rawUrl : implGenerateSavePath(
     uri, type === ResourceType.Html, keepSearch, localSrcRoot);
   if (!refSavePath) {
-    refSavePath = generateSavePath(refUri, refType === ResourceType.Html,
+    refSavePath = implGenerateSavePath(refUri, refType === ResourceType.Html,
       false, localSrcRoot);
   }
   const replaceUri = replacePathHasError ? URI(rawUrl) :

package/src/types.ts

@@ -1,7 +1,7 @@
-import type cheerio from 'cheerio';
+import type {load} from 'cheerio';
 
 // adapters for making cheerio's namespace type definitions to module
-export type CheerioStatic = ReturnType<typeof cheerio.load>;
+export type CheerioStatic = ReturnType<typeof load>;
 export type Cheerio = ReturnType<CheerioStatic>;
-export type CheerioOptionsInterface = NonNullable<Parameters<typeof cheerio.load>[1]>;
+export type CheerioOptionsInterface = NonNullable<Parameters<typeof load>[1]>;
 export type CheerioElement = Cheerio[number];

package/src/util.ts

@@ -88,10 +88,9 @@ export const hasOwnProperty = Object.prototype.hasOwnProperty;
  * may lead to typescript parser errors.
  */
 export const weakAssign = <T, U>(target: T, source: U): T & U => {
-  if (!target) return Object.assign({} as T, source);
+  if (!target) return Object.assign({}, source) as T & U;
   if (!source) return target as T & U;
-  let key: keyof U;
-  for (key in source) {
+  for (const key in source) {
     if (hasOwnProperty.call(source, key) &&
       !hasOwnProperty.call(target, key)) {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any