website-scrap-engine 0.7.0 → 0.7.1

package/lib/life-cycle/default-life-cycle.js

@@ -6,6 +6,7 @@ const detect_resource_type_1 = require("./detect-resource-type");
 const resource_1 = require("../resource");
 const download_resource_1 = require("./download-resource");
 const process_html_1 = require("./process-html");
+const process_html_meta_1 = require("./process-html-meta");
 const process_css_1 = require("./process-css");
 const process_site_map_1 = require("./process-site-map");
 const process_svg_1 = require("./process-svg");
@@ -31,6 +32,7 @@ const defaultLifeCycle = () => ({
     processAfterDownload: [
         adapters_1.processRedirectedUrl,
         process_html_1.processHtml,
+        process_html_meta_1.processHtmlMetaRefresh,
         process_svg_1.processSvg,
         process_css_1.processCss,
         process_site_map_1.processSiteMap

package/lib/life-cycle/default-life-cycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"default-life-cycle.js","sourceRoot":"","sources":["../../src/life-cycle/default-life-cycle.ts"],"names":[],"mappings":";;;AACA,6CAAuC;AACvC,iEAA0D;AAC1D,0CAA2C;AAC3C,2DAAqD;AACrD,iDAA2C;AAC3C,+CAAyC;AACzC,yDAAkD;AAClD,+CAAyC;AACzC,2DAAmD;AACnD,mEAA2D;AAC3D,yCAAgD;AAChD,+EAAwE;AACxE,+EAAsE;AAEtE;;GAEG;AACI,MAAM,gBAAgB,GAAG,GAAwB,EAAE,CAAC,CAAC;IAC1D,IAAI,EAAE,EAAE;IACR,YAAY,EAAE,CAAC,sBAAS,CAAC;IACzB,kBAAkB,EAAE,CAAC,yCAAkB,CAAC;IACxC,cAAc,EAAd,yBAAc;IACd,qBAAqB,EAAE,EAAE;IACzB,QAAQ,EAAE;QACR,oCAAgB;QAChB,uDAAyB;QACzB,qDAAuB;KACxB;IACD,oBAAoB,EAAE;QACpB,+BAAoB;QACpB,0BAAW;QACX,wBAAU;QACV,wBAAU;QACV,iCAAc;KACf;IACD,UAAU,EAAE,CAAC,kCAAc,EAAE,0CAAkB,CAAC;IAChD,OAAO,EAAE,EAAE;CACZ,CAAC,CAAC;AApBU,QAAA,gBAAgB,oBAoB1B"}
+{"version":3,"file":"default-life-cycle.js","sourceRoot":"","sources":["../../src/life-cycle/default-life-cycle.ts"],"names":[],"mappings":";;;AACA,6CAAuC;AACvC,iEAA0D;AAC1D,0CAA2C;AAC3C,2DAAqD;AACrD,iDAA2C;AAC3C,2DAA2D;AAC3D,+CAAyC;AACzC,yDAAkD;AAClD,+CAAyC;AACzC,2DAAmD;AACnD,mEAA2D;AAC3D,yCAAgD;AAChD,+EAAwE;AACxE,+EAAsE;AAEtE;;GAEG;AACI,MAAM,gBAAgB,GAAG,GAAwB,EAAE,CAAC,CAAC;IAC1D,IAAI,EAAE,EAAE;IACR,YAAY,EAAE,CAAC,sBAAS,CAAC;IACzB,kBAAkB,EAAE,CAAC,yCAAkB,CAAC;IACxC,cAAc,EAAd,yBAAc;IACd,qBAAqB,EAAE,EAAE;IACzB,QAAQ,EAAE;QACR,oCAAgB;QAChB,uDAAyB;QACzB,qDAAuB;KACxB;IACD,oBAAoB,EAAE;QACpB,+BAAoB;QACpB,0BAAW;QACX,0CAAsB;QACtB,wBAAU;QACV,wBAAU;QACV,iCAAc;KACf;IACD,UAAU,EAAE,CAAC,kCAAc,EAAE,0CAAkB,CAAC;IAChD,OAAO,EAAE,EAAE;CACZ,CAAC,CAAC;AArBU,QAAA,gBAAgB,oBAqB1B"}

package/lib/life-cycle/index.d.ts

@@ -6,6 +6,7 @@ export { streamingDownloadToFile, downloadStreamingResource, downloadStreamingRe
 export { PipelineExecutor } from './pipeline-executor';
 export { processCssText, processCss } from './process-css';
 export { processHtml } from './process-html';
+export { processHtmlMetaRefresh } from './process-html-meta';
 export { processSiteMap } from './process-site-map';
 export { processSvg } from './process-svg';
 export { getResourceBodyFromHtml, saveHtmlToDisk } from './save-html-to-disk';

package/lib/life-cycle/index.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.types = exports.skipLinks = exports.saveResourceToDisk = exports.saveHtmlToDisk = exports.getResourceBodyFromHtml = exports.processSvg = exports.processSiteMap = exports.processHtml = exports.processCss = exports.processCssText = exports.downloadStreamingResourceWithHook = exports.downloadStreamingResource = exports.streamingDownloadToFile = exports.downloadResource = exports.requestForResource = exports.getRetry = exports.beforeRetryHook = exports.detectResourceType = exports.defaultLifeCycle = exports.adapter = void 0;
+exports.types = exports.skipLinks = exports.saveResourceToDisk = exports.saveHtmlToDisk = exports.getResourceBodyFromHtml = exports.processSvg = exports.processSiteMap = exports.processHtmlMetaRefresh = exports.processHtml = exports.processCss = exports.processCssText = exports.downloadStreamingResourceWithHook = exports.downloadStreamingResource = exports.streamingDownloadToFile = exports.downloadResource = exports.requestForResource = exports.getRetry = exports.beforeRetryHook = exports.detectResourceType = exports.defaultLifeCycle = exports.adapter = void 0;
 exports.adapter = __importStar(require("./adapters"));
 var default_life_cycle_1 = require("./default-life-cycle");
 Object.defineProperty(exports, "defaultLifeCycle", { enumerable: true, get: function () { return default_life_cycle_1.defaultLifeCycle; } });
@@ -43,6 +43,8 @@ Object.defineProperty(exports, "processCssText", { enumerable: true, get: functi
 Object.defineProperty(exports, "processCss", { enumerable: true, get: function () { return process_css_1.processCss; } });
 var process_html_1 = require("./process-html");
 Object.defineProperty(exports, "processHtml", { enumerable: true, get: function () { return process_html_1.processHtml; } });
+var process_html_meta_1 = require("./process-html-meta");
+Object.defineProperty(exports, "processHtmlMetaRefresh", { enumerable: true, get: function () { return process_html_meta_1.processHtmlMetaRefresh; } });
 var process_site_map_1 = require("./process-site-map");
 Object.defineProperty(exports, "processSiteMap", { enumerable: true, get: function () { return process_site_map_1.processSiteMap; } });
 var process_svg_1 = require("./process-svg");

package/lib/life-cycle/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/life-cycle/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sDAAsC;AACtC,2DAAsD;AAA9C,sHAAA,gBAAgB,OAAA;AACxB,+DAA0D;AAAlD,0HAAA,kBAAkB,OAAA;AAC1B,yDAE6B;AAD3B,oHAAA,eAAe,OAAA;AAAE,6GAAA,QAAQ,OAAA;AAAE,uHAAA,kBAAkB,OAAA;AAAE,qHAAA,gBAAgB,OAAA;AAEjE,6EAIuC;AAHrC,sIAAA,uBAAuB,OAAA;AACvB,wIAAA,yBAAyB,OAAA;AACzB,gJAAA,iCAAiC,OAAA;AAGnC,6CAAyD;AAAjD,6GAAA,cAAc,OAAA;AAAE,yGAAA,UAAU,OAAA;AAClC,+CAA2C;AAAnC,2GAAA,WAAW,OAAA;AACnB,uDAAkD;AAA1C,kHAAA,cAAc,OAAA;AACtB,6CAAyC;AAAjC,yGAAA,UAAU,OAAA;AAClB,yDAA4E;AAApE,4HAAA,uBAAuB,OAAA;AAAE,mHAAA,cAAc,OAAA;AAC/C,iEAA2D;AAAnD,2HAAA,kBAAkB,OAAA;AAC1B,2CAAuC;AAA/B,uGAAA,SAAS,OAAA;AACjB,iDAAiC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/life-cycle/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sDAAsC;AACtC,2DAAsD;AAA9C,sHAAA,gBAAgB,OAAA;AACxB,+DAA0D;AAAlD,0HAAA,kBAAkB,OAAA;AAC1B,yDAE6B;AAD3B,oHAAA,eAAe,OAAA;AAAE,6GAAA,QAAQ,OAAA;AAAE,uHAAA,kBAAkB,OAAA;AAAE,qHAAA,gBAAgB,OAAA;AAEjE,6EAIuC;AAHrC,sIAAA,uBAAuB,OAAA;AACvB,wIAAA,yBAAyB,OAAA;AACzB,gJAAA,iCAAiC,OAAA;AAGnC,6CAAyD;AAAjD,6GAAA,cAAc,OAAA;AAAE,yGAAA,UAAU,OAAA;AAClC,+CAA2C;AAAnC,2GAAA,WAAW,OAAA;AACnB,yDAA2D;AAAnD,2HAAA,sBAAsB,OAAA;AAC9B,uDAAkD;AAA1C,kHAAA,cAAc,OAAA;AACtB,6CAAyC;AAAjC,yGAAA,UAAU,OAAA;AAClB,yDAA4E;AAApE,4HAAA,uBAAuB,OAAA;AAAE,mHAAA,cAAc,OAAA;AAC/C,iEAA2D;AAAnD,2HAAA,kBAAkB,OAAA;AAC1B,2CAAuC;AAA/B,uGAAA,SAAS,OAAA;AACjB,iDAAiC"}

package/lib/life-cycle/process-html-meta.d.ts

@@ -0,0 +1,4 @@
+import type { DownloadResource, SubmitResourceFunc } from './types';
+import type { StaticDownloadOptions } from '../options';
+import type { PipelineExecutor } from './pipeline-executor';
+export declare function processHtmlMetaRefresh(res: DownloadResource, submit: SubmitResourceFunc, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource>;

package/lib/life-cycle/process-html-meta.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processHtmlMetaRefresh = void 0;
+const resource_1 = require("../resource");
+const adapters_1 = require("./adapters");
+const logger_1 = require("../logger/logger");
+/**
+ * Originally create by https://github.com/stevenvachon at
+ * https://github.com/stevenvachon/http-equiv-refresh
+ * MIT license
+ */
+const META_REFRESH_PATTERN = /^\s*(\d+)(?:\s*;(?:\s*url\s*=)?\s*(?:["']\s*(.*?)\s*['"]|(.*?)))?\s*$/i;
+async function processHtmlMetaRefresh(res, submit, options, pipeline) {
+    if (res.type !== resource_1.ResourceType.Html) {
+        return res;
+    }
+    if (!res.meta.doc) {
+        res.meta.doc = (0, adapters_1.parseHtml)(res, options);
+    }
+    const $ = res.meta.doc;
+    const metaLinks = $('meta[http-equiv="refresh"][content]');
+    if (metaLinks.length) {
+        const refUrl = res.redirectedUrl || res.url;
+        const savePath = refUrl === res.url ? res.savePath : undefined;
+        const depth = res.depth + 1;
+        for (let index = 0; index < metaLinks.length; index++) {
+            const elem = metaLinks.eq(index);
+            const attrValue = elem.attr('content');
+            if (!attrValue) {
+                continue;
+            }
+            const match = META_REFRESH_PATTERN.exec(attrValue);
+            if (!match) {
+                continue;
+            }
+            const originalLink = match[2] || match[3];
+            if (!originalLink) {
+                continue;
+            }
+            const link = await pipeline.linkRedirect(originalLink, elem, res);
+            if (!link) {
+                continue;
+            }
+            const linkType = await pipeline.detectResourceType(link, resource_1.ResourceType.Html, elem, res);
+            if (!linkType) {
+                if (logger_1.skip.isTraceEnabled()) {
+                    logger_1.skip.trace('skip detectResourceType', originalLink, link, refUrl);
+                }
+                continue;
+            }
+            let resource = await pipeline.createResource(linkType, depth, link, refUrl, res.localRoot, options.encoding[linkType], savePath, res.type);
+            resource = await pipeline.processBeforeDownload(resource, elem, res, options);
+            if (!resource) {
+                if (logger_1.skip.isTraceEnabled()) {
+                    logger_1.skip.trace('skip processBeforeDownload', originalLink, link, linkType, refUrl);
+                }
+                continue;
+            }
+            if (!resource.shouldBeDiscardedFromDownload) {
+                submit(resource);
+            }
+            elem.attr('content', attrValue.replace(originalLink, resource.replacePath));
+        }
+    }
+    return res;
+}
+exports.processHtmlMetaRefresh = processHtmlMetaRefresh;
+//# sourceMappingURL=process-html-meta.js.map

package/lib/life-cycle/process-html-meta.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"process-html-meta.js","sourceRoot":"","sources":["../../src/life-cycle/process-html-meta.ts"],"names":[],"mappings":";;;AAGA,0CAAmD;AACnD,yCAAqC;AACrC,6CAAsC;AAEtC;;;;GAIG;AACH,MAAM,oBAAoB,GACxB,wEAAwE,CAAC;AAEpE,KAAK,UAAU,sBAAsB,CAC1C,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAG1B,IAAI,GAAG,CAAC,IAAI,KAAK,uBAAY,CAAC,IAAI,EAAE;QAClC,OAAO,GAAG,CAAC;KACZ;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE;QACjB,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,IAAA,oBAAS,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;KACxC;IACD,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;IAEvB,MAAM,SAAS,GAAG,CAAC,CAAC,qCAAqC,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,MAAM,EAAE;QACpB,MAAM,MAAM,GAAW,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,GAAG,CAAC;QACpD,MAAM,QAAQ,GAAG,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAE/D,MAAM,KAAK,GAAW,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;QAEpC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACrD,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;YACjC,MAAM,SAAS,GAAkB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,CAAC,SAAS,EAAE;gBACd,SAAS;aACV;YACD,MAAM,KAAK,GAAG,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACnD,IAAI,CAAC,KAAK,EAAE;gBACV,SAAS;aACV;YACD,MAAM,YAAY,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,CAAC,YAAY,EAAE;gBACjB,SAAS;aACV;YACD,MAAM,IAAI,GACR,MAAM,QAAQ,CAAC,YAAY,CAAC,YAAY,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;YACvD,IAAI,CAAC,IAAI,EAAE;gBACT,SAAS;aACV;YAED,MAAM,QAAQ,GACZ,MAAM,QAAQ,CAAC,kBAAkB,CAAC,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;YACxE,IAAI,CAAC,QAAQ,EAAE;gBACb,IAAI,aAAI,CAAC,cAAc,EAAE,EAAE;oBACzB,aAAI,CAAC,KAAK,CAAC,yBAAyB,EAClC,YAAY,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;iBAC/B;gBACD,SAAS;aACV;YACD,IAAI,QAAQ,GAAoB,MAAM,QAAQ,CAAC,cAAc,CAC3D,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAC7B,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EACzC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACtB,QAAQ,GAAG,MAAM,QAAQ,CAAC,qBAAqB,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9E,IAAI,CAAC,QAAQ,EAAE;gBACb,IAAI,aAAI,CAAC,cAAc,EAAE,EAAE;oBACzB,aAAI,CAAC,KAAK,CAAC,4BAA4B,EACrC,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;iBACzC;gBACD,SAAS;aACV;YACD,IAAI,CAAC,QAAQ,CAAC,6BAA6B,EAAE;gBAC3C,MAAM,CAAC,QAAQ,CAAC,CAAC;aAClB;YACD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;SAC7E;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAvED,wDAuEC"}

package/lib/sources.js

@@ -32,6 +32,7 @@ exports.sources = [
     { selector: 'meta[property="og\\:video\\:url"]', attr: 'content' },
     { selector: 'meta[property="og\\:video\\:secure_url"]', attr: 'content' },
     { selector: 'video', attr: 'src' },
+    { selector: 'video', attr: 'poster' },
     { selector: 'video source', attr: 'src' },
     { selector: 'video track', attr: 'src' },
     { selector: 'audio', attr: 'src' },

package/lib/sources.js.map

@@ -1 +1 @@
-{"version":3,"file":"sources.js","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":";;;AAAA,yCAAwC;AAQxC,0DAA0D;AAC1D,wEAAwE;AAC3D,QAAA,OAAO,GAAuB;IACzC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAY,CAAC,SAAS,EAAC;IACjD,EAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAY,CAAC,SAAS,EAAC;IAClE,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAC;IAC9B,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAC;IACjC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAC;IAClC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAC;IAChD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAC;IACjC,EAAC,QAAQ,EAAE,wBAAwB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAY,CAAC,GAAG,EAAC;IAC1E,EAAC,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAC;IAC7C,EAAC,QAAQ,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,EAAC;IAChD,yCAAyC;IACzC,qDAAqD;IACrD,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,YAAY,EAAC;IACrD,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAC;IACvC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC5C,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IACzD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IAC1D,EAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IACtD,mEAAmE;IACnE,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAC;CAC/C,CAAC,GAAG,CAAC,CAAC,GAA8B,EAAE,EAAE;IACvC,IAAI,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE;QAC/D,GAAG,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC;KACjC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;QACb,GAAG,CAAC,IAAI,GAAG,uBAAY,CAAC,MAAM,CAAC;KAChC;IACD,OAAO,GAAuB,CAAC;AACjC,CAAC,CAAC,CAAC"}
+{"version":3,"file":"sources.js","sourceRoot":"","sources":["../src/sources.ts"],"names":[],"mappings":";;;AAAA,yCAAwC;AAQxC,0DAA0D;AAC1D,wEAAwE;AAC3D,QAAA,OAAO,GAAuB;IACzC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAY,CAAC,SAAS,EAAC;IACjD,EAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,uBAAY,CAAC,SAAS,EAAC;IAClE,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAC;IAC9B,EAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAC;IACjC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAC;IAClC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,OAAO,EAAC;IAChD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAC;IACjC,EAAC,QAAQ,EAAE,wBAAwB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAY,CAAC,GAAG,EAAC;IAC1E,EAAC,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAC;IAC7C,EAAC,QAAQ,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,EAAC;IAChD,yCAAyC;IACzC,qDAAqD;IACrD,EAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,YAAY,EAAC;IACrD,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAC;IACvC,EAAC,QAAQ,EAAE,gBAAgB,EAAE,IAAI,EAAE,QAAQ,EAAC;IAC5C,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,6BAA6B,EAAE,IAAI,EAAE,SAAS,EAAC;IAC1D,EAAC,QAAQ,EAAE,mCAAmC,EAAE,IAAI,EAAE,SAAS,EAAC;IAChE,EAAC,QAAQ,EAAE,0CAA0C,EAAE,IAAI,EAAE,SAAS,EAAC;IACvE,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAC;IACnC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAC;IAChC,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,KAAK,EAAC;IACvC,EAAC,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,EAAC;IACtC,EAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IACzD,EAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IAC1D,EAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAY,CAAC,IAAI,EAAC;IACtD,mEAAmE;IACnE,EAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,EAAE,YAAY,EAAC;CAC/C,CAAC,GAAG,CAAC,CAAC,GAA8B,EAAE,EAAE;IACvC,IAAI,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE;QAC/D,GAAG,CAAC,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC;KACjC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;QACb,GAAG,CAAC,IAAI,GAAG,uBAAY,CAAC,MAAM,CAAC;KAChC;IACD,OAAO,GAAuB,CAAC;AACjC,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "website-scrap-engine",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Configurable website scraper in typescript",
   "main": "lib",
   "types": "lib",
@@ -25,20 +25,20 @@
     "css-url-parser": "^1.1.3",
     "got": "^11.8.6",
     "log4js": "^6.9.1",
-    "mkdirp": "^3.0.0",
+    "mkdirp": "^3.0.1",
     "p-queue": "^6.6.2",
     "srcset": "^4.0.0",
     "urijs": "^1.19.11"
   },
   "devDependencies": {
-    "@types/jest": "^27.5.2",
-    "@types/node": "^18.15.13",
-    "@types/urijs": "^1.19.19",
-    "@typescript-eslint/eslint-plugin": "^5.59.0",
-    "@typescript-eslint/parser": "^5.59.0",
-    "eslint": "^8.38.0",
-    "jest": "^27.5.1",
-    "ts-jest": "^27.1.5",
+    "@types/jest": "^28.1.1",
+    "@types/node": "^20.8.7",
+    "@types/urijs": "^1.19.20",
+    "@typescript-eslint/eslint-plugin": "^5.62.0",
+    "@typescript-eslint/parser": "^5.62.0",
+    "eslint": "^8.52.0",
+    "jest": "^28.1.3",
+    "ts-jest": "^28.0.8",
     "typescript": "^5.0.4"
   },
   "files": [

package/src/life-cycle/default-life-cycle.ts

@@ -4,6 +4,7 @@ import {detectResourceType} from './detect-resource-type';
 import {createResource} from '../resource';
 import {downloadResource} from './download-resource';
 import {processHtml} from './process-html';
+import {processHtmlMetaRefresh} from './process-html-meta';
 import {processCss} from './process-css';
 import {processSiteMap} from './process-site-map';
 import {processSvg} from './process-svg';
@@ -30,6 +31,7 @@ export const defaultLifeCycle = (): ProcessingLifeCycle => ({
   processAfterDownload: [
     processRedirectedUrl,
     processHtml,
+    processHtmlMetaRefresh,
     processSvg,
     processCss,
     processSiteMap

package/src/life-cycle/index.ts

@@ -12,6 +12,7 @@ export {
 export {PipelineExecutor} from './pipeline-executor';
 export {processCssText, processCss} from './process-css';
 export {processHtml} from './process-html';
+export {processHtmlMetaRefresh} from './process-html-meta';
 export {processSiteMap} from './process-site-map';
 export {processSvg} from './process-svg';
 export {getResourceBodyFromHtml, saveHtmlToDisk} from './save-html-to-disk';

package/src/life-cycle/process-html-meta.ts

@@ -0,0 +1,87 @@
+import type {DownloadResource, SubmitResourceFunc} from './types';
+import type {StaticDownloadOptions} from '../options';
+import type {PipelineExecutor} from './pipeline-executor';
+import {Resource, ResourceType} from '../resource';
+import {parseHtml} from './adapters';
+import {skip} from '../logger/logger';
+
+/**
+ * Originally create by https://github.com/stevenvachon at
+ * https://github.com/stevenvachon/http-equiv-refresh
+ * MIT license
+ */
+const META_REFRESH_PATTERN =
+  /^\s*(\d+)(?:\s*;(?:\s*url\s*=)?\s*(?:["']\s*(.*?)\s*['"]|(.*?)))?\s*$/i;
+
+export async function processHtmlMetaRefresh(
+  res: DownloadResource,
+  submit: SubmitResourceFunc,
+  options: StaticDownloadOptions,
+  pipeline: PipelineExecutor
+): Promise<DownloadResource> {
+
+  if (res.type !== ResourceType.Html) {
+    return res;
+  }
+  if (!res.meta.doc) {
+    res.meta.doc = parseHtml(res, options);
+  }
+  const $ = res.meta.doc;
+
+  const metaLinks = $('meta[http-equiv="refresh"][content]');
+  if (metaLinks.length) {
+    const refUrl: string = res.redirectedUrl || res.url;
+    const savePath = refUrl === res.url ? res.savePath : undefined;
+
+    const depth: number = res.depth + 1;
+
+    for (let index = 0; index < metaLinks.length; index++) {
+      const elem = metaLinks.eq(index);
+      const attrValue: string | void = elem.attr('content');
+      if (!attrValue) {
+        continue;
+      }
+      const match = META_REFRESH_PATTERN.exec(attrValue);
+      if (!match) {
+        continue;
+      }
+      const originalLink = match[2] || match[3];
+      if (!originalLink) {
+        continue;
+      }
+      const link: string | void =
+        await pipeline.linkRedirect(originalLink, elem, res);
+      if (!link) {
+        continue;
+      }
+
+      const linkType: ResourceType | void =
+        await pipeline.detectResourceType(link, ResourceType.Html, elem, res);
+      if (!linkType) {
+        if (skip.isTraceEnabled()) {
+          skip.trace('skip detectResourceType',
+            originalLink, link, refUrl);
+        }
+        continue;
+      }
+      let resource: Resource | void = await pipeline.createResource(
+        linkType, depth, link, refUrl,
+        res.localRoot, options.encoding[linkType],
+        savePath, res.type);
+      resource = await pipeline.processBeforeDownload(resource, elem, res, options);
+      if (!resource) {
+        if (skip.isTraceEnabled()) {
+          skip.trace('skip processBeforeDownload',
+            originalLink, link, linkType, refUrl);
+        }
+        continue;
+      }
+      if (!resource.shouldBeDiscardedFromDownload) {
+        submit(resource);
+      }
+      elem.attr('content', attrValue.replace(originalLink, resource.replacePath));
+    }
+  }
+
+  return res;
+}

package/src/sources.ts

@@ -36,6 +36,7 @@ export const sources: SourceDefinition[] = [
   {selector: 'meta[property="og\\:video\\:url"]', attr: 'content'},
   {selector: 'meta[property="og\\:video\\:secure_url"]', attr: 'content'},
   {selector: 'video', attr: 'src'},
+  {selector: 'video', attr: 'poster'},
   {selector: 'video source', attr: 'src'},
   {selector: 'video track', attr: 'src'},
   {selector: 'audio', attr: 'src'},

package/CHANGELOG.md

@@ -1,175 +0,0 @@
-0.7.0
-============
-
-BREAKING
-------------
-* build(deps): bump mkdirp from 2.1.6 to 3.0.0
-* build(deps-dev): bump typescript from 4.9.5 to 5.0.4
-
-0.6.0
-============
-
-BREAKING
-------------
-* resource: custom callback for rewriting savePath
-* life-cycle: custom callback for rewriting savePath (<https://github.com/website-local/website-scrap-engine/issues/383>)
-
-Fix
-------------
-* cheerio: replace deprecated api
-
-Test
-------------
-* test: migrating to eslint v8 and typescript-eslint v5
-* cheerio: fix a test
-* resource: add a test
-* ci: run tests on node.js 18.x (<https://github.com/website-local/website-scrap-engine/issues/610>)
-
-Misc
-------------
-* package-lock-resolved: process registry.npmmirror.com
-* logger: fix type conflict
-* util: fix compatibility with typescript 4.8
-* npm: drop @types/mkdirp
-* update deps
-
-0.5.0
-============
-
-BREAKING
-------------
-* typescript 4.4 support
-  * WorkerMessage: `error` can be `unknown`
-  * StreamingDownloadErrorHook: `e` can be `unknown`
-* pipeline-executor-impl fix keepSearch param
-* resource: redirectedSavePath not set after redirect
-
-Test
-------------
-* test: adapt for jest 27 and ts-jest 27
-
-0.4.0
-============
-
-BREAKING
-------------
-* worker-pool: load based worker pool (#11)
-* cheerio: adapt for version 1.0.0-rc.10 (#271)
-* test: adapt for URI.js v1.19.7 (#301)
-
-Fix
-------------
-* downloader: correctly transfer resource body
-* correctly convent ArrayBufferView to Buffer
-* worker-pool: fix ready
-
-Enhancement
-------------
-* npm: update
-* life-cycle: add init and dispose life cycle
-* resource: optionally redirected savePath
-* resource: take a log on replacing long search string
-* save-to-disk: optionally use remote date
-* worker-pool: log worker errors
-* worker-pool: custom initializer of worker
-
-Test
-------------
-* worker-pool: basic unit test
-* save-html-to-disk: initial unit tests with mocked fs
-* save-to-disk: refactor tests
-
-0.3.2
-============
-
-* resource: fix redirected path processing (#157)
-* downloader: optional wait for this.init in method onIdle (#152)
-* typescript: prefer type only import
-
-0.3.1
-============
-
-* resource: use correct file scheme for windows (#145)
-
-0.3.0
-============
-
-New Feature
-------------
-* life-cycle: extract and process source maps (#123)
-* adapters: async processHtml
-* life-cycle: add read-or-copy-local-resource
-* resource: support file protocol (#126)
-
-Misc
-------------
-* types: export type CheerioElement
-* resource: optional skip replacePath processing in case of parser error (#107)
-* resource: fix new type of Buffer.from (#116)
-* build(deps): bump cheerio from 1.0.0-rc.3 to 1.0.0-rc.5
-* io: mkdirRetry returns no string
-* life-cycle: add download-streaming-resource to default
-* skip-links: skip unix scheme
-* skip-links: allow file protocol
-* download: skip non-http url
-* (BREAKING) resource: refactor createResource (#139)
-
-0.2.0
-============
-* life-cycle: streaming download and save binary resource to disk
-* build(deps-dev): bump @types/cheerio from 0.22.21 to 0.22.22
-
-0.1.7
-============
-* resource: parse and process standalone svg images
-* save-html-to-disk: keep location hash in redirect placeholder
-* detect-resource-type: export lowerCaseExtension
-* downloader: log downloadLink instead of rawUrl
-* typescript: update to v4.0
-
-0.1.6
-============
-* save-resource-to-disk: compare redirectedUrl with url
-* process-html: submit resources from inline css
-* downloader: correctly use adjustTimer on start
-* downloader: deduplicate on redirectedUrl
-  downloader: do not wait for complete on add
-
-0.1.5
-============
-* downloader: do not wait for complete on add
-* process-html: fix detecting type
-* npm: update p-queue to 6.6.0
-* npm: move copy script to build
-
-0.1.4
-============
-* save-html-to-disk: fix redirect check
-* logger: add logger for skipExternal
-
-0.1.3
-============
-* save-html-to-disk: fix redirect placeholder path
-
-0.1.2
-============
-* adapters: make processRedirectedUrl named function
-* options: move initialUrl and logSubDir to StaticDownloadOptions
-* options: retry on error codes
-* download-resource: manually retry on got internal errors
-* io: refactor mkdirRetry
-* process-html: skip invalid srcset
-
-0.1.1
-============
-* io: remove mkdirRetrySync and update writeFile
-* util: arrayToMap could freeze the object returned if required
-* detect-resource-type: fix url with search and hash
-* options: allow merging got options from StaticDownloadOptions
-* options: add comments
-* life-cycle: convent default life cycle fn to named function
-
-0.1.0
-============
-Initial release.
-