website-scrap-engine 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/downloader/adjust-concurrency.d.ts +2 -1
- package/lib/downloader/adjust-concurrency.d.ts.map +1 -0
- package/lib/downloader/adjust-concurrency.js +4 -8
- package/lib/downloader/adjust-concurrency.js.map +1 -1
- package/lib/downloader/index.d.ts +9 -8
- package/lib/downloader/index.d.ts.map +1 -0
- package/lib/downloader/index.js +8 -40
- package/lib/downloader/index.js.map +1 -1
- package/lib/downloader/main.d.ts +15 -6
- package/lib/downloader/main.d.ts.map +1 -0
- package/lib/downloader/main.js +49 -32
- package/lib/downloader/main.js.map +1 -1
- package/lib/downloader/multi.d.ts +7 -5
- package/lib/downloader/multi.d.ts.map +1 -0
- package/lib/downloader/multi.js +10 -17
- package/lib/downloader/multi.js.map +1 -1
- package/lib/downloader/pipeline-executor-impl.d.ts +8 -7
- package/lib/downloader/pipeline-executor-impl.d.ts.map +1 -0
- package/lib/downloader/pipeline-executor-impl.js +1 -5
- package/lib/downloader/pipeline-executor-impl.js.map +1 -1
- package/lib/downloader/single.d.ts +4 -3
- package/lib/downloader/single.d.ts.map +1 -0
- package/lib/downloader/single.js +7 -11
- package/lib/downloader/single.js.map +1 -1
- package/lib/downloader/types.d.ts +4 -4
- package/lib/downloader/types.d.ts.map +1 -0
- package/lib/downloader/types.js +2 -5
- package/lib/downloader/types.js.map +1 -1
- package/lib/downloader/worker-pool.d.ts +6 -7
- package/lib/downloader/worker-pool.d.ts.map +1 -0
- package/lib/downloader/worker-pool.js +7 -35
- package/lib/downloader/worker-pool.js.map +1 -1
- package/lib/downloader/worker-type.d.ts +4 -3
- package/lib/downloader/worker-type.d.ts.map +1 -0
- package/lib/downloader/worker-type.js +1 -2
- package/lib/downloader/worker.d.ts +1 -0
- package/lib/downloader/worker.d.ts.map +1 -0
- package/lib/downloader/worker.js +52 -27
- package/lib/downloader/worker.js.map +1 -1
- package/lib/index.d.ts +9 -8
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +7 -33
- package/lib/index.js.map +1 -1
- package/lib/io.d.ts +2 -1
- package/lib/io.d.ts.map +1 -0
- package/lib/io.js +17 -25
- package/lib/io.js.map +1 -1
- package/lib/life-cycle/adapters.d.ts +7 -5
- package/lib/life-cycle/adapters.d.ts.map +1 -0
- package/lib/life-cycle/adapters.js +18 -30
- package/lib/life-cycle/adapters.js.map +1 -1
- package/lib/life-cycle/default-life-cycle.d.ts +2 -1
- package/lib/life-cycle/default-life-cycle.d.ts.map +1 -0
- package/lib/life-cycle/default-life-cycle.js +28 -32
- package/lib/life-cycle/default-life-cycle.js.map +1 -1
- package/lib/life-cycle/detect-resource-type.d.ts +2 -1
- package/lib/life-cycle/detect-resource-type.d.ts.map +1 -0
- package/lib/life-cycle/detect-resource-type.js +12 -17
- package/lib/life-cycle/detect-resource-type.js.map +1 -1
- package/lib/life-cycle/download-resource.d.ts +6 -7
- package/lib/life-cycle/download-resource.d.ts.map +1 -0
- package/lib/life-cycle/download-resource.js +23 -52
- package/lib/life-cycle/download-resource.js.map +1 -1
- package/lib/life-cycle/download-streaming-resource.d.ts +6 -5
- package/lib/life-cycle/download-streaming-resource.d.ts.map +1 -0
- package/lib/life-cycle/download-streaming-resource.js +39 -74
- package/lib/life-cycle/download-streaming-resource.js.map +1 -1
- package/lib/life-cycle/index.d.ts +16 -15
- package/lib/life-cycle/index.d.ts.map +1 -0
- package/lib/life-cycle/index.js +14 -59
- package/lib/life-cycle/index.js.map +1 -1
- package/lib/life-cycle/pipeline-executor.d.ts +7 -6
- package/lib/life-cycle/pipeline-executor.d.ts.map +1 -0
- package/lib/life-cycle/pipeline-executor.js +1 -2
- package/lib/life-cycle/process-css.d.ts +5 -4
- package/lib/life-cycle/process-css.d.ts.map +1 -0
- package/lib/life-cycle/process-css.js +10 -18
- package/lib/life-cycle/process-css.js.map +1 -1
- package/lib/life-cycle/process-html-meta.d.ts +4 -3
- package/lib/life-cycle/process-html-meta.d.ts.map +1 -0
- package/lib/life-cycle/process-html-meta.js +11 -15
- package/lib/life-cycle/process-html-meta.js.map +1 -1
- package/lib/life-cycle/process-html.d.ts +4 -3
- package/lib/life-cycle/process-html.d.ts.map +1 -0
- package/lib/life-cycle/process-html.js +27 -31
- package/lib/life-cycle/process-html.js.map +1 -1
- package/lib/life-cycle/process-site-map.d.ts +4 -3
- package/lib/life-cycle/process-site-map.d.ts.map +1 -0
- package/lib/life-cycle/process-site-map.js +7 -11
- package/lib/life-cycle/process-site-map.js.map +1 -1
- package/lib/life-cycle/process-source-map.d.ts +4 -4
- package/lib/life-cycle/process-source-map.d.ts.map +1 -0
- package/lib/life-cycle/process-source-map.js +16 -21
- package/lib/life-cycle/process-source-map.js.map +1 -1
- package/lib/life-cycle/process-svg.d.ts +4 -3
- package/lib/life-cycle/process-svg.d.ts.map +1 -0
- package/lib/life-cycle/process-svg.js +17 -21
- package/lib/life-cycle/process-svg.js.map +1 -1
- package/lib/life-cycle/read-or-copy-local-resource.d.ts +4 -3
- package/lib/life-cycle/read-or-copy-local-resource.d.ts.map +1 -0
- package/lib/life-cycle/read-or-copy-local-resource.js +15 -42
- package/lib/life-cycle/read-or-copy-local-resource.js.map +1 -1
- package/lib/life-cycle/save-html-to-disk.d.ts +6 -4
- package/lib/life-cycle/save-html-to-disk.d.ts.map +1 -0
- package/lib/life-cycle/save-html-to-disk.js +24 -33
- package/lib/life-cycle/save-html-to-disk.js.map +1 -1
- package/lib/life-cycle/save-resource-to-disk.d.ts +4 -3
- package/lib/life-cycle/save-resource-to-disk.d.ts.map +1 -0
- package/lib/life-cycle/save-resource-to-disk.js +10 -17
- package/lib/life-cycle/save-resource-to-disk.js.map +1 -1
- package/lib/life-cycle/skip-links.d.ts +1 -0
- package/lib/life-cycle/skip-links.d.ts.map +1 -0
- package/lib/life-cycle/skip-links.js +6 -10
- package/lib/life-cycle/skip-links.js.map +1 -1
- package/lib/life-cycle/types.d.ts +8 -7
- package/lib/life-cycle/types.d.ts.map +1 -0
- package/lib/life-cycle/types.js +1 -2
- package/lib/logger/config-logger.d.ts +2 -1
- package/lib/logger/config-logger.d.ts.map +1 -0
- package/lib/logger/config-logger.js +4 -30
- package/lib/logger/config-logger.js.map +1 -1
- package/lib/logger/logger-worker.d.ts +3 -2
- package/lib/logger/logger-worker.d.ts.map +1 -0
- package/lib/logger/logger-worker.js +11 -13
- package/lib/logger/logger-worker.js.map +1 -1
- package/lib/logger/logger.d.ts +2 -1
- package/lib/logger/logger.d.ts.map +1 -0
- package/lib/logger/logger.js +15 -17
- package/lib/logger/logger.js.map +1 -1
- package/lib/options.d.ts +8 -8
- package/lib/options.d.ts.map +1 -0
- package/lib/options.js +22 -32
- package/lib/options.js.map +1 -1
- package/lib/resource.d.ts +3 -4
- package/lib/resource.d.ts.map +1 -0
- package/lib/resource.js +34 -70
- package/lib/resource.js.map +1 -1
- package/lib/sources.d.ts +2 -1
- package/lib/sources.d.ts.map +1 -0
- package/lib/sources.js +9 -12
- package/lib/sources.js.map +1 -1
- package/lib/types.d.ts +1 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +1 -2
- package/lib/util.d.ts +4 -3
- package/lib/util.d.ts.map +1 -0
- package/lib/util.js +17 -34
- package/lib/util.js.map +1 -1
- package/package.json +19 -21
- package/src/downloader/adjust-concurrency.ts +2 -2
- package/src/downloader/index.ts +8 -8
- package/src/downloader/main.ts +50 -28
- package/src/downloader/multi.ts +11 -10
- package/src/downloader/pipeline-executor-impl.ts +7 -7
- package/src/downloader/single.ts +9 -6
- package/src/downloader/types.ts +3 -3
- package/src/downloader/worker-pool.ts +9 -9
- package/src/downloader/worker-type.ts +3 -3
- package/src/downloader/worker.ts +51 -29
- package/src/index.ts +8 -8
- package/src/io.ts +6 -6
- package/src/life-cycle/adapters.ts +7 -6
- package/src/life-cycle/css-url-parser.d.ts +1 -1
- package/src/life-cycle/default-life-cycle.ts +15 -15
- package/src/life-cycle/detect-resource-type.ts +2 -2
- package/src/life-cycle/download-resource.ts +18 -20
- package/src/life-cycle/download-streaming-resource.ts +20 -18
- package/src/life-cycle/index.ts +15 -15
- package/src/life-cycle/pipeline-executor.ts +6 -6
- package/src/life-cycle/process-css.ts +6 -5
- package/src/life-cycle/process-html-meta.ts +7 -6
- package/src/life-cycle/process-html.ts +21 -13
- package/src/life-cycle/process-site-map.ts +7 -6
- package/src/life-cycle/process-source-map.ts +5 -4
- package/src/life-cycle/process-svg.ts +10 -9
- package/src/life-cycle/read-or-copy-local-resource.ts +9 -7
- package/src/life-cycle/save-html-to-disk.ts +9 -13
- package/src/life-cycle/save-resource-to-disk.ts +6 -6
- package/src/life-cycle/types.ts +7 -7
- package/src/logger/config-logger.ts +5 -3
- package/src/logger/logger-worker.ts +8 -4
- package/src/logger/logger.ts +6 -4
- package/src/options.ts +15 -19
- package/src/resource.ts +10 -5
- package/src/sources.ts +1 -1
- package/src/util.ts +6 -10
- package/tsconfig.json +6 -2
|
@@ -1,35 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.readOrCopyLocalResource = void 0;
|
|
27
|
-
const path = __importStar(require("path"));
|
|
28
|
-
const fs_1 = require("fs");
|
|
29
|
-
const resource_1 = require("../resource");
|
|
30
|
-
const logger_1 = require("../logger/logger");
|
|
1
|
+
import * as path from 'node:path';
|
|
2
|
+
import { promises } from 'node:fs';
|
|
3
|
+
import { ResourceType } from '../resource.js';
|
|
4
|
+
import { error as errorLogger } from '../logger/logger.js';
|
|
31
5
|
const FILE_PREFIX = 'file://';
|
|
32
|
-
async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
6
|
+
export async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
33
7
|
var _a;
|
|
34
8
|
if (res.body) {
|
|
35
9
|
return res;
|
|
@@ -46,12 +20,12 @@ async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
|
46
20
|
return;
|
|
47
21
|
}
|
|
48
22
|
// index.html handling
|
|
49
|
-
let stats;
|
|
50
|
-
if (res.type ===
|
|
51
|
-
stats = await
|
|
23
|
+
let stats = void 0;
|
|
24
|
+
if (res.type === ResourceType.Html) {
|
|
25
|
+
stats = await promises.stat(fileSrcPath);
|
|
52
26
|
if (stats.isDirectory()) {
|
|
53
27
|
for (const index of ['index.html', 'index.htm']) {
|
|
54
|
-
if (await
|
|
28
|
+
if (await promises.access(fileSrcPath + '/' + index)
|
|
55
29
|
.then(() => true).catch(() => false)) {
|
|
56
30
|
fileSrcPath += '/' + index;
|
|
57
31
|
break;
|
|
@@ -59,18 +33,18 @@ async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
|
59
33
|
}
|
|
60
34
|
}
|
|
61
35
|
}
|
|
62
|
-
if (res.type ===
|
|
36
|
+
if (res.type === ResourceType.StreamingBinary) {
|
|
63
37
|
const fileDestPath = path.join((_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot, res.savePath);
|
|
64
|
-
await
|
|
38
|
+
await promises.copyFile(fileSrcPath, fileDestPath);
|
|
65
39
|
}
|
|
66
40
|
else {
|
|
67
|
-
res.body = await
|
|
41
|
+
res.body = await promises.readFile(fileSrcPath, {
|
|
68
42
|
encoding: res.encoding
|
|
69
43
|
});
|
|
70
44
|
}
|
|
71
45
|
try {
|
|
72
46
|
if (!stats) {
|
|
73
|
-
stats = await
|
|
47
|
+
stats = await promises.stat(fileSrcPath);
|
|
74
48
|
}
|
|
75
49
|
if (stats) {
|
|
76
50
|
res.meta.headers = {
|
|
@@ -80,15 +54,14 @@ async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
|
80
54
|
}
|
|
81
55
|
}
|
|
82
56
|
catch (e) {
|
|
83
|
-
|
|
57
|
+
errorLogger.warn('stat ' + fileSrcPath, e);
|
|
84
58
|
}
|
|
85
59
|
res.finishTimestamp = Date.now();
|
|
86
60
|
res.downloadTime =
|
|
87
61
|
res.finishTimestamp - res.downloadStartTimestamp;
|
|
88
|
-
if (res.type ===
|
|
62
|
+
if (res.type === ResourceType.StreamingBinary) {
|
|
89
63
|
return;
|
|
90
64
|
}
|
|
91
65
|
return res;
|
|
92
66
|
}
|
|
93
|
-
exports.readOrCopyLocalResource = readOrCopyLocalResource;
|
|
94
67
|
//# sourceMappingURL=read-or-copy-local-resource.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read-or-copy-local-resource.js","sourceRoot":"","sources":["../../src/life-cycle/read-or-copy-local-resource.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"read-or-copy-local-resource.js","sourceRoot":"","sources":["../../src/life-cycle/read-or-copy-local-resource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;AAEjC,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG5C,OAAO,EAAC,KAAK,IAAI,WAAW,EAAC,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,GAAG,SAAS,CAAC;AAE9B,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAa,EACb,cAA8B,EAC9B,OAA8B;;IAE9B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,GAAuB,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;QAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;IAClE,CAAC;IACD,IAAI,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IACD,sBAAsB;IACtB,IAAI,KAAK,GAAiB,KAAK,CAAC,CAAC;IACjC,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QACnC,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;gBAChD,IAAI,MAAM,QAAQ,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;qBACjD,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;oBACvC,WAAW,IAAI,GAAG,GAAG,KAAK,CAAC;oBAC3B,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAM,YAAY,CAAC,eAAe,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,MAAM,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACrD,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE;YAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ;SACvB,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG;gBACjB,eAAe,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;gBAC1C,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,WAAW,CAAC,IAAI,CAAC,OAAO,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,GAAG,CAAC,YAAY;QACd,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,sBAAsB,CAAC;IAEnD,IAAI,GAAG,CAAC,IAAI,KAAM,YAAY,CAAC,eAAe,EAAE,CAAC;QAC/C,OAAO;IACT,CAAC;IACD,OAAO,GAAuB,CAAC;AACjC,CAAC"}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import type { DownloadResource } from './types';
|
|
2
|
-
import type { StaticDownloadOptions } from '../options';
|
|
3
|
-
import { ResourceBody, ResourceEncoding
|
|
4
|
-
import
|
|
1
|
+
import type { DownloadResource } from './types.js';
|
|
2
|
+
import type { StaticDownloadOptions } from '../options.js';
|
|
3
|
+
import type { ResourceBody, ResourceEncoding } from '../resource.js';
|
|
4
|
+
import { ResourceType } from '../resource.js';
|
|
5
|
+
import type { PipelineExecutor } from './pipeline-executor.js';
|
|
5
6
|
export declare function getResourceBodyFromHtml(res: DownloadResource & {
|
|
6
7
|
type: ResourceType.Html;
|
|
7
8
|
}, options: StaticDownloadOptions): ResourceBody;
|
|
8
9
|
export declare function redirectHtml(relativePath: string, encoding?: ResourceEncoding): string;
|
|
9
10
|
export declare function saveHtmlToDisk(res: DownloadResource, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource | void>;
|
|
11
|
+
//# sourceMappingURL=save-html-to-disk.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"save-html-to-disk.d.ts","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAC,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AAG3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAgB,uBAAuB,CACrC,GAAG,EAAE,gBAAgB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;CAAE,EACnD,OAAO,EAAE,qBAAqB,GAC7B,YAAY,CAQd;AAED,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAUtF;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAiD9D"}
|
|
@@ -1,15 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
const path_1 = __importDefault(require("path"));
|
|
8
|
-
const urijs_1 = __importDefault(require("urijs"));
|
|
9
|
-
const resource_1 = require("../resource");
|
|
10
|
-
const util_1 = require("../util");
|
|
11
|
-
const io_1 = require("../io");
|
|
12
|
-
function getResourceBodyFromHtml(res, options) {
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import URI from 'urijs';
|
|
3
|
+
import { ResourceType, urlOfSavePath } from '../resource.js';
|
|
4
|
+
import { escapePath } from '../util.js';
|
|
5
|
+
import { writeFile } from '../io.js';
|
|
6
|
+
export function getResourceBodyFromHtml(res, options) {
|
|
13
7
|
if (!res.meta.doc) {
|
|
14
8
|
return res.body;
|
|
15
9
|
}
|
|
@@ -18,8 +12,7 @@ function getResourceBodyFromHtml(res, options) {
|
|
|
18
12
|
}
|
|
19
13
|
return res.meta.doc.html();
|
|
20
14
|
}
|
|
21
|
-
|
|
22
|
-
function redirectHtml(relativePath, encoding) {
|
|
15
|
+
export function redirectHtml(relativePath, encoding) {
|
|
23
16
|
// language=HTML
|
|
24
17
|
return `<html lang="en">
|
|
25
18
|
<head>
|
|
@@ -30,51 +23,49 @@ function redirectHtml(relativePath, encoding) {
|
|
|
30
23
|
</head>
|
|
31
24
|
</html>`;
|
|
32
25
|
}
|
|
33
|
-
|
|
34
|
-
async function saveHtmlToDisk(res, options, pipeline) {
|
|
26
|
+
export async function saveHtmlToDisk(res, options, pipeline) {
|
|
35
27
|
var _a, _b, _c, _d;
|
|
36
|
-
if (res.type !==
|
|
28
|
+
if (res.type !== ResourceType.Html) {
|
|
37
29
|
return res;
|
|
38
30
|
}
|
|
39
31
|
const localRoot = (_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot;
|
|
40
32
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
41
|
-
let mtime;
|
|
33
|
+
let mtime = void 0;
|
|
42
34
|
if (options.preferRemoteLastModifiedTime && ((_c = (_b = res.meta) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['last-modified'])) {
|
|
43
35
|
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']);
|
|
44
36
|
}
|
|
45
37
|
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
|
|
46
38
|
if (res.redirectedSavePath) {
|
|
47
39
|
if (res.redirectedSavePath !== res.savePath) {
|
|
48
|
-
const replaceUri = (
|
|
49
|
-
.relativeTo(
|
|
50
|
-
const relativePath =
|
|
51
|
-
await
|
|
40
|
+
const replaceUri = URI(urlOfSavePath(res.redirectedSavePath))
|
|
41
|
+
.relativeTo(urlOfSavePath(res.savePath));
|
|
42
|
+
const relativePath = escapePath(replaceUri.toString());
|
|
43
|
+
await writeFile(path.join(localRoot, decodeURI(res.savePath)), redirectHtml(relativePath, res.encoding), res.encoding, mtime);
|
|
52
44
|
const body = getResourceBodyFromHtml(res, options);
|
|
53
|
-
await
|
|
45
|
+
await writeFile(path.join(localRoot, decodeURI(res.redirectedSavePath)), body, res.encoding, mtime);
|
|
54
46
|
}
|
|
55
47
|
else {
|
|
56
48
|
const body = getResourceBodyFromHtml(res, options);
|
|
57
|
-
const filePath =
|
|
58
|
-
await
|
|
49
|
+
const filePath = path.join(localRoot, decodeURI(res.savePath));
|
|
50
|
+
await writeFile(filePath, body, res.encoding, mtime);
|
|
59
51
|
}
|
|
60
52
|
return;
|
|
61
53
|
}
|
|
62
|
-
const redirectResource = await pipeline.createResource(
|
|
54
|
+
const redirectResource = await pipeline.createResource(ResourceType.Html, res.depth, res.redirectedUrl, res.url, localRoot, undefined, res.refSavePath);
|
|
63
55
|
if (redirectResource.replacePath) {
|
|
64
|
-
const relativePath =
|
|
56
|
+
const relativePath = escapePath(redirectResource.replacePath);
|
|
65
57
|
const savePath = decodeURI(res.savePath);
|
|
66
|
-
await
|
|
67
|
-
const redirectedResource = await pipeline.createResource(
|
|
58
|
+
await writeFile(path.join(localRoot, savePath), redirectHtml(relativePath, res.encoding), res.encoding, mtime);
|
|
59
|
+
const redirectedResource = await pipeline.createResource(ResourceType.Html, res.depth, res.redirectedUrl, res.refUrl, res.localRoot, res.encoding, undefined, ResourceType.Html);
|
|
68
60
|
const redirectedSavePath = decodeURI(redirectedResource.savePath);
|
|
69
61
|
const body = getResourceBodyFromHtml(res, options);
|
|
70
|
-
await
|
|
62
|
+
await writeFile(path.join(localRoot, redirectedSavePath), body, res.encoding, mtime);
|
|
71
63
|
return;
|
|
72
64
|
}
|
|
73
65
|
}
|
|
74
66
|
const body = getResourceBodyFromHtml(res, options);
|
|
75
|
-
const filePath =
|
|
76
|
-
await
|
|
67
|
+
const filePath = path.join(localRoot, decodeURI(res.savePath));
|
|
68
|
+
await writeFile(filePath, body, res.encoding, mtime);
|
|
77
69
|
return;
|
|
78
70
|
}
|
|
79
|
-
exports.saveHtmlToDisk = saveHtmlToDisk;
|
|
80
71
|
//# sourceMappingURL=save-html-to-disk.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"save-html-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"save-html-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,GAAG,MAAM,OAAO,CAAC;AAIxB,OAAO,EAAC,YAAY,EAAE,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAC,UAAU,EAAC,MAAM,YAAY,CAAC;AACtC,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AAGnC,MAAM,UAAU,uBAAuB,CACrC,GAAmD,EACnD,OAA8B;IAE9B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAClB,OAAO,GAAG,CAAC,IAAI,CAAC;IAClB,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,QAA2B;IAC5E,gBAAgB;IAChB,OAAO;;iBAEQ,QAAQ,IAAI,MAAM;6CACU,YAAY;4BAC7B,YAAY;;;QAGhC,CAAC;AACT,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAqB,EACrB,OAA8B,EAC9B,QAA0B;;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,SAAS,GAAW,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,CAAC;IAC7D,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QACvD,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;qBAC1D,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC3C,MAAM,YAAY,GAAW,UAAU,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC/D,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAC3D,YAAY,CAAC,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACjE,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACjE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,EACrE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACjE,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACvD,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EACtE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,EAAE,SAAS,EAChD,SAAS,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,gBAAgB,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,YAAY,GAAW,UAAU,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACtE,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAC5C,YAAY,CAAC,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjE,MAAM,kBAAkB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EACxE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EACvD,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAClE,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACjE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACrD,OAAO;AACT,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { DownloadResource } from './types';
|
|
2
|
-
import type { StaticDownloadOptions } from '../options';
|
|
3
|
-
import type { PipelineExecutor } from './pipeline-executor';
|
|
1
|
+
import type { DownloadResource } from './types.js';
|
|
2
|
+
import type { StaticDownloadOptions } from '../options.js';
|
|
3
|
+
import type { PipelineExecutor } from './pipeline-executor.js';
|
|
4
4
|
export declare function saveResourceToDisk(res: DownloadResource, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource | void>;
|
|
5
|
+
//# sourceMappingURL=save-resource-to-disk.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"save-resource-to-disk.d.ts","sourceRoot":"","sources":["../../src/life-cycle/save-resource-to-disk.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AAEzD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAqC9D"}
|
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.saveResourceToDisk = void 0;
|
|
7
|
-
const path_1 = __importDefault(require("path"));
|
|
8
|
-
const io_1 = require("../io");
|
|
9
|
-
async function saveResourceToDisk(res, options, pipeline) {
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { writeFile } from '../io.js';
|
|
3
|
+
export async function saveResourceToDisk(res, options, pipeline) {
|
|
10
4
|
var _a, _b, _c, _d;
|
|
11
5
|
const localRoot = (_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot;
|
|
12
6
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
13
|
-
let mtime;
|
|
7
|
+
let mtime = void 0;
|
|
14
8
|
if (options.preferRemoteLastModifiedTime && ((_c = (_b = res.meta) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['last-modified'])) {
|
|
15
9
|
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']);
|
|
16
10
|
}
|
|
@@ -18,26 +12,25 @@ async function saveResourceToDisk(res, options, pipeline) {
|
|
|
18
12
|
if (res.redirectedSavePath) {
|
|
19
13
|
if (res.redirectedSavePath !== res.savePath) {
|
|
20
14
|
const redirectedSavePath = decodeURI(res.redirectedSavePath);
|
|
21
|
-
await
|
|
15
|
+
await writeFile(path.join(localRoot, redirectedSavePath), res.body, res.encoding, mtime);
|
|
22
16
|
}
|
|
23
17
|
const savePath = decodeURI(res.savePath);
|
|
24
|
-
await
|
|
18
|
+
await writeFile(path.join(localRoot, savePath), res.body, res.encoding, mtime);
|
|
25
19
|
return;
|
|
26
20
|
}
|
|
27
21
|
const redirectResource = await pipeline.createResource(res.type, res.depth, res.url, res.redirectedUrl, localRoot, res.encoding, undefined, res.type);
|
|
28
22
|
// maybe we can try module:fs/promises.symlink first
|
|
29
23
|
if (redirectResource.replacePath) {
|
|
30
24
|
const savePath = decodeURI(res.savePath);
|
|
31
|
-
await
|
|
25
|
+
await writeFile(path.join(localRoot, savePath), res.body, res.encoding, mtime);
|
|
32
26
|
const redirectedResource = await pipeline.createResource(res.type, res.depth, res.redirectedUrl, res.refUrl, res.localRoot, res.encoding, undefined, res.type);
|
|
33
27
|
const redirectedSavePath = decodeURI(redirectedResource.savePath);
|
|
34
|
-
await
|
|
28
|
+
await writeFile(path.join(localRoot, redirectedSavePath), res.body, res.encoding, mtime);
|
|
35
29
|
return;
|
|
36
30
|
}
|
|
37
31
|
}
|
|
38
|
-
const filePath =
|
|
39
|
-
await
|
|
32
|
+
const filePath = path.join(localRoot, decodeURI(res.savePath));
|
|
33
|
+
await writeFile(filePath, res.body, res.encoding, mtime);
|
|
40
34
|
return;
|
|
41
35
|
}
|
|
42
|
-
exports.saveResourceToDisk = saveResourceToDisk;
|
|
43
36
|
//# sourceMappingURL=save-resource-to-disk.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"save-resource-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-resource-to-disk.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"save-resource-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-resource-to-disk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AAGnC,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAqB,EACrB,OAA8B,EAC9B,QAA0B;;IAC1B,MAAM,SAAS,GAAW,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,CAAC;IAC7D,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QACvD,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAC5C,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBAC7D,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE,GAAG,CAAC,IAAI,EAChE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/E,OAAO;QACT,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAC7D,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,SAAS,EAChD,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,oDAAoD;QACpD,IAAI,gBAAgB,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/E,MAAM,kBAAkB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAC/D,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EACvD,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAClE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE,GAAG,CAAC,IAAI,EAChE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzD,OAAO;AACT,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skip-links.d.ts","sourceRoot":"","sources":["../../src/life-cycle/skip-links.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,uBAAuB,UAiGnC,CAAC;AAEF,eAAO,MAAM,8BAA8B,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAWhE,CAAC;AAEL;;GAEG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAepD"}
|
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.skipLinks = exports.fastUnProcessableUriSchemesMap = exports.unProcessableUriSchemes = void 0;
|
|
4
1
|
// subset of https://en.wikipedia.org/wiki/List_of_URI_schemes
|
|
5
|
-
|
|
2
|
+
export const unProcessableUriSchemes = [
|
|
6
3
|
// Official IANA-registered schemes
|
|
7
4
|
'about',
|
|
8
5
|
'attachment',
|
|
@@ -100,10 +97,10 @@ exports.unProcessableUriSchemes = [
|
|
|
100
97
|
// Unix sockets is supported by got, but not yet supported here
|
|
101
98
|
'unix'
|
|
102
99
|
];
|
|
103
|
-
|
|
100
|
+
export const fastUnProcessableUriSchemesMap = (() => {
|
|
104
101
|
const map = {};
|
|
105
|
-
for (let i = 0, item, l =
|
|
106
|
-
item =
|
|
102
|
+
for (let i = 0, item, l = unProcessableUriSchemes.length; i < l; i++) {
|
|
103
|
+
item = unProcessableUriSchemes[i];
|
|
107
104
|
if (map[item[0]]) {
|
|
108
105
|
map[item[0]].push(item);
|
|
109
106
|
}
|
|
@@ -116,11 +113,11 @@ exports.fastUnProcessableUriSchemesMap = (() => {
|
|
|
116
113
|
/**
|
|
117
114
|
* Skip unprocessable links
|
|
118
115
|
*/
|
|
119
|
-
function skipLinks(url) {
|
|
116
|
+
export function skipLinks(url) {
|
|
120
117
|
if (url.startsWith('#')) {
|
|
121
118
|
return;
|
|
122
119
|
}
|
|
123
|
-
const unProcessableUriSchemeList =
|
|
120
|
+
const unProcessableUriSchemeList = fastUnProcessableUriSchemesMap[url[0]];
|
|
124
121
|
if (unProcessableUriSchemeList && unProcessableUriSchemeList.length) {
|
|
125
122
|
for (let i = 0, item, il, l = unProcessableUriSchemeList.length; i < l; i++) {
|
|
126
123
|
item = unProcessableUriSchemeList[i];
|
|
@@ -132,5 +129,4 @@ function skipLinks(url) {
|
|
|
132
129
|
}
|
|
133
130
|
return url;
|
|
134
131
|
}
|
|
135
|
-
exports.skipLinks = skipLinks;
|
|
136
132
|
//# sourceMappingURL=skip-links.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skip-links.js","sourceRoot":"","sources":["../../src/life-cycle/skip-links.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"skip-links.js","sourceRoot":"","sources":["../../src/life-cycle/skip-links.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,mCAAmC;IACnC,OAAO;IACP,YAAY;IACZ,MAAM;IACN,KAAK;IACL,QAAQ;IACR,kBAAkB;IAClB,KAAK;IACL,SAAS;IACT,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,SAAS;IACT,MAAM;IACN,YAAY;IACZ,KAAK;IACL,KAAK;IACL,KAAK;IACL,MAAM;IACN,IAAI;IACJ,MAAM;IACN,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,OAAO;IACP,QAAQ;IACR,YAAY;IACZ,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,SAAS;IACT,KAAK;IACL,KAAK;IACL,OAAO;IACP,SAAS;IACT,aAAa;IACb,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,QAAQ;IACR,UAAU;IACV,KAAK;IACL,OAAO;IACP,QAAQ;IACR,KAAK;IACL,UAAU;IACV,KAAK;IACL,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,IAAI;IACJ,SAAS;IACT,MAAM;IACN,OAAO;IACP,KAAK;IACL,MAAM;IACN,OAAO;IACP,KAAK;IACL,KAAK;IACL,OAAO;IACP,MAAM;IACN,SAAS;IACT,KAAK;IACL,OAAO;IACP,KAAK;IACL,KAAK;IACL,KAAK;IACL,QAAQ;IACR,MAAM;IACN,KAAK;IACL,QAAQ;IACR,KAAK;IACL,aAAa;IACb,KAAK;IACL,IAAI;IACJ,KAAK;IACL,KAAK;IACL,oCAAoC;IACpC,OAAO;IACP,KAAK;IACL,YAAY;IACZ,MAAM;IACN,MAAM;IACN,+DAA+D;IAC/D,MAAM;CACP,CAAC;AAEF,MAAM,CAAC,MAAM,8BAA8B,GAA6B,CAAC,GAAG,EAAE;IAC5E,MAAM,GAAG,GAA6B,EAAE,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,uBAAuB,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACrE,IAAI,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC,EAAE,CAAC;AAEL;;GAEG;AACH,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IACD,MAAM,0BAA0B,GAAG,8BAA8B,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,IAAI,0BAA0B,IAAI,0BAA0B,CAAC,MAAM,EAAE,CAAC;QACpE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,GAAG,0BAA0B,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5E,IAAI,GAAG,0BAA0B,CAAC,CAAC,CAAC,CAAC;YACrC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;YACjB,IAAI,GAAG,CAAC,MAAM,GAAG,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC;gBAC/D,OAAO;YACT,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { createResource, GenerateSavePathFn, Resource, ResourceBody, ResourceType } from '../resource';
|
|
3
|
-
import type { StaticDownloadOptions } from '../options';
|
|
4
|
-
import type { PipelineExecutor } from './pipeline-executor';
|
|
5
|
-
import type { Cheerio } from '../types';
|
|
6
|
-
import type { DownloaderWithMeta } from '../downloader/types';
|
|
7
|
-
import type { WorkerInfo } from '../downloader/worker-pool';
|
|
1
|
+
import type { OptionsInit as GotOptions } from 'got';
|
|
2
|
+
import type { createResource, GenerateSavePathFn, Resource, ResourceBody, ResourceType } from '../resource.js';
|
|
3
|
+
import type { StaticDownloadOptions } from '../options.js';
|
|
4
|
+
import type { PipelineExecutor } from './pipeline-executor.js';
|
|
5
|
+
import type { Cheerio } from '../types.js';
|
|
6
|
+
import type { DownloaderWithMeta } from '../downloader/types.js';
|
|
7
|
+
import type { WorkerInfo } from '../downloader/worker-pool.js';
|
|
8
8
|
export type AsyncResult<T> = T | Promise<T>;
|
|
9
9
|
export interface InitLifeCycleFunc {
|
|
10
10
|
/**
|
|
@@ -155,3 +155,4 @@ export interface ProcessingLifeCycle {
|
|
|
155
155
|
saveToDisk: SaveToDiskFunc[];
|
|
156
156
|
dispose: DisposeLifeCycle[];
|
|
157
157
|
}
|
|
158
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/life-cycle/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,IAAI,UAAU,EAAC,MAAM,KAAK,CAAC;AACnD,OAAO,KAAK,EACV,cAAc,EACd,kBAAkB,EAClB,QAAQ,EACR,YAAY,EACZ,YAAY,EACb,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,wBAAwB,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,8BAA8B,CAAC;AAE7D,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAE5C,MAAM,WAAW,iBAAiB;IAChC;;;;;;;;;;;;;;;;OAgBG;IACH,CAAC,QAAQ,EAAE,gBAAgB,EAAE,UAAU,CAAC,EAAE,kBAAkB,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;CAClF;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;;;;OAUG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,EAC7D,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,sBAAsB;IACrC;;;;;;;;;;OAUG;IACH,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EACxD,MAAM,EAAE,QAAQ,GAAG,IAAI,EACvB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;CAChE;AAED,MAAM,WAAW,iCAAiC;IAChD;;;;;;;;;OASG;IACH,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EACtC,MAAM,EAAE,QAAQ,GAAG,IAAI,EACvB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;CAC5D;AAED,MAAM,MAAM,cAAc,GAAG,UAAU,CAAC;AAExC;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;;;;OAQG;IACH,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,cAAc,EAAE,OAAO,EAAE,qBAAqB,EAC7E,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,gBAAgB,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC;CAC/E;AAED,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,CAAC,GAAG,EAAE,QAAQ,GAAG,QAAQ,EAAE,GAAG,IAAI,CAAC;CAEpC;AAED,MAAM,WAAW,gBAAiB,SAAQ,QAAQ;IAChD,IAAI,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,WAAW,gCAAgC;IAC/C;;;;;;;OAOG;IACH,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,EACjD,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,cAAc;IAC7B;;;;;;;OAOG;IACH,CAAC,GAAG,EAAE,gBAAgB,EAAE,OAAO,EAAE,qBAAqB,EACrD,QAAQ,EAAE,gBAAgB,GAAG,WAAW,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;CACpE;AAED,MAAM,WAAW,gBAAgB;IAE/B;;;;;;;;;OASG;IACH,CAAC,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,EAC9B,UAAU,CAAC,EAAE,UAAU,EACvB,cAAc,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,iBAAiB,EAAE,CAAC;IAC1B,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC,kBAAkB,EAAE,sBAAsB,EAAE,CAAC;IAC7C,gBAAgB,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC7C,cAAc,EAAE,OAAO,cAAc,CAAC;IACtC;;OAEG;IACH,qBAAqB,EAAE,iCAAiC,EAAE,CAAC;IAC3D;;OAEG;IACH,QAAQ,EAAE,oBAAoB,EAAE,CAAC;IACjC,oBAAoB,EAAE,gCAAgC,EAAE,CAAC;IACzD,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7B,OAAO,EAAE,gBAAgB,EAAE,CAAC;CAC7B"}
|
package/lib/life-cycle/types.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config-logger.d.ts","sourceRoot":"","sources":["../../src/logger/config-logger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AAKnC,eAAO,MAAM,eAAe,cAAe,MAAM,UAAU,MAAM,KAAG,MAyFhE,CAAC"}
|
|
@@ -1,32 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.configureLogger = void 0;
|
|
27
|
-
const log4js_1 = require("log4js");
|
|
28
|
-
const path = __importStar(require("path"));
|
|
29
|
-
const configureLogger = (localRoot, subDir) => (0, log4js_1.configure)({
|
|
1
|
+
// https://github.com/jestjs/jest/issues/11563
|
|
2
|
+
import log4js from 'log4js';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
export const configureLogger = (localRoot, subDir) => log4js.configure({
|
|
30
5
|
appenders: {
|
|
31
6
|
'retry': {
|
|
32
7
|
type: 'file',
|
|
@@ -114,5 +89,4 @@ const configureLogger = (localRoot, subDir) => (0, log4js_1.configure)({
|
|
|
114
89
|
}
|
|
115
90
|
}
|
|
116
91
|
});
|
|
117
|
-
exports.configureLogger = configureLogger;
|
|
118
92
|
//# sourceMappingURL=config-logger.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config-logger.js","sourceRoot":"","sources":["../../src/logger/config-logger.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"config-logger.js","sourceRoot":"","sources":["../../src/logger/config-logger.ts"],"names":[],"mappings":"AACA,8CAA8C;AAC9C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,SAAiB,EAAE,MAAc,EAAU,EAAE,CAC3E,MAAM,CAAC,SAAS,CAAC;IACf,SAAS,EAAE;QACT,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC;SAC5D;QACD,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC;SAC5D;QACD,OAAO,EAAE;YACP,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,CAAC;SAC5D;QACD,MAAM,EAAE;YACN,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC;SAC3D;QACD,KAAK,EAAE;YACL,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC;SAC1D;QACD,UAAU,EAAE;YACV,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,CAAC;SAC/D;QACD,SAAS,EAAE;YACT,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,CAAC;SAC9D;QACD,UAAU,EAAE;YACV,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,CAAC;SAC/D;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;SACf;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ;SACf;KACF;IAED,UAAU,EAAE;QACV,OAAO,EAAE;YACP,SAAS,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC;YAC9B,KAAK,EAAE,OAAO;SACf;QACD,OAAO,EAAE;YACP,SAAS,EAAE,CAAC,OAAO,CAAC;YACpB,KAAK,EAAE,OAAO;SACf;QACD,OAAO,EAAE;YACP,SAAS,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC;YAC9B,KAAK,EAAE,OAAO;SACf;QACD,MAAM,EAAE;YACN,SAAS,EAAE,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC7B,KAAK,EAAE,OAAO;SACf;QACD,cAAc,EAAE;YACd,SAAS,EAAE,CAAC,MAAM,CAAC;YACnB,KAAK,EAAE,OAAO;SACf;QACD,UAAU,EAAE;YACV,SAAS,EAAE,CAAC,KAAK,CAAC;YAClB,KAAK,EAAE,OAAO;SACf;QACD,UAAU,EAAE;YACV,SAAS,EAAE,CAAC,UAAU,CAAC;YACvB,KAAK,EAAE,OAAO;SACf;QACD,SAAS,EAAE;YACT,SAAS,EAAE,CAAC,SAAS,CAAC;YACtB,KAAK,EAAE,OAAO;SACf;QACD,UAAU,EAAE;YACV,SAAS,EAAE,CAAC,UAAU,CAAC;YACvB,KAAK,EAAE,OAAO;SACf;QACD,mBAAmB,EAAE;YACnB,SAAS,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC;YACjC,KAAK,EAAE,OAAO;SACf;QACD,SAAS,EAAE;YACT,SAAS,EAAE,CAAC,QAAQ,EAAE,UAAU,CAAC;YACjC,KAAK,EAAE,OAAO;SACf;KACF;CACF,CAAC,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { Logger } from 'log4js';
|
|
2
|
-
import type { WorkerLog } from '../downloader/worker-type';
|
|
1
|
+
import type { Logger } from 'log4js';
|
|
2
|
+
import type { WorkerLog } from '../downloader/worker-type.js';
|
|
3
3
|
export declare const logLevels: readonly ["trace", "debug", "info", "warn", "error", "fatal", "mark"];
|
|
4
4
|
export declare function getWorkerLogger(category: WorkerLog['logger']): Logger;
|
|
5
|
+
//# sourceMappingURL=logger-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger-worker.d.ts","sourceRoot":"","sources":["../../src/logger/logger-worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AAInC,OAAO,KAAK,EAAmB,SAAS,EAAC,MAAM,8BAA8B,CAAC;AAK9E,eAAO,MAAM,SAAS,uEAEZ,CAAC;AAEX,wBAAgB,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,GAAG,MAAM,CAiBrE"}
|
|
@@ -1,29 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const
|
|
6
|
-
const
|
|
7
|
-
exports.logLevels = [
|
|
1
|
+
// https://github.com/jestjs/jest/issues/11563
|
|
2
|
+
import log4js from 'log4js';
|
|
3
|
+
import { parentPort } from 'node:worker_threads';
|
|
4
|
+
import { WorkerMessageType } from '../downloader/types.js';
|
|
5
|
+
const getLogger = log4js.getLogger;
|
|
6
|
+
export const logLevels = [
|
|
8
7
|
'trace', 'debug', 'info', 'warn', 'error', 'fatal', 'mark'
|
|
9
8
|
];
|
|
10
|
-
function getWorkerLogger(category) {
|
|
11
|
-
const logger =
|
|
12
|
-
|
|
9
|
+
export function getWorkerLogger(category) {
|
|
10
|
+
const logger = getLogger(category);
|
|
11
|
+
logLevels.forEach((level) => {
|
|
13
12
|
logger[level] = (...content) => {
|
|
14
13
|
const msg = {
|
|
15
14
|
taskId: -1,
|
|
16
|
-
type:
|
|
15
|
+
type: WorkerMessageType.Log,
|
|
17
16
|
body: {
|
|
18
17
|
level,
|
|
19
18
|
logger: category,
|
|
20
19
|
content
|
|
21
20
|
}
|
|
22
21
|
};
|
|
23
|
-
|
|
22
|
+
parentPort === null || parentPort === void 0 ? void 0 : parentPort.postMessage(msg);
|
|
24
23
|
};
|
|
25
24
|
});
|
|
26
25
|
return logger;
|
|
27
26
|
}
|
|
28
|
-
exports.getWorkerLogger = getWorkerLogger;
|
|
29
27
|
//# sourceMappingURL=logger-worker.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger-worker.js","sourceRoot":"","sources":["../../src/logger/logger-worker.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"logger-worker.js","sourceRoot":"","sources":["../../src/logger/logger-worker.ts"],"names":[],"mappings":"AACA,8CAA8C;AAC9C,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAC,UAAU,EAAC,MAAM,qBAAqB,CAAC;AAE/C,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AAEzD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;AAEnC,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;CAClD,CAAC;AAEX,MAAM,UAAU,eAAe,CAAC,QAA6B;IAC3D,MAAM,MAAM,GAAW,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC3C,SAAS,CAAC,OAAO,CAAC,CAAC,KAA+B,EAAE,EAAE;QACpD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAI,GAAG,OAAY,EAAE,EAAE;YACrC,MAAM,GAAG,GAAwB;gBAC/B,MAAM,EAAE,CAAC,CAAC;gBACV,IAAI,EAAE,iBAAiB,CAAC,GAAG;gBAC3B,IAAI,EAAE;oBACJ,KAAK;oBACL,MAAM,EAAE,QAAQ;oBAChB,OAAO;iBACR;aACF,CAAC;YACF,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/lib/logger/logger.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Logger } from 'log4js';
|
|
1
|
+
import type { Logger } from 'log4js';
|
|
2
2
|
export declare const notFound: Logger;
|
|
3
3
|
export declare const retry: Logger;
|
|
4
4
|
export declare const mkdir: Logger;
|
|
@@ -9,3 +9,4 @@ export declare const complete: Logger;
|
|
|
9
9
|
export declare const skip: Logger;
|
|
10
10
|
export declare const skipExternal: Logger;
|
|
11
11
|
export declare const adjustConcurrency: Logger;
|
|
12
|
+
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger/logger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,QAAQ,CAAC;AASnC,eAAO,MAAM,QAAQ,EAAE,MAA8B,CAAC;AACtD,eAAO,MAAM,KAAK,EAAE,MAA2B,CAAC;AAChD,eAAO,MAAM,KAAK,EAAE,MAA2B,CAAC;AAChD,eAAO,MAAM,OAAO,EAAE,MAA6B,CAAC;AACpD,eAAO,MAAM,QAAQ,EAAE,MAA8B,CAAC;AACtD,eAAO,MAAM,KAAK,EAAE,MAA2B,CAAC;AAChD,eAAO,MAAM,QAAQ,EAAE,MAA8B,CAAC;AACtD,eAAO,MAAM,IAAI,EAAE,MAA0B,CAAC;AAC9C,eAAO,MAAM,YAAY,EAAE,MAAkC,CAAC;AAC9D,eAAO,MAAM,iBAAiB,EAAE,MAAuC,CAAC"}
|