website-scrap-engine 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +167 -0
- package/lib/downloader/index.js +5 -1
- package/lib/downloader/index.js.map +1 -1
- package/lib/downloader/main.d.ts +1 -1
- package/lib/downloader/main.js +7 -6
- package/lib/downloader/main.js.map +1 -1
- package/lib/downloader/pipeline-executor-impl.js +4 -4
- package/lib/downloader/pipeline-executor-impl.js.map +1 -1
- package/lib/downloader/types.d.ts +1 -1
- package/lib/downloader/worker-pool.d.ts +1 -0
- package/lib/downloader/worker-pool.js +17 -3
- package/lib/downloader/worker-pool.js.map +1 -1
- package/lib/downloader/worker.js +5 -4
- package/lib/downloader/worker.js.map +1 -1
- package/lib/index.js +5 -1
- package/lib/index.js.map +1 -1
- package/lib/io.js +3 -3
- package/lib/io.js.map +1 -1
- package/lib/life-cycle/adapters.js +10 -8
- package/lib/life-cycle/adapters.js.map +1 -1
- package/lib/life-cycle/detect-resource-type.js +2 -2
- package/lib/life-cycle/detect-resource-type.js.map +1 -1
- package/lib/life-cycle/download-resource.d.ts +6 -1
- package/lib/life-cycle/download-resource.js +27 -13
- package/lib/life-cycle/download-resource.js.map +1 -1
- package/lib/life-cycle/download-streaming-resource.d.ts +1 -1
- package/lib/life-cycle/download-streaming-resource.js +12 -7
- package/lib/life-cycle/download-streaming-resource.js.map +1 -1
- package/lib/life-cycle/index.js +5 -1
- package/lib/life-cycle/index.js.map +1 -1
- package/lib/life-cycle/process-css.js +2 -2
- package/lib/life-cycle/process-css.js.map +1 -1
- package/lib/life-cycle/process-html.js +3 -3
- package/lib/life-cycle/process-html.js.map +1 -1
- package/lib/life-cycle/process-site-map.js +2 -5
- package/lib/life-cycle/process-site-map.js.map +1 -1
- package/lib/life-cycle/process-source-map.js +2 -2
- package/lib/life-cycle/process-source-map.js.map +1 -1
- package/lib/life-cycle/process-svg.js +2 -2
- package/lib/life-cycle/process-svg.js.map +1 -1
- package/lib/life-cycle/read-or-copy-local-resource.js +5 -1
- package/lib/life-cycle/read-or-copy-local-resource.js.map +1 -1
- package/lib/life-cycle/save-html-to-disk.js +10 -10
- package/lib/life-cycle/save-html-to-disk.js.map +1 -1
- package/lib/life-cycle/save-resource-to-disk.js +5 -5
- package/lib/life-cycle/save-resource-to-disk.js.map +1 -1
- package/lib/life-cycle/types.d.ts +4 -3
- package/lib/logger/config-logger.js +6 -2
- package/lib/logger/config-logger.js.map +1 -1
- package/lib/logger/logger-worker.d.ts +1 -1
- package/lib/logger/logger-worker.js +2 -2
- package/lib/logger/logger-worker.js.map +1 -1
- package/lib/options.d.ts +1 -1
- package/lib/options.js +1 -1
- package/lib/options.js.map +1 -1
- package/lib/resource.d.ts +11 -3
- package/lib/resource.js +25 -18
- package/lib/resource.js.map +1 -1
- package/lib/types.d.ts +5 -5
- package/lib/util.d.ts +2 -2
- package/lib/util.js +2 -3
- package/lib/util.js.map +1 -1
- package/package.json +19 -19
- package/src/downloader/main.ts +3 -2
- package/src/downloader/pipeline-executor-impl.ts +3 -2
- package/src/downloader/types.ts +1 -1
- package/src/downloader/worker-pool.ts +10 -2
- package/src/downloader/worker.ts +2 -1
- package/src/io.ts +1 -1
- package/src/life-cycle/adapters.ts +8 -3
- package/src/life-cycle/download-resource.ts +29 -12
- package/src/life-cycle/download-streaming-resource.ts +3 -2
- package/src/life-cycle/process-site-map.ts +2 -2
- package/src/life-cycle/types.ts +2 -0
- package/src/logger/logger-worker.ts +1 -1
- package/src/resource.ts +14 -3
- package/src/types.ts +3 -3
- package/src/util.ts +2 -3
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
0.6.0
|
|
2
|
+
============
|
|
3
|
+
|
|
4
|
+
BREAKING
|
|
5
|
+
------------
|
|
6
|
+
* resource: custom callback for rewriting savePath
|
|
7
|
+
* life-cycle: custom callback for rewriting savePath (<https://github.com/website-local/website-scrap-engine/issues/383>)
|
|
8
|
+
|
|
9
|
+
Fix
|
|
10
|
+
------------
|
|
11
|
+
* cheerio: replace deprecated api
|
|
12
|
+
|
|
13
|
+
Test
|
|
14
|
+
------------
|
|
15
|
+
* test: migrating to eslint v8 and typescript-eslint v5
|
|
16
|
+
* cheerio: fix a test
|
|
17
|
+
* resource: add a test
|
|
18
|
+
* ci: run tests on node.js 18.x (<https://github.com/website-local/website-scrap-engine/issues/610>)
|
|
19
|
+
|
|
20
|
+
Misc
|
|
21
|
+
------------
|
|
22
|
+
* package-lock-resolved: process registry.npmmirror.com
|
|
23
|
+
* logger: fix type conflict
|
|
24
|
+
* util: fix compatibility with typescript 4.8
|
|
25
|
+
* npm: drop @types/mkdirp
|
|
26
|
+
* update deps
|
|
27
|
+
|
|
28
|
+
0.5.0
|
|
29
|
+
============
|
|
30
|
+
|
|
31
|
+
BREAKING
|
|
32
|
+
------------
|
|
33
|
+
* typescript 4.4 support
|
|
34
|
+
* WorkerMessage: `error` can be `unknown`
|
|
35
|
+
* StreamingDownloadErrorHook: `e` can be `unknown`
|
|
36
|
+
* pipeline-executor-impl fix keepSearch param
|
|
37
|
+
* resource: redirectedSavePath not set after redirect
|
|
38
|
+
|
|
39
|
+
Test
|
|
40
|
+
------------
|
|
41
|
+
* test: adapt for jest 27 and ts-jest 27
|
|
42
|
+
|
|
43
|
+
0.4.0
|
|
44
|
+
============
|
|
45
|
+
|
|
46
|
+
BREAKING
|
|
47
|
+
------------
|
|
48
|
+
* worker-pool: load based worker pool (#11)
|
|
49
|
+
* cheerio: adapt for version 1.0.0-rc.10 (#271)
|
|
50
|
+
* test: adapt for URI.js v1.19.7 (#301)
|
|
51
|
+
|
|
52
|
+
Fix
|
|
53
|
+
------------
|
|
54
|
+
* downloader: correctly transfer resource body
|
|
55
|
+
* correctly convent ArrayBufferView to Buffer
|
|
56
|
+
* worker-pool: fix ready
|
|
57
|
+
|
|
58
|
+
Enhancement
|
|
59
|
+
------------
|
|
60
|
+
* npm: update
|
|
61
|
+
* life-cycle: add init and dispose life cycle
|
|
62
|
+
* resource: optionally redirected savePath
|
|
63
|
+
* resource: take a log on replacing long search string
|
|
64
|
+
* save-to-disk: optionally use remote date
|
|
65
|
+
* worker-pool: log worker errors
|
|
66
|
+
* worker-pool: custom initializer of worker
|
|
67
|
+
|
|
68
|
+
Test
|
|
69
|
+
------------
|
|
70
|
+
* worker-pool: basic unit test
|
|
71
|
+
* save-html-to-disk: initial unit tests with mocked fs
|
|
72
|
+
* save-to-disk: refactor tests
|
|
73
|
+
|
|
74
|
+
0.3.2
|
|
75
|
+
============
|
|
76
|
+
|
|
77
|
+
* resource: fix redirected path processing (#157)
|
|
78
|
+
* downloader: optional wait for this.init in method onIdle (#152)
|
|
79
|
+
* typescript: prefer type only import
|
|
80
|
+
|
|
81
|
+
0.3.1
|
|
82
|
+
============
|
|
83
|
+
|
|
84
|
+
* resource: use correct file scheme for windows (#145)
|
|
85
|
+
|
|
86
|
+
0.3.0
|
|
87
|
+
============
|
|
88
|
+
|
|
89
|
+
New Feature
|
|
90
|
+
------------
|
|
91
|
+
* life-cycle: extract and process source maps (#123)
|
|
92
|
+
* adapters: async processHtml
|
|
93
|
+
* life-cycle: add read-or-copy-local-resource
|
|
94
|
+
* resource: support file protocol (#126)
|
|
95
|
+
|
|
96
|
+
Misc
|
|
97
|
+
------------
|
|
98
|
+
* types: export type CheerioElement
|
|
99
|
+
* resource: optional skip replacePath processing in case of parser error (#107)
|
|
100
|
+
* resource: fix new type of Buffer.from (#116)
|
|
101
|
+
* build(deps): bump cheerio from 1.0.0-rc.3 to 1.0.0-rc.5
|
|
102
|
+
* io: mkdirRetry returns no string
|
|
103
|
+
* life-cycle: add download-streaming-resource to default
|
|
104
|
+
* skip-links: skip unix scheme
|
|
105
|
+
* skip-links: allow file protocol
|
|
106
|
+
* download: skip non-http url
|
|
107
|
+
* (BREAKING) resource: refactor createResource (#139)
|
|
108
|
+
|
|
109
|
+
0.2.0
|
|
110
|
+
============
|
|
111
|
+
* life-cycle: streaming download and save binary resource to disk
|
|
112
|
+
* build(deps-dev): bump @types/cheerio from 0.22.21 to 0.22.22
|
|
113
|
+
|
|
114
|
+
0.1.7
|
|
115
|
+
============
|
|
116
|
+
* resource: parse and process standalone svg images
|
|
117
|
+
* save-html-to-disk: keep location hash in redirect placeholder
|
|
118
|
+
* detect-resource-type: export lowerCaseExtension
|
|
119
|
+
* downloader: log downloadLink instead of rawUrl
|
|
120
|
+
* typescript: update to v4.0
|
|
121
|
+
|
|
122
|
+
0.1.6
|
|
123
|
+
============
|
|
124
|
+
* save-resource-to-disk: compare redirectedUrl with url
|
|
125
|
+
* process-html: submit resources from inline css
|
|
126
|
+
* downloader: correctly use adjustTimer on start
|
|
127
|
+
* downloader: deduplicate on redirectedUrl
|
|
128
|
+
downloader: do not wait for complete on add
|
|
129
|
+
|
|
130
|
+
0.1.5
|
|
131
|
+
============
|
|
132
|
+
* downloader: do not wait for complete on add
|
|
133
|
+
* process-html: fix detecting type
|
|
134
|
+
* npm: update p-queue to 6.6.0
|
|
135
|
+
* npm: move copy script to build
|
|
136
|
+
|
|
137
|
+
0.1.4
|
|
138
|
+
============
|
|
139
|
+
* save-html-to-disk: fix redirect check
|
|
140
|
+
* logger: add logger for skipExternal
|
|
141
|
+
|
|
142
|
+
0.1.3
|
|
143
|
+
============
|
|
144
|
+
* save-html-to-disk: fix redirect placeholder path
|
|
145
|
+
|
|
146
|
+
0.1.2
|
|
147
|
+
============
|
|
148
|
+
* adapters: make processRedirectedUrl named function
|
|
149
|
+
* options: move initialUrl and logSubDir to StaticDownloadOptions
|
|
150
|
+
* options: retry on error codes
|
|
151
|
+
* download-resource: manually retry on got internal errors
|
|
152
|
+
* io: refactor mkdirRetry
|
|
153
|
+
* process-html: skip invalid srcset
|
|
154
|
+
|
|
155
|
+
0.1.1
|
|
156
|
+
============
|
|
157
|
+
* io: remove mkdirRetrySync and update writeFile
|
|
158
|
+
* util: arrayToMap could freeze the object returned if required
|
|
159
|
+
* detect-resource-type: fix url with search and hash
|
|
160
|
+
* options: allow merging got options from StaticDownloadOptions
|
|
161
|
+
* options: add comments
|
|
162
|
+
* life-cycle: convent default life cycle fn to named function
|
|
163
|
+
|
|
164
|
+
0.1.0
|
|
165
|
+
============
|
|
166
|
+
Initial release.
|
|
167
|
+
|
package/lib/downloader/index.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/downloader/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/downloader/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2DAA4C;AAApC,4GAAA,MAAM,OAAA;AACd,+BAA0C;AAAlC,0GAAA,kBAAkB,OAAA;AAC1B,iCAA8C;AAAtC,8GAAA,qBAAqB,OAAA;AAC7B,mEAA8D;AAAtD,8HAAA,oBAAoB,OAAA;AAC5B,mCAAgD;AAAxC,gHAAA,sBAAsB,OAAA;AAC9B,iDAAiC;AACjC,6CAAyC;AAAjC,yGAAA,UAAU,OAAA;AAClB,4DAA4C"}
|
package/lib/downloader/main.d.ts
CHANGED
|
@@ -23,7 +23,7 @@ export declare abstract class AbstractDownloader implements DownloaderWithMeta {
|
|
|
23
23
|
protected _addProcessedResource(res: RawResource): boolean | void;
|
|
24
24
|
abstract downloadAndProcessResource(res: RawResource): Promise<boolean | void>;
|
|
25
25
|
addProcessedResource(res: RawResource): boolean | void;
|
|
26
|
-
handleError(err: Error | null, cause: string, resource: RawResource): void;
|
|
26
|
+
handleError(err: Error | unknown | null, cause: string, resource: RawResource): void;
|
|
27
27
|
get downloadedCount(): number;
|
|
28
28
|
start(): void;
|
|
29
29
|
stop(): void;
|
package/lib/downloader/main.js
CHANGED
|
@@ -23,7 +23,7 @@ class AbstractDownloader {
|
|
|
23
23
|
lastPeriodTotalCount: 0
|
|
24
24
|
};
|
|
25
25
|
this.adjustTimer = undefined;
|
|
26
|
-
this.options = options_1.mergeOverrideOptions(util_1.importDefaultFromPath(pathToOptions), overrideOptions);
|
|
26
|
+
this.options = (0, options_1.mergeOverrideOptions)((0, util_1.importDefaultFromPath)(pathToOptions), overrideOptions);
|
|
27
27
|
this.queue = new p_queue_1.default({ concurrency: this.options.concurrency });
|
|
28
28
|
this.pipeline = new pipeline_executor_impl_1.PipelineExecutorImpl(this.options, this.options.req, this.options);
|
|
29
29
|
this.options.configureLogger(this.options.localRoot, this.options.logSubDir || '');
|
|
@@ -63,14 +63,14 @@ class AbstractDownloader {
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
_addProcessedResource(res) {
|
|
66
|
-
var _a
|
|
66
|
+
var _a;
|
|
67
67
|
// noinspection DuplicatedCode
|
|
68
68
|
if (res.depth > this.options.maxDepth) {
|
|
69
69
|
logger_1.skip.info('skipped max depth', res.url, res.refUrl, res.depth);
|
|
70
70
|
return false;
|
|
71
71
|
}
|
|
72
72
|
let url;
|
|
73
|
-
const uri = (((
|
|
73
|
+
const uri = (((_a = res === null || res === void 0 ? void 0 : res.uri) === null || _a === void 0 ? void 0 : _a.clone()) || (0, urijs_1.default)(res.url)).hash('');
|
|
74
74
|
if (this.options.deduplicateStripSearch) {
|
|
75
75
|
url = uri.search('').toString();
|
|
76
76
|
}
|
|
@@ -81,7 +81,7 @@ class AbstractDownloader {
|
|
|
81
81
|
return false;
|
|
82
82
|
}
|
|
83
83
|
this.queuedUrl.add(url);
|
|
84
|
-
const resource = resource_1.normalizeResource(res);
|
|
84
|
+
const resource = (0, resource_1.normalizeResource)(res);
|
|
85
85
|
// cut the call stack
|
|
86
86
|
// noinspection JSIgnoredPromiseFromCall
|
|
87
87
|
this.queue.add(() => new Promise(r => setImmediate(() => r(this.downloadAndProcessResource(resource)))));
|
|
@@ -96,9 +96,10 @@ class AbstractDownloader {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
handleError(err, cause, resource) {
|
|
99
|
-
var _a
|
|
99
|
+
var _a;
|
|
100
|
+
// force cast in case of typescript 4.4
|
|
100
101
|
if (err && err.name === 'HTTPError' &&
|
|
101
|
-
((
|
|
102
|
+
((_a = err === null || err === void 0 ? void 0 : err.response) === null || _a === void 0 ? void 0 : _a.statusCode) === 404) {
|
|
102
103
|
logger_1.notFound.error(resource.url, resource.downloadLink, resource.refUrl);
|
|
103
104
|
}
|
|
104
105
|
else if (err) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/downloader/main.ts"],"names":[],"mappings":";;;;;;AAAA,sDAA6B;AAE7B,kDAAwB;AACxB,wCAIoB;AACpB,0CAKqB;AACrB,6CAAuD;AACvD,kCAA8C;AAE9C,qEAA8D;AAE9D,MAAsB,kBAAkB;IActC,YAA6B,aAAqB,EAChD,eAA4E;QADjD,kBAAa,GAAb,aAAa,CAAQ;QAVzC,kBAAa,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC/C,cAAS,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC3C,SAAI,GAAoB;YAC/B,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,CAAC;YAClB,oBAAoB,EAAE,CAAC;SACxB,CAAC;QACF,gBAAW,GAA0C,SAAS,CAAC;QAI7D,IAAI,CAAC,OAAO,GAAG,8BAAoB,
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/downloader/main.ts"],"names":[],"mappings":";;;;;;AAAA,sDAA6B;AAE7B,kDAAwB;AACxB,wCAIoB;AACpB,0CAKqB;AACrB,6CAAuD;AACvD,kCAA8C;AAE9C,qEAA8D;AAE9D,MAAsB,kBAAkB;IActC,YAA6B,aAAqB,EAChD,eAA4E;QADjD,kBAAa,GAAb,aAAa,CAAQ;QAVzC,kBAAa,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC/C,cAAS,GAAgB,IAAI,GAAG,EAAU,CAAC;QAC3C,SAAI,GAAoB;YAC/B,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,eAAe,EAAE,CAAC;YAClB,oBAAoB,EAAE,CAAC;SACxB,CAAC;QACF,gBAAW,GAA0C,SAAS,CAAC;QAI7D,IAAI,CAAC,OAAO,GAAG,IAAA,8BAAoB,EAAC,IAAA,4BAAqB,EAAC,aAAa,CAAC,EAAE,eAAe,CAAC,CAAC;QAC3F,IAAI,CAAC,KAAK,GAAG,IAAI,iBAAM,CAAC,EAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,EAAC,CAAC,CAAC;QACjE,IAAI,CAAC,QAAQ,GAAG,IAAI,6CAAoB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACvF,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;IAChC,CAAC;IAED,IAAI,WAAW,CAAC,cAAsB;QACpC,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,cAAc,CAAC;IAC1C,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,MAAgB;QACvC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC9C,8BAA8B;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAC7C,IAAI,GAAG,GAAkB,MAAM,CAAC,CAAC,CAAC,CAAC;YACnC,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,MAAM,IAAI,GAAwB,MAAM,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CACtE,GAAG,EAAE,uBAAY,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,CAAC,GAAoB,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,CACzD,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EACjB,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7D,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE;gBACpC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC;aAC9B;SACF;IACH,CAAC;IAES,qBAAqB,CAAC,GAAgB;;QAC9C,8BAA8B;QAC9B,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE;YACrC,aAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/D,OAAO,KAAK,CAAC;SACd;QACD,IAAI,GAAW,CAAC;QAChB,MAAM,GAAG,GAAQ,CAAC,CAAA,MAAC,GAAgB,aAAhB,GAAG,uBAAH,GAAG,CAAe,GAAG,0CAAE,KAAK,EAAE,KAAI,IAAA,eAAG,EAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC5E,IAAI,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE;YACvC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SACjC;aAAM;YACL,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;SACtB;QACD,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YAC3B,OAAO,KAAK,CAAC;SACd;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxB,MAAM,QAAQ,GAAa,IAAA,4BAAiB,EAAC,GAAG,CAAC,CAAC;QAClD,qBAAqB;QACrB,wCAAwC;QACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,YAAY,CAChD,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1D,CAAC;IAID,oBAAoB,CAAC,GAAgB;QACnC,IAAI;YACF,OAAO,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC;SACxC;QAAC,OAAO,CAAC,EAAE;YACV,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,iBAAiB,EAAE,GAAG,CAAC,CAAC;YAC5C,OAAO,KAAK,CAAC;SACd;IACH,CAAC;IAED,WAAW,CAAC,GAA2B,EAAE,KAAa,EAAE,QAAqB;;QAC3E,uCAAuC;QACvC,IAAI,GAAG,IAAK,GAAuB,CAAC,IAAI,KAAK,WAAW;YACtD,CAAA,MAAC,GAAiB,aAAjB,GAAG,uBAAH,GAAG,CAAgB,QAAQ,0CAAE,UAAU,MAAK,GAAG,EAAE;YAClD,iBAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;SACtE;aAAM,IAAI,GAAG,EAAE;YACd,cAAK,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;SAC/E;aAAM;YACL,cAAK,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;SAC1E;IACH,CAAC;IAGD,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;IACjC,CAAC;IAED,KAAK;QACH,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,KAAK,UAAU,EAAE;YAC5D,IAAI,IAAI,CAAC,WAAW,EAAE;gBACpB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;aACjC;YACD,IAAI,CAAC,WAAW,GAAG,WAAW,CAC5B,GAAG,EAAE,eAAC,OAAA,MAAA,MAAA,IAAI,CAAC,OAAO,EAAC,qBAAqB,mDAAG,IAAI,CAAC,CAAA,EAAA,EAChD,IAAI,CAAC,OAAO,CAAC,uBAAuB,IAAI,KAAK,CAAC,CAAC;SAClD;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,IAAI,IAAI,CAAC,WAAW,EAAE;YACpB,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;SACjC;QACD,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACnD,CAAC;CAEF;AA7ID,gDA6IC"}
|
|
@@ -64,8 +64,9 @@ class PipelineExecutorImpl {
|
|
|
64
64
|
localRoot: localRoot !== null && localRoot !== void 0 ? localRoot : this.options.localRoot,
|
|
65
65
|
localSrcRoot: this.options.localSrcRoot,
|
|
66
66
|
encoding: (_a = encoding !== null && encoding !== void 0 ? encoding : this.options.encoding[type]) !== null && _a !== void 0 ? _a : 'utf8',
|
|
67
|
-
keepSearch: this.options.deduplicateStripSearch,
|
|
68
|
-
skipReplacePathError: this.options.skipReplacePathError
|
|
67
|
+
keepSearch: !this.options.deduplicateStripSearch,
|
|
68
|
+
skipReplacePathError: this.options.skipReplacePathError,
|
|
69
|
+
generateSavePathFn: this.lifeCycle.generateSavePath,
|
|
69
70
|
};
|
|
70
71
|
return this.lifeCycle.createResource(arg);
|
|
71
72
|
}
|
|
@@ -84,7 +85,6 @@ class PipelineExecutorImpl {
|
|
|
84
85
|
return processedResource;
|
|
85
86
|
}
|
|
86
87
|
async download(res, requestOptions, options) {
|
|
87
|
-
var _a;
|
|
88
88
|
if (res.shouldBeDiscardedFromDownload) {
|
|
89
89
|
return undefined;
|
|
90
90
|
}
|
|
@@ -101,7 +101,7 @@ class PipelineExecutorImpl {
|
|
|
101
101
|
return undefined;
|
|
102
102
|
}
|
|
103
103
|
// if downloaded, end loop and return
|
|
104
|
-
if (
|
|
104
|
+
if (downloadedResource === null || downloadedResource === void 0 ? void 0 : downloadedResource.body) {
|
|
105
105
|
return downloadedResource;
|
|
106
106
|
}
|
|
107
107
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pipeline-executor-impl.js","sourceRoot":"","sources":["../../src/downloader/pipeline-executor-impl.ts"],"names":[],"mappings":";;;AAmBA;;GAEG;AACH,MAAa,oBAAoB;IAC/B,YAAmB,SAA8B,EAC9B,cAA8B,EAC9B,OAA8B;QAF9B,cAAS,GAAT,SAAS,CAAqB;QAC9B,mBAAc,GAAd,cAAc,CAAgB;QAC9B,YAAO,GAAP,OAAO,CAAuB;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAA0B,EAC1B,UAA+B;QAE/B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI;YAAE,OAAO;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;YACtC,MAAM,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;SAClC;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,MAAc,EACd,WAAyB,EACzB,KAA2B,EAC3B,OAAuB,EACvB,MAAgB;QAEhB,MAAM,GAAG,GAAkB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9E,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,GAAG,CAAC;QAClD,MAAM,QAAQ,GAAG,MAAM,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,EACtE,MAAM,EACN,MAAM,CAAC,SAAS,EAChB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC3B,QAAQ,EACR,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,OAAO,MAAM,IAAI,CAAC,qBAAqB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,GAAW,EACX,OAAuB,EACvB,MAAuB;QAEvB,IAAI,aAAa,GAAkB,GAAG,CAAC;QACvC,KAAK,MAAM,gBAAgB,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC1D,IAAI,CAAC,aAAa;gBAChB,MAAM,gBAAgB,CAAC,aAAuB,EAC5C,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE;gBACvD,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,GAAW,EACX,IAAkB,EAClB,OAAuB,EACvB,MAAuB;QAGvB,IAAI,YAAY,GAAwB,IAAI,CAAC;QAC7C,KAAK,MAAM,sBAAsB,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;YACtE,IAAI,CAAC,YAAY;gBACb,MAAM,sBAAsB,CAAC,GAAG,EAAE,YAA4B,EAC5D,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;oBACrC,SAAS,EAAE;gBACf,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,cAAc,CACZ,IAAkB,EAClB,KAAa,EACb,GAAW,EACX,MAAc,EACd,SAAkB,EAClB,QAA2B,EAC3B,WAAoB,EACpB,OAAsB;;QAEtB,MAAM,GAAG,GAA2B;YAClC,IAAI;YACJ,KAAK;YACL,GAAG;YACH,MAAM;YACN,WAAW;YACX,OAAO;YACP,SAAS,EAAE,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,IAAI,CAAC,OAAO,CAAC,SAAS;YAC9C,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;YACvC,QAAQ,EAAE,MAAA,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,mCAAI,MAAM;YAC3D,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,sBAAsB;
|
|
1
|
+
{"version":3,"file":"pipeline-executor-impl.js","sourceRoot":"","sources":["../../src/downloader/pipeline-executor-impl.ts"],"names":[],"mappings":";;;AAmBA;;GAEG;AACH,MAAa,oBAAoB;IAC/B,YAAmB,SAA8B,EAC9B,cAA8B,EAC9B,OAA8B;QAF9B,cAAS,GAAT,SAAS,CAAqB;QAC9B,mBAAc,GAAd,cAAc,CAAgB;QAC9B,YAAO,GAAP,OAAO,CAAuB;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAA0B,EAC1B,UAA+B;QAE/B,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI;YAAE,OAAO;QACjC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;YACtC,MAAM,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;SAClC;IACH,CAAC;IAED,KAAK,CAAC,wBAAwB,CAC5B,MAAc,EACd,WAAyB,EACzB,KAA2B,EAC3B,OAAuB,EACvB,MAAgB;QAEhB,MAAM,GAAG,GAAkB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9E,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,GAAG,CAAC;QAClD,MAAM,QAAQ,GAAG,MAAM,KAAK,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,EACtE,MAAM,EACN,MAAM,CAAC,SAAS,EAChB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC3B,QAAQ,EACR,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,OAAO,MAAM,IAAI,CAAC,qBAAqB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,GAAW,EACX,OAAuB,EACvB,MAAuB;QAEvB,IAAI,aAAa,GAAkB,GAAG,CAAC;QACvC,KAAK,MAAM,gBAAgB,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE;YAC1D,IAAI,CAAC,aAAa;gBAChB,MAAM,gBAAgB,CAAC,aAAuB,EAC5C,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,KAAK,SAAS,EAAE;gBACvD,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,GAAW,EACX,IAAkB,EAClB,OAAuB,EACvB,MAAuB;QAGvB,IAAI,YAAY,GAAwB,IAAI,CAAC;QAC7C,KAAK,MAAM,sBAAsB,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;YACtE,IAAI,CAAC,YAAY;gBACb,MAAM,sBAAsB,CAAC,GAAG,EAAE,YAA4B,EAC5D,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;oBACrC,SAAS,EAAE;gBACf,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,cAAc,CACZ,IAAkB,EAClB,KAAa,EACb,GAAW,EACX,MAAc,EACd,SAAkB,EAClB,QAA2B,EAC3B,WAAoB,EACpB,OAAsB;;QAEtB,MAAM,GAAG,GAA2B;YAClC,IAAI;YACJ,KAAK;YACL,GAAG;YACH,MAAM;YACN,WAAW;YACX,OAAO;YACP,SAAS,EAAE,SAAS,aAAT,SAAS,cAAT,SAAS,GAAI,IAAI,CAAC,OAAO,CAAC,SAAS;YAC9C,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY;YACvC,QAAQ,EAAE,MAAA,QAAQ,aAAR,QAAQ,cAAR,QAAQ,GAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,mCAAI,MAAM;YAC3D,UAAU,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,sBAAsB;YAChD,oBAAoB,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;YACvD,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB;SACpD,CAAC;QACF,OAAO,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,GAAa,EACb,OAAuB,EACvB,MAAuB,EACvB,OAA+B;QAE/B,IAAI,CAAC,OAAO,EAAE;YACZ,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;SACxB;QACD,IAAI,iBAAiB,GAAoB,GAAG,CAAC;QAC7C,KAAK,MAAM,qBAAqB,IAAI,IAAI,CAAC,SAAS,CAAC,qBAAqB,EAAE;YACxE,IAAI,CAAC,iBAAiB;gBAClB,MAAM,qBAAqB,CAAC,iBAAqC,EAC/D,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;oBAChC,SAAS,EAAE;gBACf,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,GAAa,EACb,cAA+B,EAC/B,OAA+B;QAE/B,IAAI,GAAG,CAAC,6BAA6B,EAAE;YACrC,OAAO,SAAS,CAAC;SAClB;QACD,IAAI,CAAC,cAAc,EAAE;YACnB,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;SACtC;QACD,IAAI,CAAC,OAAO,EAAE;YACZ,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;SACxB;QACD,IAAI,kBAAkB,GAAuC,GAAG,CAAC;QACjE,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE;YAC9C,IAAI,CAAC,kBAAkB,GAAG,MAAM,QAAQ,CACtC,kBAA8B,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;oBAC3D,SAAS,EAAE;gBACf,OAAO,SAAS,CAAC;aAClB;YACD,qCAAqC;YACrC,IAAK,kBAA+B,aAA/B,kBAAkB,uBAAlB,kBAAkB,CAAe,IAAI,EAAE;gBAC1C,OAAO,kBAAsC,CAAC;aAC/C;SACF;QACD,iBAAiB;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,oBAAoB,CACxB,GAAqB,EACrB,MAA0B,EAC1B,OAA+B;QAE/B,IAAI,CAAC,OAAO,EAAE;YACZ,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;SACxB;QACD,IAAI,kBAAkB,GAA4B,GAAG,CAAC;QACtD,KAAK,MAAM,oBAAoB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EAAE;YACtE,IAAI,CAAC,kBAAkB,GAAG,MAAM,oBAAoB,CAClD,kBAAsC,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;oBAC3D,SAAS,EAAE;gBACf,OAAO,SAAS,CAAC;aAClB;SACF;QACD,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,UAAU,CACd,GAAqB,EACrB,OAA+B;QAE/B,IAAI,CAAC,OAAO,EAAE;YACZ,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;SACxB;QACD,IAAI,kBAAkB,GAA4B,GAAG,CAAC;QACtD,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE;YAClD,IAAI,CAAC,kBAAkB,GAAG,MAAM,UAAU,CACxC,kBAAsC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;oBACnD,SAAS,EAAE;gBACf,qBAAqB;gBACrB,OAAO,SAAS,CAAC;aAClB;SACF;QACD,iBAAiB;QACjB,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CACX,QAA0B,EAC1B,UAA8B,EAC9B,UAAuB,EACvB,cAAuB;QAEvB,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO;YAAE,OAAO;QACpC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YAC5C,MAAM,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;SACjE;IACH,CAAC;CAEF;AAjND,oDAiNC"}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
|
@@ -61,8 +65,18 @@ class WorkerPool {
|
|
|
61
65
|
this.takeLog(info, message);
|
|
62
66
|
}
|
|
63
67
|
takeLog(info, message) {
|
|
64
|
-
var _a, _b, _c;
|
|
65
|
-
|
|
68
|
+
var _a, _b, _c, _d, _e;
|
|
69
|
+
if (!(message === null || message === void 0 ? void 0 : message.body)) {
|
|
70
|
+
logger.error.warn('Invalid formatted log', info.id);
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const content = (_a = message === null || message === void 0 ? void 0 : message.body) === null || _a === void 0 ? void 0 : _a.content;
|
|
74
|
+
if (content === null || content === void 0 ? void 0 : content.length) {
|
|
75
|
+
(_c = (_b = logger === null || logger === void 0 ? void 0 : logger[message.body.logger]) === null || _b === void 0 ? void 0 : _b[message.body.level]) === null || _c === void 0 ? void 0 : _c.call(_b, info.id, ...content);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
(_e = (_d = logger === null || logger === void 0 ? void 0 : logger[message.body.logger]) === null || _d === void 0 ? void 0 : _d[message.body.level]) === null || _e === void 0 ? void 0 : _e.call(_d, info.id);
|
|
79
|
+
}
|
|
66
80
|
}
|
|
67
81
|
complete(info, message) {
|
|
68
82
|
--info.load;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,mDAAsC;AACtC,yDAA2C;AAE3C,mCAKiB;AAQjB,MAAa,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAPD,wCAOC;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,uBAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAa,UAAU;IAOrB,YACS,QAAgB,EACvB,YAAoB,EACpB,UAAmC,EAC5B,UAAkB,CAAC,CAAC,EAC3B,UAAyB,oBAAoB;QAJtC,aAAQ,GAAR,QAAQ,CAAQ;QAGhB,YAAO,GAAP,OAAO,CAAa;QAVpB,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YACnD,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAC/B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAC,OAAO,CAAC,CAAC,CAAC,CAAC;SAC1D;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,aAAa,CAAC,IAAgB,EAAE,GAAU;QACxC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,SAAS,CAAC,IAAgB,EAAE,OAAsB;QAChD,IAAI,OAAO,CAAC,IAAI,KAAK,yBAAiB,CAAC,QAAQ,EAAE;YAC/C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;SAC9B;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAA2B,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,CAAC,IAAgB,EAAE,OAAyB;;QACjD,IAAI,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,CAAA,EAAE;YAClB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,OAAO;SACR;QACD,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,0CAAE,OAAO,CAAC;QACvC,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,EAAE;YACnB,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;SAC5E;aAAM;YACL,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,CAAC,CAAC;SAChE;IACH,CAAC;IAED,QAAQ,CAAC,IAAgB,EAAE,OAAsB;QAC/C,EAAE,IAAI,CAAC,IAAI,CAAC;QACZ,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpC,MAAM,OAAO,GAA+B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,UAAU,CACR,QAAW,EACX,YAA+C;QAC/C,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,IAAI,GAA8B;gBACtC,MAAM,EAAE,EAAE,IAAI,CAAC,aAAa;gBAC5B,OAAO;gBACP,MAAM;gBACN,IAAI,EAAE,QAAQ;gBACd,YAAY;aACb,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;YAC7B,OAAO;SACR;QACD,gDAAgD;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAC9B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE;YAClE,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACV,OACE,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;gBAC/C,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EACxC;gBACA,MAAM,IAAI,GACR,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,IAAI,EAAE;oBACT,MAAM;iBACP;gBACD,IAAI;oBACF,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;wBACtB,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;qBAChB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBACtB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAsB,CAAC;iBACzD;gBAAC,OAAO,CAAC,EAAE;oBACV,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;oBACf,SAAS;iBACV;gBACD,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB;YACD,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE;gBAC7B,MAAM;aACP;SACF;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE;YACtC,qCAAqC;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YACxD,qCAAqC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;SAClC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AA9HD,gCA8HC"}
|
package/lib/downloader/worker.js
CHANGED
|
@@ -10,7 +10,7 @@ const pipeline_executor_impl_1 = require("./pipeline-executor-impl");
|
|
|
10
10
|
const { pathToOptions, overrideOptions } = worker_threads_1.workerData;
|
|
11
11
|
const options =
|
|
12
12
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
13
|
-
options_1.mergeOverrideOptions(util_1.importDefaultFromPath(pathToOptions), overrideOptions);
|
|
13
|
+
(0, options_1.mergeOverrideOptions)((0, util_1.importDefaultFromPath)(pathToOptions), overrideOptions);
|
|
14
14
|
const pipeline = new pipeline_executor_impl_1.PipelineExecutorImpl(options, options.req, options);
|
|
15
15
|
options.configureLogger(options.localRoot, options.logSubDir || '');
|
|
16
16
|
const init = pipeline.init(pipeline);
|
|
@@ -21,15 +21,15 @@ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ?
|
|
|
21
21
|
try {
|
|
22
22
|
await init;
|
|
23
23
|
const res = msg.body;
|
|
24
|
-
const downloadResource = resource_1.normalizeResource(res);
|
|
24
|
+
const downloadResource = (0, resource_1.normalizeResource)(res);
|
|
25
25
|
const submit = (resources) => {
|
|
26
26
|
if (Array.isArray(resources)) {
|
|
27
27
|
for (let i = 0; i < resources.length; i++) {
|
|
28
|
-
collectedResource.push(resource_1.prepareResourceForClone(resources[i]));
|
|
28
|
+
collectedResource.push((0, resource_1.prepareResourceForClone)(resources[i]));
|
|
29
29
|
}
|
|
30
30
|
}
|
|
31
31
|
else {
|
|
32
|
-
collectedResource.push(resource_1.prepareResourceForClone(resources));
|
|
32
|
+
collectedResource.push((0, resource_1.prepareResourceForClone)(resources));
|
|
33
33
|
}
|
|
34
34
|
};
|
|
35
35
|
const processedResource = await pipeline.processAfterDownload(downloadResource, submit);
|
|
@@ -45,6 +45,7 @@ worker_threads_1.parentPort === null || worker_threads_1.parentPort === void 0 ?
|
|
|
45
45
|
}
|
|
46
46
|
}
|
|
47
47
|
catch (e) {
|
|
48
|
+
// TODO: handle if object could not be cloned here
|
|
48
49
|
error = e;
|
|
49
50
|
}
|
|
50
51
|
finally {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":";;AAAA,mDAAsD;AACtD,wCAIoB;AAEpB,0CAKqB;AACrB,6CAAsC;AACtC,kCAA8C;AAC9C,mCAAiE;AACjE,qEAA8D;AAK9D,MAAM,EAAC,aAAa,EAAE,eAAe,EAAC,GAGlC,2BAAU,CAAC;AAEf,MAAM,OAAO;AACX,8DAA8D;AAC9D,8BAAoB,
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":";;AAAA,mDAAsD;AACtD,wCAIoB;AAEpB,0CAKqB;AACrB,6CAAsC;AACtC,kCAA8C;AAC9C,mCAAiE;AACjE,qEAA8D;AAK9D,MAAM,EAAC,aAAa,EAAE,eAAe,EAAC,GAGlC,2BAAU,CAAC;AAEf,MAAM,OAAO;AACX,8DAA8D;AAC9D,IAAA,8BAAoB,EAAC,IAAA,4BAAqB,EAAC,aAAa,CAAC,EAAE,eAAe,CAAC,CAAC;AAE9E,MAAM,QAAQ,GACZ,IAAI,6CAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAE1D,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;AAEpE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAErC,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,GAAmC,EAAE,EAAE;IAC/E,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAC5C,IAAI,KAA6B,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI;QACF,MAAM,IAAI,CAAC;QACX,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;QACrB,MAAM,gBAAgB,GAAqB,IAAA,4BAAiB,EAAC,GAAG,CAAqB,CAAC;QACtF,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;gBAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACzC,iBAAiB,CAAC,IAAI,CAAC,IAAA,kCAAuB,EAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC/D;aACF;iBAAM;gBACL,iBAAiB,CAAC,IAAI,CAAC,IAAA,kCAAuB,EAAC,SAAS,CAAC,CAAC,CAAC;aAC5D;QACH,CAAC,CAAC;QACF,MAAM,iBAAiB,GACrB,MAAM,QAAQ,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE;YACtB,aAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;SAClD;aAAM,IAAI,MAAM,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;YACvD,aAAI,CAAC,IAAI,CAAC,+BAA+B,EACvC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;SAClD;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;YACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE;YAC3D,aAAa,GAAG,iBAAiB,CAAC,aAAa,CAAC;SACjD;KACF;IAAC,OAAO,CAAC,EAAE;QACV,kDAAkD;QAClD,KAAK,GAAG,CAAC,CAAC;KACX;YAAS;QACR,MAAM,OAAO,GAA0B;YACrC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,IAAI,EAAE,yBAAiB,CAAC,QAAQ;YAChC,IAAI,EAAE,iBAAiB;YACvB,KAAK;YACL,aAAa;SACd,CAAC;QACF,2BAAU,aAAV,2BAAU,uBAAV,2BAAU,CAAE,WAAW,CAAC,OAAO,CAAC,CAAC;KAClC;AAEH,CAAC,CAAC,CAAC"}
|
package/lib/index.js
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
3
|
if (k2 === undefined) k2 = k;
|
|
4
|
-
Object.
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
5
9
|
}) : (function(o, m, k, k2) {
|
|
6
10
|
if (k2 === undefined) k2 = k;
|
|
7
11
|
o[k2] = m[k];
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,0DAA0C;AAC1C,iEAAiD;AACjD,gEAAgD;AAChD,2CAA2B;AAC3B,qDAAqC;AACrC,uDAAuC;AAEvC,+CAA+B"}
|
package/lib/io.js
CHANGED
|
@@ -13,7 +13,7 @@ const mkdirRetry = async (dir, retry = 3) => {
|
|
|
13
13
|
for (let i = 0; i < retry; i++) {
|
|
14
14
|
error = undefined;
|
|
15
15
|
try {
|
|
16
|
-
await mkdirp_1.default(dir);
|
|
16
|
+
await (0, mkdirp_1.default)(dir);
|
|
17
17
|
}
|
|
18
18
|
catch (e) {
|
|
19
19
|
error = e;
|
|
@@ -34,9 +34,9 @@ const mkdirRetry = async (dir, retry = 3) => {
|
|
|
34
34
|
};
|
|
35
35
|
exports.mkdirRetry = mkdirRetry;
|
|
36
36
|
const writeFile = async (filePath, data, encoding, mtime, atime) => {
|
|
37
|
-
const dir = path_1.dirname(filePath);
|
|
37
|
+
const dir = (0, path_1.dirname)(filePath);
|
|
38
38
|
if (!fs_1.default.existsSync(dir)) {
|
|
39
|
-
await exports.mkdirRetry(dir);
|
|
39
|
+
await (0, exports.mkdirRetry)(dir);
|
|
40
40
|
}
|
|
41
41
|
let fileData;
|
|
42
42
|
let options;
|
package/lib/io.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"io.js","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AAEpB,+BAA6B;AAC7B,oDAA4B;AAE5B,4CAA2E;AAEpE,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAAiB,EAAE;IACxE,IAAI,
|
|
1
|
+
{"version":3,"file":"io.js","sourceRoot":"","sources":["../src/io.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AAEpB,+BAA6B;AAC7B,oDAA4B;AAE5B,4CAA2E;AAEpE,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,KAAK,GAAG,CAAC,EAAiB,EAAE;IACxE,IAAI,KAAqB,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE;QAC9B,KAAK,GAAG,SAAS,CAAC;QAClB,IAAI;YACF,MAAM,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC;SACnB;QAAC,OAAO,CAAC,EAAE;YACV,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,CAAC,GAAG,CAAC,EAAE;gBACT,cAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;aACxD;iBAAM;gBACL,cAAW,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;aACxD;YACD,SAAS;SACV;QACD,KAAK,GAAG,SAAS,CAAC;QAClB,OAAO;KACR;IACD,IAAI,KAAK,EAAE;QACT,MAAM,KAAK,CAAC;KACb;AACH,CAAC,CAAC;AArBW,QAAA,UAAU,cAqBrB;AAEK,MAAM,SAAS,GAAG,KAAK,EAC5B,QAAgB,EAChB,IAAkB,EAClB,QAA0B,EAC1B,KAAqB,EACrB,KAAqB,EACN,EAAE;IACjB,MAAM,GAAG,GAAW,IAAA,cAAO,EAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;QACvB,MAAM,IAAA,kBAAU,EAAC,GAAG,CAAC,CAAC;KACvB;IACD,IAAI,QAA6B,CAAC;IAClC,IAAI,OAAqC,CAAC;IAC1C,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC5B,QAAQ,GAAG,IAAI,CAAC;QAChB,OAAO,GAAG,EAAC,QAAQ,EAAC,CAAC;KACtB;SAAM,IAAI,IAAI,YAAY,WAAW,EAAE;QACtC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;SAAM,IAAI,IAAI,YAAY,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC9D,QAAQ,GAAG,IAAI,CAAC;KACjB;SAAM,IAAI,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACnC,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;KACvE;SAAM;QACL,oBAAoB;QACpB,MAAM,IAAI,SAAS,CAAC,6BAA6B,CAAC,CAAC;KACpD;IACD,IAAI,OAAO,EAAE;QACX,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;KAC1D;SAAM;QACL,MAAM,YAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;KACjD;IACD,qBAAqB;IACrB,IAAI,KAAK,EAAE;QACT,IAAI,CAAC,KAAK,EAAE;YACV,KAAK,GAAG,KAAK,CAAC;SACf;QACD,IAAI;YACF,MAAM,YAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SAClD;QAAC,OAAO,CAAC,EAAE;YACV,cAAW,CAAC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;SACpD;KACF;AACH,CAAC,CAAC;AA1CW,QAAA,SAAS,aA0CpB"}
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
exports.processHtmlAsync = exports.processHtml = exports.parseHtml = exports.processRedirectedUrl = exports.redirectFilter = exports.requestRedirect = exports.preProcess = exports.dropResource = exports.skipProcess = void 0;
|
|
7
|
-
const cheerio_1 =
|
|
4
|
+
const cheerio_1 = require("cheerio");
|
|
8
5
|
const resource_1 = require("../resource");
|
|
9
6
|
const util_1 = require("../util");
|
|
10
7
|
const skipProcess = (fn) => (url, element, parent) => fn(url, element, parent) ? undefined : url;
|
|
@@ -44,6 +41,11 @@ async function processRedirectedUrl(res, submit, options, pipeline) {
|
|
|
44
41
|
const redirectedRes = await pipeline.createAndProcessResource(res.redirectedUrl, res.type, res.depth, null, res);
|
|
45
42
|
if (redirectedRes) {
|
|
46
43
|
res.redirectedUrl = redirectedRes.url;
|
|
44
|
+
// https://github.com/website-local/website-scrap-engine/issues/385
|
|
45
|
+
// 2011/11/15
|
|
46
|
+
if (redirectedRes.savePath) {
|
|
47
|
+
res.redirectedSavePath = redirectedRes.savePath;
|
|
48
|
+
}
|
|
47
49
|
}
|
|
48
50
|
}
|
|
49
51
|
return res;
|
|
@@ -52,15 +54,15 @@ exports.processRedirectedUrl = processRedirectedUrl;
|
|
|
52
54
|
const parseHtml = (res, options) => {
|
|
53
55
|
const encoding = res.encoding || options.encoding[res.type] || 'utf8';
|
|
54
56
|
if (options.cheerioParse) {
|
|
55
|
-
return cheerio_1.
|
|
57
|
+
return (0, cheerio_1.load)((0, util_1.toString)(res.body, encoding), options.cheerioParse);
|
|
56
58
|
}
|
|
57
|
-
return cheerio_1.
|
|
59
|
+
return (0, cheerio_1.load)((0, util_1.toString)(res.body, encoding));
|
|
58
60
|
};
|
|
59
61
|
exports.parseHtml = parseHtml;
|
|
60
62
|
const processHtml = (fn) => (res, submit, options) => {
|
|
61
63
|
if (res.type === resource_1.ResourceType.Html) {
|
|
62
64
|
if (!res.meta.doc) {
|
|
63
|
-
res.meta.doc = exports.parseHtml(res, options);
|
|
65
|
+
res.meta.doc = (0, exports.parseHtml)(res, options);
|
|
64
66
|
}
|
|
65
67
|
res.meta.doc = fn(res.meta.doc, res);
|
|
66
68
|
}
|
|
@@ -70,7 +72,7 @@ exports.processHtml = processHtml;
|
|
|
70
72
|
const processHtmlAsync = (fn) => async (res, submit, options) => {
|
|
71
73
|
if (res.type === resource_1.ResourceType.Html) {
|
|
72
74
|
if (!res.meta.doc) {
|
|
73
|
-
res.meta.doc = exports.parseHtml(res, options);
|
|
75
|
+
res.meta.doc = (0, exports.parseHtml)(res, options);
|
|
74
76
|
}
|
|
75
77
|
res.meta.doc = await fn(res.meta.doc, res);
|
|
76
78
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapters.js","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"adapters.js","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":";;;AAAA,qCAA6B;AAC7B,0CAAqE;AASrE,kCAAiC;AAS1B,MAAM,WAAW,GAAG,CAAC,EAAmB,EAAoB,EAAE,CACnE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;AAD1D,QAAA,WAAW,eAC+C;AAMhE,MAAM,YAAY,GAAG,CAAC,EAAoB,EAAqC,EAAE,CACtF,GAAG,CAAC,EAAE;IACJ,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE;QACX,GAAG,CAAC,6BAA6B,GAAG,IAAI,CAAC;KAC1C;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AANS,QAAA,YAAY,gBAMrB;AAOG,MAAM,UAAU,GAAG,CAAC,EAA0B,EAAqC,EAAE,CAC1F,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;IACvB,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAJS,QAAA,UAAU,cAInB;AAMG,MAAM,eAAe,GAAG,CAAC,EAAuB,EAAqC,EAAE,CAC5F,GAAG,CAAC,EAAE;IACJ,IAAI,GAAG,CAAC,YAAY,EAAE;QACpB,MAAM,YAAY,GAAkB,EAAE,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;QAC3E,IAAI,CAAC,YAAY,EAAE;YACjB,OAAO;SACR;QACD,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;KACjC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAVS,QAAA,eAAe,mBAUxB;AAEG,MAAM,cAAc,GAAG,CAAC,EAAuB,EAAoC,EAAE,CAC1F,GAAG,CAAC,EAAE;IACJ,IAAI,GAAG,CAAC,aAAa,EAAE;QACrB,GAAG,CAAC,aAAa,GAAG,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;KAC7D;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AANS,QAAA,cAAc,kBAMvB;AAEG,KAAK,UAAU,oBAAoB,CACxC,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAE1B,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE;QACtD,MAAM,aAAa,GAAoB,MAAM,QAAQ,CAAC,wBAAwB,CAC5E,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE;YACjB,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC;YACtC,mEAAmE;YACnE,aAAa;YACb,IAAI,aAAa,CAAC,QAAQ,EAAE;gBAC1B,GAAG,CAAC,kBAAkB,GAAG,aAAa,CAAC,QAAQ,CAAC;aACjD;SACF;KACF;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAnBD,oDAmBC;AAMM,MAAM,SAAS,GAAG,CACvB,GAAmD,EACnD,OAA8B,EACf,EAAE;IACjB,MAAM,QAAQ,GACZ,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC;IACvD,IAAI,OAAO,CAAC,YAAY,EAAE;QACxB,OAAO,IAAA,cAAI,EAAC,IAAA,eAAQ,EAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;KACjE;IACD,OAAO,IAAA,cAAI,EAAC,IAAA,eAAQ,EAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAVW,QAAA,SAAS,aAUpB;AAEK,MAAM,WAAW,GAAG,CAAC,EAAmB,EAAoC,EAAE,CACnF,CAAC,GAAqB,EAAE,MAA0B,EAAE,OAA8B,EAAE,EAAE;IACpF,IAAI,GAAG,CAAC,IAAI,KAAK,uBAAY,CAAC,IAAI,EAAE;QAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE;YACjB,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,IAAA,iBAAS,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SACxC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;KACtC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AATS,QAAA,WAAW,eASpB;AAOG,MAAM,gBAAgB,GAAG,CAAC,EAAwB,EAAoC,EAAE,CAC7F,KAAK,EAAE,GAAqB,EAAE,MAA0B,EAAE,OAA8B,EAAE,EAAE;IAC1F,IAAI,GAAG,CAAC,IAAI,KAAK,uBAAY,CAAC,IAAI,EAAE;QAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE;YACjB,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,IAAA,iBAAS,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SACxC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;KAC5C;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AATS,QAAA,gBAAgB,oBASzB"}
|
|
@@ -4,7 +4,7 @@ exports.detectResourceType = exports.lowerCaseExtension = exports.binaryExtensio
|
|
|
4
4
|
const resource_1 = require("../resource");
|
|
5
5
|
const util_1 = require("../util");
|
|
6
6
|
// immutable
|
|
7
|
-
exports.binaryExtension = util_1.arrayToMap([
|
|
7
|
+
exports.binaryExtension = (0, util_1.arrayToMap)([
|
|
8
8
|
'gif', 'jpg', 'jpeg', 'png',
|
|
9
9
|
'js', 'jsm', 'json', 'txt',
|
|
10
10
|
'woff2', 'ttf', 'ttc',
|
|
@@ -51,7 +51,7 @@ function lowerCaseExtension(url) {
|
|
|
51
51
|
}
|
|
52
52
|
exports.lowerCaseExtension = lowerCaseExtension;
|
|
53
53
|
function detectResourceType(url, type) {
|
|
54
|
-
if (util_1.isSiteMap(url)) {
|
|
54
|
+
if ((0, util_1.isSiteMap)(url)) {
|
|
55
55
|
return resource_1.ResourceType.SiteMap;
|
|
56
56
|
}
|
|
57
57
|
if (type === resource_1.ResourceType.Html) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"detect-resource-type.js","sourceRoot":"","sources":["../../src/life-cycle/detect-resource-type.ts"],"names":[],"mappings":";;;AAAA,0CAAyC;AACzC,kCAA8C;AAE9C,YAAY;AACC,QAAA,eAAe,GAAG,iBAAU,
|
|
1
|
+
{"version":3,"file":"detect-resource-type.js","sourceRoot":"","sources":["../../src/life-cycle/detect-resource-type.ts"],"names":[],"mappings":";;;AAAA,0CAAyC;AACzC,kCAA8C;AAE9C,YAAY;AACC,QAAA,eAAe,GAAG,IAAA,iBAAU,EAAC;IACxC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;IAC3B,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK;IAC1B,OAAO,EAAE,KAAK,EAAE,KAAK;IACrB,KAAK;IACL,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI;IACnD,KAAK,EAAE,KAAK;IACZ,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;IAClC,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;IACL,KAAK;CACN,EAAE,IAAI,CAAC,CAAC;AAET;;;;;;;;GAQG;AACH,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,MAAM,SAAS,GAAW,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,WAAW,GAAW,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QACtB,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAW,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3C,SAAS,CAAC,CAAC;QACX,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACpE,MAAM,OAAO,GAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QACtB,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACjC,MAAM,SAAS,GAAW,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACzC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;QACtB,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACjC,IAAI,SAAS,KAAK,CAAC,CAAC,IAAI,SAAS,GAAG,OAAO,EAAE;QAC3C,OAAQ,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACxC,GAAG,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;KACpD;AACH,CAAC;AAnBD,gDAmBC;AAED,SAAgB,kBAAkB,CAChC,GAAW,EACX,IAAkB;IAElB,IAAI,IAAA,gBAAS,EAAC,GAAG,CAAC,EAAE;QAClB,OAAO,uBAAY,CAAC,OAAO,CAAC;KAC7B;IACD,IAAI,IAAI,KAAK,uBAAY,CAAC,IAAI,EAAE;QAC9B,MAAM,SAAS,GAAkB,kBAAkB,CAAC,GAAG,CAAC,CAAC;QACzD,IAAI,SAAS,EAAE;YACb,IAAI,uBAAe,CAAC,SAAS,CAAC,EAAE;gBAC9B,OAAO,uBAAY,CAAC,MAAM,CAAC;aAC5B;iBAAM,IAAI,KAAK,KAAK,SAAS,EAAE;gBAC9B,OAAO,uBAAY,CAAC,GAAG,CAAC;aACzB;iBAAM,IAAI,KAAK,KAAK,SAAS,EAAE;gBAC9B,OAAO,uBAAY,CAAC,GAAG,CAAC;aACzB;SACF;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AApBD,gDAoBC"}
|
|
@@ -6,6 +6,11 @@ import { Resource } from '../resource';
|
|
|
6
6
|
import type { StaticDownloadOptions } from '../options';
|
|
7
7
|
/** Take logs before retry */
|
|
8
8
|
export declare const beforeRetryHook: BeforeRetryHook;
|
|
9
|
+
export interface DownloadError extends Partial<Error> {
|
|
10
|
+
retryLimitExceeded?: boolean;
|
|
11
|
+
code?: string;
|
|
12
|
+
event?: string;
|
|
13
|
+
}
|
|
9
14
|
/**
|
|
10
15
|
* workaround for retry premature close on node 12
|
|
11
16
|
* retry on empty body
|
|
@@ -16,5 +21,5 @@ export declare const beforeRetryHook: BeforeRetryHook;
|
|
|
16
21
|
export declare function getRetry(url: string, options: Options): Promise<Response<Buffer | string> | void>;
|
|
17
22
|
export declare function requestForResource(res: Resource & {
|
|
18
23
|
downloadStartTimestamp: number;
|
|
19
|
-
}, requestOptions: RequestOptions): Promise<DownloadResource | Resource | void>;
|
|
24
|
+
}, requestOptions: RequestOptions, options?: StaticDownloadOptions): Promise<DownloadResource | Resource | void>;
|
|
20
25
|
export declare function downloadResource(res: Resource, requestOptions: RequestOptions, options: StaticDownloadOptions): Promise<DownloadResource | Resource | void>;
|