website-scrap-engine 0.7.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/downloader/adjust-concurrency.d.ts +2 -1
- package/lib/downloader/adjust-concurrency.d.ts.map +1 -0
- package/lib/downloader/adjust-concurrency.js +4 -8
- package/lib/downloader/adjust-concurrency.js.map +1 -1
- package/lib/downloader/index.d.ts +9 -8
- package/lib/downloader/index.d.ts.map +1 -0
- package/lib/downloader/index.js +8 -40
- package/lib/downloader/index.js.map +1 -1
- package/lib/downloader/main.d.ts +15 -6
- package/lib/downloader/main.d.ts.map +1 -0
- package/lib/downloader/main.js +49 -32
- package/lib/downloader/main.js.map +1 -1
- package/lib/downloader/multi.d.ts +7 -5
- package/lib/downloader/multi.d.ts.map +1 -0
- package/lib/downloader/multi.js +10 -17
- package/lib/downloader/multi.js.map +1 -1
- package/lib/downloader/pipeline-executor-impl.d.ts +8 -7
- package/lib/downloader/pipeline-executor-impl.d.ts.map +1 -0
- package/lib/downloader/pipeline-executor-impl.js +1 -5
- package/lib/downloader/pipeline-executor-impl.js.map +1 -1
- package/lib/downloader/single.d.ts +4 -3
- package/lib/downloader/single.d.ts.map +1 -0
- package/lib/downloader/single.js +7 -11
- package/lib/downloader/single.js.map +1 -1
- package/lib/downloader/types.d.ts +4 -4
- package/lib/downloader/types.d.ts.map +1 -0
- package/lib/downloader/types.js +2 -5
- package/lib/downloader/types.js.map +1 -1
- package/lib/downloader/worker-pool.d.ts +6 -7
- package/lib/downloader/worker-pool.d.ts.map +1 -0
- package/lib/downloader/worker-pool.js +7 -35
- package/lib/downloader/worker-pool.js.map +1 -1
- package/lib/downloader/worker-type.d.ts +4 -3
- package/lib/downloader/worker-type.d.ts.map +1 -0
- package/lib/downloader/worker-type.js +1 -2
- package/lib/downloader/worker.d.ts +1 -0
- package/lib/downloader/worker.d.ts.map +1 -0
- package/lib/downloader/worker.js +52 -27
- package/lib/downloader/worker.js.map +1 -1
- package/lib/index.d.ts +9 -8
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +7 -33
- package/lib/index.js.map +1 -1
- package/lib/io.d.ts +2 -1
- package/lib/io.d.ts.map +1 -0
- package/lib/io.js +17 -25
- package/lib/io.js.map +1 -1
- package/lib/life-cycle/adapters.d.ts +7 -5
- package/lib/life-cycle/adapters.d.ts.map +1 -0
- package/lib/life-cycle/adapters.js +18 -30
- package/lib/life-cycle/adapters.js.map +1 -1
- package/lib/life-cycle/default-life-cycle.d.ts +2 -1
- package/lib/life-cycle/default-life-cycle.d.ts.map +1 -0
- package/lib/life-cycle/default-life-cycle.js +28 -32
- package/lib/life-cycle/default-life-cycle.js.map +1 -1
- package/lib/life-cycle/detect-resource-type.d.ts +2 -1
- package/lib/life-cycle/detect-resource-type.d.ts.map +1 -0
- package/lib/life-cycle/detect-resource-type.js +12 -17
- package/lib/life-cycle/detect-resource-type.js.map +1 -1
- package/lib/life-cycle/download-resource.d.ts +6 -7
- package/lib/life-cycle/download-resource.d.ts.map +1 -0
- package/lib/life-cycle/download-resource.js +23 -52
- package/lib/life-cycle/download-resource.js.map +1 -1
- package/lib/life-cycle/download-streaming-resource.d.ts +6 -5
- package/lib/life-cycle/download-streaming-resource.d.ts.map +1 -0
- package/lib/life-cycle/download-streaming-resource.js +39 -74
- package/lib/life-cycle/download-streaming-resource.js.map +1 -1
- package/lib/life-cycle/index.d.ts +16 -15
- package/lib/life-cycle/index.d.ts.map +1 -0
- package/lib/life-cycle/index.js +14 -59
- package/lib/life-cycle/index.js.map +1 -1
- package/lib/life-cycle/pipeline-executor.d.ts +7 -6
- package/lib/life-cycle/pipeline-executor.d.ts.map +1 -0
- package/lib/life-cycle/pipeline-executor.js +1 -2
- package/lib/life-cycle/process-css.d.ts +5 -4
- package/lib/life-cycle/process-css.d.ts.map +1 -0
- package/lib/life-cycle/process-css.js +10 -18
- package/lib/life-cycle/process-css.js.map +1 -1
- package/lib/life-cycle/process-html-meta.d.ts +4 -3
- package/lib/life-cycle/process-html-meta.d.ts.map +1 -0
- package/lib/life-cycle/process-html-meta.js +11 -15
- package/lib/life-cycle/process-html-meta.js.map +1 -1
- package/lib/life-cycle/process-html.d.ts +4 -3
- package/lib/life-cycle/process-html.d.ts.map +1 -0
- package/lib/life-cycle/process-html.js +27 -31
- package/lib/life-cycle/process-html.js.map +1 -1
- package/lib/life-cycle/process-site-map.d.ts +4 -3
- package/lib/life-cycle/process-site-map.d.ts.map +1 -0
- package/lib/life-cycle/process-site-map.js +7 -11
- package/lib/life-cycle/process-site-map.js.map +1 -1
- package/lib/life-cycle/process-source-map.d.ts +4 -4
- package/lib/life-cycle/process-source-map.d.ts.map +1 -0
- package/lib/life-cycle/process-source-map.js +16 -21
- package/lib/life-cycle/process-source-map.js.map +1 -1
- package/lib/life-cycle/process-svg.d.ts +4 -3
- package/lib/life-cycle/process-svg.d.ts.map +1 -0
- package/lib/life-cycle/process-svg.js +17 -21
- package/lib/life-cycle/process-svg.js.map +1 -1
- package/lib/life-cycle/read-or-copy-local-resource.d.ts +4 -3
- package/lib/life-cycle/read-or-copy-local-resource.d.ts.map +1 -0
- package/lib/life-cycle/read-or-copy-local-resource.js +15 -42
- package/lib/life-cycle/read-or-copy-local-resource.js.map +1 -1
- package/lib/life-cycle/save-html-to-disk.d.ts +6 -4
- package/lib/life-cycle/save-html-to-disk.d.ts.map +1 -0
- package/lib/life-cycle/save-html-to-disk.js +24 -33
- package/lib/life-cycle/save-html-to-disk.js.map +1 -1
- package/lib/life-cycle/save-resource-to-disk.d.ts +4 -3
- package/lib/life-cycle/save-resource-to-disk.d.ts.map +1 -0
- package/lib/life-cycle/save-resource-to-disk.js +10 -17
- package/lib/life-cycle/save-resource-to-disk.js.map +1 -1
- package/lib/life-cycle/skip-links.d.ts +1 -0
- package/lib/life-cycle/skip-links.d.ts.map +1 -0
- package/lib/life-cycle/skip-links.js +6 -10
- package/lib/life-cycle/skip-links.js.map +1 -1
- package/lib/life-cycle/types.d.ts +8 -7
- package/lib/life-cycle/types.d.ts.map +1 -0
- package/lib/life-cycle/types.js +1 -2
- package/lib/logger/config-logger.d.ts +2 -1
- package/lib/logger/config-logger.d.ts.map +1 -0
- package/lib/logger/config-logger.js +4 -30
- package/lib/logger/config-logger.js.map +1 -1
- package/lib/logger/logger-worker.d.ts +3 -2
- package/lib/logger/logger-worker.d.ts.map +1 -0
- package/lib/logger/logger-worker.js +11 -13
- package/lib/logger/logger-worker.js.map +1 -1
- package/lib/logger/logger.d.ts +2 -1
- package/lib/logger/logger.d.ts.map +1 -0
- package/lib/logger/logger.js +15 -17
- package/lib/logger/logger.js.map +1 -1
- package/lib/options.d.ts +8 -8
- package/lib/options.d.ts.map +1 -0
- package/lib/options.js +22 -32
- package/lib/options.js.map +1 -1
- package/lib/resource.d.ts +3 -4
- package/lib/resource.d.ts.map +1 -0
- package/lib/resource.js +34 -70
- package/lib/resource.js.map +1 -1
- package/lib/sources.d.ts +2 -1
- package/lib/sources.d.ts.map +1 -0
- package/lib/sources.js +9 -12
- package/lib/sources.js.map +1 -1
- package/lib/types.d.ts +1 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +1 -2
- package/lib/util.d.ts +4 -3
- package/lib/util.d.ts.map +1 -0
- package/lib/util.js +17 -34
- package/lib/util.js.map +1 -1
- package/package.json +19 -21
- package/src/downloader/adjust-concurrency.ts +2 -2
- package/src/downloader/index.ts +8 -8
- package/src/downloader/main.ts +50 -28
- package/src/downloader/multi.ts +11 -10
- package/src/downloader/pipeline-executor-impl.ts +7 -7
- package/src/downloader/single.ts +9 -6
- package/src/downloader/types.ts +3 -3
- package/src/downloader/worker-pool.ts +9 -9
- package/src/downloader/worker-type.ts +3 -3
- package/src/downloader/worker.ts +51 -29
- package/src/index.ts +8 -8
- package/src/io.ts +6 -6
- package/src/life-cycle/adapters.ts +7 -6
- package/src/life-cycle/css-url-parser.d.ts +1 -1
- package/src/life-cycle/default-life-cycle.ts +15 -15
- package/src/life-cycle/detect-resource-type.ts +2 -2
- package/src/life-cycle/download-resource.ts +18 -20
- package/src/life-cycle/download-streaming-resource.ts +20 -18
- package/src/life-cycle/index.ts +15 -15
- package/src/life-cycle/pipeline-executor.ts +6 -6
- package/src/life-cycle/process-css.ts +6 -5
- package/src/life-cycle/process-html-meta.ts +7 -6
- package/src/life-cycle/process-html.ts +21 -13
- package/src/life-cycle/process-site-map.ts +7 -6
- package/src/life-cycle/process-source-map.ts +5 -4
- package/src/life-cycle/process-svg.ts +10 -9
- package/src/life-cycle/read-or-copy-local-resource.ts +9 -7
- package/src/life-cycle/save-html-to-disk.ts +9 -13
- package/src/life-cycle/save-resource-to-disk.ts +6 -6
- package/src/life-cycle/types.ts +7 -7
- package/src/logger/config-logger.ts +5 -3
- package/src/logger/logger-worker.ts +8 -4
- package/src/logger/logger.ts +6 -4
- package/src/options.ts +15 -19
- package/src/resource.ts +10 -5
- package/src/sources.ts +1 -1
- package/src/util.ts +6 -10
- package/tsconfig.json +6 -2
|
@@ -1,43 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.downloadStreamingResourceWithHook = exports.downloadStreamingResource = exports.optionallySetLastModifiedTime = exports.streamingDownloadToFile = exports.isSameRangeStart = exports.isBytesAccepted = void 0;
|
|
30
|
-
const got_1 = __importStar(require("got"));
|
|
31
|
-
const path_1 = __importDefault(require("path"));
|
|
32
|
-
const fs_1 = require("fs");
|
|
33
|
-
const resource_1 = require("../resource");
|
|
34
|
-
const io_1 = require("../io");
|
|
35
|
-
const stream_1 = require("stream");
|
|
36
|
-
const util_1 = require("util");
|
|
37
|
-
const logger_1 = require("../logger/logger");
|
|
38
|
-
const util_2 = require("../util");
|
|
39
|
-
const promisifyPipeline = (0, util_1.promisify)(stream_1.pipeline);
|
|
40
|
-
function isBytesAccepted(acceptRange) {
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { constants, createWriteStream, promises as fs } from 'node:fs';
|
|
3
|
+
import { pipeline } from 'node:stream';
|
|
4
|
+
import { promisify } from 'node:util';
|
|
5
|
+
import got, { HTTPError } from 'got';
|
|
6
|
+
import { ResourceType } from '../resource.js';
|
|
7
|
+
import { mkdirRetry } from '../io.js';
|
|
8
|
+
import { error as errorLogger, retry as retryLogger } from '../logger/logger.js';
|
|
9
|
+
import { isUrlHttp } from '../util.js';
|
|
10
|
+
const promisifyPipeline = promisify(pipeline);
|
|
11
|
+
export function isBytesAccepted(acceptRange) {
|
|
41
12
|
if (!acceptRange) {
|
|
42
13
|
return false;
|
|
43
14
|
}
|
|
@@ -49,8 +20,7 @@ function isBytesAccepted(acceptRange) {
|
|
|
49
20
|
}
|
|
50
21
|
return false;
|
|
51
22
|
}
|
|
52
|
-
|
|
53
|
-
function isSameRangeStart(rangeStart, contentRange) {
|
|
23
|
+
export function isSameRangeStart(rangeStart, contentRange) {
|
|
54
24
|
if (!contentRange) {
|
|
55
25
|
return false;
|
|
56
26
|
}
|
|
@@ -62,16 +32,15 @@ function isSameRangeStart(rangeStart, contentRange) {
|
|
|
62
32
|
ranges = ranges[1].split('-');
|
|
63
33
|
return +ranges[0] === rangeStart;
|
|
64
34
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const savePath = path_1.default.join(res.localRoot, decodeURI(res.savePath));
|
|
35
|
+
export async function streamingDownloadToFile(res, requestOptions) {
|
|
36
|
+
const savePath = path.join(res.localRoot, decodeURI(res.savePath));
|
|
68
37
|
try {
|
|
69
|
-
await
|
|
38
|
+
await fs.access(savePath, constants.W_OK);
|
|
70
39
|
}
|
|
71
40
|
catch (e) {
|
|
72
41
|
// force cast for typescript 4.4
|
|
73
42
|
if (e && e.code === 'ENOENT') {
|
|
74
|
-
await
|
|
43
|
+
await mkdirRetry(path.dirname(savePath));
|
|
75
44
|
}
|
|
76
45
|
else {
|
|
77
46
|
throw e;
|
|
@@ -96,12 +65,12 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
96
65
|
options.headers = {};
|
|
97
66
|
}
|
|
98
67
|
options.headers.range = `bytes=${rangeStart}-`;
|
|
99
|
-
fileWriteStream =
|
|
68
|
+
fileWriteStream = createWriteStream(savePath, {
|
|
100
69
|
flags: 'a',
|
|
101
70
|
start: rangeStart
|
|
102
71
|
});
|
|
103
72
|
}
|
|
104
|
-
const request =
|
|
73
|
+
const request = got.stream(res.downloadLink, options);
|
|
105
74
|
request.retryCount = retryCount;
|
|
106
75
|
request.once('response', async (response) => {
|
|
107
76
|
response.retryCount = retryCount;
|
|
@@ -114,20 +83,20 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
114
83
|
if (rangeIsSupported && fileWriteStream && rangeStart &&
|
|
115
84
|
(response.statusCode !== 206 ||
|
|
116
85
|
!isSameRangeStart(rangeStart, response.headers['content-range']))) {
|
|
117
|
-
|
|
86
|
+
errorLogger.warn('Unexpected response for range', rangeStart, response.headers['content-range'], response.statusCode);
|
|
118
87
|
rangeIsSupported = false;
|
|
119
88
|
fileWriteStream.destroy();
|
|
120
89
|
fileWriteStream = undefined;
|
|
121
90
|
rangeStart = undefined;
|
|
122
91
|
}
|
|
123
|
-
if (response.request.
|
|
92
|
+
if (response.request.isAborted) {
|
|
124
93
|
// Canceled while downloading
|
|
125
94
|
//- will throw a `CancelError` or `TimeoutError` error
|
|
126
95
|
return;
|
|
127
96
|
}
|
|
128
97
|
// Download body
|
|
129
98
|
if (!fileWriteStream) {
|
|
130
|
-
fileWriteStream =
|
|
99
|
+
fileWriteStream = createWriteStream(savePath, {
|
|
131
100
|
flags: 'w'
|
|
132
101
|
});
|
|
133
102
|
}
|
|
@@ -140,9 +109,9 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
140
109
|
// See request.once('error')
|
|
141
110
|
return;
|
|
142
111
|
}
|
|
143
|
-
if (request._isAboutToError) {
|
|
144
|
-
|
|
145
|
-
}
|
|
112
|
+
// if (request._isAboutToError) {
|
|
113
|
+
// return;
|
|
114
|
+
// }
|
|
146
115
|
resolve(response);
|
|
147
116
|
});
|
|
148
117
|
const destroyStream = () => {
|
|
@@ -168,13 +137,13 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
168
137
|
const onError = (error) => {
|
|
169
138
|
// https://developer.mozilla.org/docs/Web/HTTP/Headers/Range
|
|
170
139
|
// https://developer.mozilla.org/docs/Web/HTTP/Status/416
|
|
171
|
-
if (error instanceof
|
|
172
|
-
|
|
140
|
+
if (error instanceof HTTPError && error.response.statusCode === 416) {
|
|
141
|
+
errorLogger.warn('Unexpected response for range', rangeStart, error.response.headers['content-range'], error.response.statusCode);
|
|
173
142
|
rangeIsSupported = false;
|
|
174
143
|
}
|
|
175
144
|
destroyStream();
|
|
176
145
|
const { options } = request;
|
|
177
|
-
if (error instanceof
|
|
146
|
+
if (error instanceof HTTPError && !options.throwHttpErrors) {
|
|
178
147
|
const { response } = error;
|
|
179
148
|
resolve(response);
|
|
180
149
|
return;
|
|
@@ -182,7 +151,7 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
182
151
|
if (!isRetry) {
|
|
183
152
|
let retry = -1;
|
|
184
153
|
if (error && error.message === 'premature close') {
|
|
185
|
-
|
|
154
|
+
retryLogger.warn(retryCount, res.downloadLink, 'manually retry on premature close',
|
|
186
155
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
187
156
|
error.name, error.code, error.event, error.message);
|
|
188
157
|
retry = retryCount * 200;
|
|
@@ -195,7 +164,7 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
195
164
|
// at Object.exports.default (got\dist\source\core\utils\timed-out.js:56:23)
|
|
196
165
|
// error.code === undefined
|
|
197
166
|
(error.code === 'ETIMEDOUT' || error.code === undefined)) {
|
|
198
|
-
|
|
167
|
+
retryLogger.warn(retryCount, res.downloadLink,
|
|
199
168
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
200
169
|
`manually retry on ${error.event} timeout`, error.name, error.code, error.message);
|
|
201
170
|
retry = retryCount * 300;
|
|
@@ -217,31 +186,29 @@ async function streamingDownloadToFile(res, requestOptions) {
|
|
|
217
186
|
makeRequest(0);
|
|
218
187
|
});
|
|
219
188
|
}
|
|
220
|
-
|
|
221
|
-
async function optionallySetLastModifiedTime(res, options) {
|
|
189
|
+
export async function optionallySetLastModifiedTime(res, options) {
|
|
222
190
|
var _a, _b, _c;
|
|
223
191
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
224
|
-
let mtime;
|
|
192
|
+
let mtime = void 0;
|
|
225
193
|
if (options.preferRemoteLastModifiedTime && ((_b = (_a = res.meta) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['last-modified'])) {
|
|
226
194
|
mtime = Date.parse((_c = res.meta.headers) === null || _c === void 0 ? void 0 : _c['last-modified']);
|
|
227
195
|
}
|
|
228
196
|
// void and NaN check
|
|
229
197
|
if (mtime) {
|
|
230
|
-
const savePath =
|
|
198
|
+
const savePath = path.join(res.localRoot, decodeURI(res.savePath));
|
|
231
199
|
try {
|
|
232
|
-
await
|
|
200
|
+
await fs.utimes(savePath, mtime, mtime);
|
|
233
201
|
}
|
|
234
202
|
catch (e) {
|
|
235
|
-
|
|
203
|
+
errorLogger.warn('skipping utimes ' + savePath, e);
|
|
236
204
|
}
|
|
237
205
|
}
|
|
238
206
|
}
|
|
239
|
-
|
|
240
|
-
async function downloadStreamingResource(res, requestOptions, options) {
|
|
207
|
+
export async function downloadStreamingResource(res, requestOptions, options) {
|
|
241
208
|
if (res.body) {
|
|
242
209
|
return res;
|
|
243
210
|
}
|
|
244
|
-
if (res.type !==
|
|
211
|
+
if (res.type !== ResourceType.StreamingBinary) {
|
|
245
212
|
return res;
|
|
246
213
|
}
|
|
247
214
|
if (!res.downloadStartTimestamp) {
|
|
@@ -256,8 +223,7 @@ async function downloadStreamingResource(res, requestOptions, options) {
|
|
|
256
223
|
// res.finishTimestamp - res.downloadStartTimestamp;
|
|
257
224
|
return;
|
|
258
225
|
}
|
|
259
|
-
|
|
260
|
-
function downloadStreamingResourceWithHook(beforeDownload, afterDownload, downloadError) {
|
|
226
|
+
export function downloadStreamingResourceWithHook(beforeDownload, afterDownload, downloadError) {
|
|
261
227
|
if (!beforeDownload && !afterDownload && !downloadError) {
|
|
262
228
|
return downloadStreamingResource;
|
|
263
229
|
}
|
|
@@ -265,10 +231,10 @@ function downloadStreamingResourceWithHook(beforeDownload, afterDownload, downlo
|
|
|
265
231
|
if (res.body) {
|
|
266
232
|
return res;
|
|
267
233
|
}
|
|
268
|
-
if (res.type !==
|
|
234
|
+
if (res.type !== ResourceType.StreamingBinary) {
|
|
269
235
|
return res;
|
|
270
236
|
}
|
|
271
|
-
if (!
|
|
237
|
+
if (!isUrlHttp(res.downloadLink)) {
|
|
272
238
|
return res;
|
|
273
239
|
}
|
|
274
240
|
let resource = res;
|
|
@@ -310,5 +276,4 @@ function downloadStreamingResourceWithHook(beforeDownload, afterDownload, downlo
|
|
|
310
276
|
return;
|
|
311
277
|
};
|
|
312
278
|
}
|
|
313
|
-
exports.downloadStreamingResourceWithHook = downloadStreamingResourceWithHook;
|
|
314
279
|
//# sourceMappingURL=download-streaming-resource.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-streaming-resource.js","sourceRoot":"","sources":["../../src/life-cycle/download-streaming-resource.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"download-streaming-resource.js","sourceRoot":"","sources":["../../src/life-cycle/download-streaming-resource.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAC,SAAS,EAAE,iBAAiB,EAAE,QAAQ,IAAI,EAAE,EAAC,MAAM,SAAS,CAAC;AACrE,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AACrC,OAAO,EAAC,SAAS,EAAC,MAAM,WAAW,CAAC;AAEpC,OAAO,GAAG,EAAE,EAAC,SAAS,EAAC,MAAM,KAAK,CAAC;AAEnC,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAO5C,OAAO,EAAC,UAAU,EAAC,MAAM,UAAU,CAAC;AACpC,OAAO,EAAC,KAAK,IAAI,WAAW,EAAE,KAAK,IAAI,WAAW,EAAC,MAAM,qBAAqB,CAAC;AAG/E,OAAO,EAAC,SAAS,EAAC,MAAM,YAAY,CAAC;AAErC,MAAM,iBAAiB,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,WAAoB;IAClD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,YAAqB;IACxE,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAkD,EAClD,cAA8B;IAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,gCAAgC;QAChC,IAAI,CAAC,IAAK,CAA4B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,4BAA4B;IAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE;QAChD,QAAQ,EAAE,IAAI;KACf,CAEA,CAAC;IACF,IAAI,eAAmC,CAAC;IAExC,OAAO,IAAI,OAAO,CAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC/C,IAAI,gBAAgC,CAAC;QACrC,IAAI,UAAyB,CAAC;QAC9B,MAAM,WAAW,GAAG,CAAC,UAAkB,EAAQ,EAAE;YAC/C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,gBAAgB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzC,UAAU,GAAG,SAAS,CAAC;gBACvB,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,UAAU,IAAI,gBAAgB,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,SAAS,UAAU,GAAG,CAAC;gBAC/C,eAAe,GAAG,iBAAiB,CAAC,QAAQ,EAAE;oBAC5C,KAAK,EAAE,GAAG;oBACV,KAAK,EAAE,UAAU;iBAClB,CAAC,CAAC;YACL,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;YAEhC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,QAAkB,EAAE,EAAE;gBACpD,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;gBACpC,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;oBACnC,IAAI,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;wBACvD,gBAAgB,GAAG,IAAI,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBACD,IAAI,gBAAgB,IAAI,eAAe,IAAI,UAAU;oBACnD,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG;wBAC1B,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtE,WAAW,CAAC,IAAI,CAAC,+BAA+B,EAC9C,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;oBACtE,gBAAgB,GAAG,KAAK,CAAC;oBACzB,eAAe,CAAC,OAAO,EAAE,CAAC;oBAC1B,eAAe,GAAG,SAAS,CAAC;oBAC5B,UAAU,GAAG,SAAS,CAAC;gBACzB,CAAC;gBAED,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;oBAC/B,6BAA6B;oBAC7B,sDAAsD;oBACtD,OAAO;gBACT,CAAC;gBACD,gBAAgB;gBAChB,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,iBAAiB,CAAC,QAAQ,EAAE;wBAC5C,KAAK,EAAE,GAAG;qBACX,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC;oBACH,iCAAiC;oBACjC,MAAM,iBAAiB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;gBACpD,CAAC;gBAAC,WAAM,CAAC;oBACP,kCAAkC;oBAClC,4BAA4B;oBAC5B,OAAO;gBACT,CAAC;gBAED,iCAAiC;gBACjC,YAAY;gBACZ,IAAI;gBAEJ,OAAO,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,GAAG,EAAE;gBACzB,IAAI,eAAe,EAAE,CAAC;oBACpB,IAAI,gBAAgB,EAAE,CAAC;wBACrB,IAAI,UAAU,EAAE,CAAC;4BACf,UAAU,IAAI,eAAe,CAAC,YAAY,CAAC;wBAC7C,CAAC;6BAAM,CAAC;4BACN,UAAU,GAAG,eAAe,CAAC,YAAY,CAAC;wBAC5C,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,UAAU,GAAG,SAAS,CAAC;oBACzB,CAAC;oBACD,eAAe,CAAC,OAAO,EAAE,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACN,UAAU,GAAG,SAAS,CAAC;gBACzB,CAAC;gBACD,eAAe,GAAG,SAAS,CAAC;YAC9B,CAAC,CAAC;YAEF,MAAM,OAAO,GAAG,CAAC,KAAmB,EAAE,EAAE;gBACtC,4DAA4D;gBAC5D,yDAAyD;gBACzD,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACpE,WAAW,CAAC,IAAI,CAAC,+BAA+B,EAC9C,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EACnD,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;oBAC7B,gBAAgB,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,EAAC,OAAO,EAAC,GAAG,OAAO,CAAC;gBAE1B,IAAI,KAAK,YAAY,SAAS,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC3D,MAAM,EAAC,QAAQ,EAAC,GAAG,KAAK,CAAC;oBACzB,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAClB,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;oBACf,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,iBAAiB,EAAE,CAAC;wBACjD,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAC3C,mCAAmC;wBACnC,8DAA8D;wBAC9D,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAG,KAAa,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC/D,KAAK,GAAG,UAAU,GAAG,GAAG,CAAC;oBAC3B,CAAC;oBACD,+CAA+C;oBAC/C,8DAA8D;oBAC9D,IAAI,KAAK,IAAI,CAAE,KAAa,CAAC,kBAAkB;wBAC7C,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;wBAChE,4DAA4D;wBAC5D,4EAA4E;wBAC5E,2BAA2B;wBAC3B,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;wBAC3D,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY;wBAC3C,8DAA8D;wBAC9D,qBAAsB,KAAa,CAAC,KAAK,UAAU,EACnD,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;wBACzC,KAAK,GAAG,UAAU,GAAG,GAAG,CAAC;oBAC3B,CAAC;oBACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;wBACd,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;wBAC5B,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE/B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,aAAqB,EAAE,EAAE;gBAC9C,aAAa,EAAE,CAAC;gBAChB,OAAO,GAAG,IAAI,CAAC;gBACf,WAAW,CAAC,aAAa,CAAC,CAAC;YAC7B,CAAC,CAAC,CAAC;QAEL,CAAC,CAAC;QAEF,WAAW,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,GAAa,EAAE,OAA8B;;IAE7C,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,qBAAqB;IACrB,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,WAAW,CAAC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,GAAa,EACb,cAA8B,EAC9B,OAA8B;IAE9B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,GAAuB,CAAC;IACjC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;QAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;IAClE,CAAC;IACD,MAAM,uBAAuB,CAC3B,GAAsD,EAAE,cAAc,CAAC,CAAC;IAE1E,MAAM,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAClD,qBAAqB;IACrB,oCAAoC;IACpC,qBAAqB;IACrB,sDAAsD;IACtD,OAAO;AACT,CAAC;AA+CD,MAAM,UAAU,iCAAiC,CAC/C,cAA4C,EAC5C,aAA0C,EAC1C,aAA0C;IAE1C,IAAI,CAAC,cAAc,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,EAAE,CAAC;QACxD,OAAO,yBAAyB,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,EACV,GAAa,EACb,cAA8B,EAC9B,OAA8B,EAC9B,QAA0B,EAC1B,EAAE;QACF,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACb,OAAO,GAAuB,CAAC;QACjC,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;YAC9C,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAI,QAAQ,GAAuC,GAAG,CAAC;QACvD,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACxE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClB,OAAO,QAA4B,CAAC;YACtC,CAAC;YACD,IAAI,QAAQ,CAAC,6BAA6B,EAAE,CAAC;gBAC3C,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;YAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;QAClE,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,uBAAuB,CAC3B,GAAsD,EAAE,cAAc,CAAC,CAAC;YAC1E,MAAM,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,uBAAuB,CAC3B,GAAsD,EAAE,cAAc,CAAC,CAAC;gBAC1E,MAAM,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,aAAa,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QACD,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,GAAG,CAAC,YAAY;YACd,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,sBAAsB,CAAC;QACnD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO;IACT,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
export * as adapter from './adapters';
|
|
2
|
-
export { defaultLifeCycle } from './default-life-cycle';
|
|
3
|
-
export { detectResourceType } from './detect-resource-type';
|
|
4
|
-
export { beforeRetryHook, getRetry, requestForResource, downloadResource } from './download-resource';
|
|
5
|
-
export { streamingDownloadToFile, downloadStreamingResource, downloadStreamingResourceWithHook } from './download-streaming-resource';
|
|
6
|
-
export { PipelineExecutor } from './pipeline-executor';
|
|
7
|
-
export { processCssText, processCss } from './process-css';
|
|
8
|
-
export { processHtml } from './process-html';
|
|
9
|
-
export { processHtmlMetaRefresh } from './process-html-meta';
|
|
10
|
-
export { processSiteMap } from './process-site-map';
|
|
11
|
-
export { processSvg } from './process-svg';
|
|
12
|
-
export { getResourceBodyFromHtml, saveHtmlToDisk } from './save-html-to-disk';
|
|
13
|
-
export { saveResourceToDisk } from './save-resource-to-disk';
|
|
14
|
-
export { skipLinks } from './skip-links';
|
|
15
|
-
export * as types from './types';
|
|
1
|
+
export * as adapter from './adapters.js';
|
|
2
|
+
export { defaultLifeCycle } from './default-life-cycle.js';
|
|
3
|
+
export { detectResourceType } from './detect-resource-type.js';
|
|
4
|
+
export { beforeRetryHook, getRetry, requestForResource, downloadResource } from './download-resource.js';
|
|
5
|
+
export { streamingDownloadToFile, downloadStreamingResource, downloadStreamingResourceWithHook } from './download-streaming-resource.js';
|
|
6
|
+
export type { PipelineExecutor } from './pipeline-executor.js';
|
|
7
|
+
export { processCssText, processCss } from './process-css.js';
|
|
8
|
+
export { processHtml } from './process-html.js';
|
|
9
|
+
export { processHtmlMetaRefresh } from './process-html-meta.js';
|
|
10
|
+
export { processSiteMap } from './process-site-map.js';
|
|
11
|
+
export { processSvg } from './process-svg.js';
|
|
12
|
+
export { getResourceBodyFromHtml, saveHtmlToDisk } from './save-html-to-disk.js';
|
|
13
|
+
export { saveResourceToDisk } from './save-resource-to-disk.js';
|
|
14
|
+
export { skipLinks } from './skip-links.js';
|
|
15
|
+
export * as types from './types.js';
|
|
16
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/life-cycle/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,eAAe,CAAC;AACzC,OAAO,EAAC,gBAAgB,EAAC,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EACL,eAAe,EAAE,QAAQ,EAAE,kBAAkB,EAAE,gBAAgB,EAChE,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,uBAAuB,EACvB,yBAAyB,EACzB,iCAAiC,EAClC,MAAM,kCAAkC,CAAC;AAC1C,YAAY,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAC,cAAc,EAAE,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAC,sBAAsB,EAAC,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,cAAc,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,uBAAuB,EAAE,cAAc,EAAC,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAC"}
|
package/lib/life-cycle/index.js
CHANGED
|
@@ -1,60 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.types = exports.skipLinks = exports.saveResourceToDisk = exports.saveHtmlToDisk = exports.getResourceBodyFromHtml = exports.processSvg = exports.processSiteMap = exports.processHtmlMetaRefresh = exports.processHtml = exports.processCss = exports.processCssText = exports.downloadStreamingResourceWithHook = exports.downloadStreamingResource = exports.streamingDownloadToFile = exports.downloadResource = exports.requestForResource = exports.getRetry = exports.beforeRetryHook = exports.detectResourceType = exports.defaultLifeCycle = exports.adapter = void 0;
|
|
27
|
-
exports.adapter = __importStar(require("./adapters"));
|
|
28
|
-
var default_life_cycle_1 = require("./default-life-cycle");
|
|
29
|
-
Object.defineProperty(exports, "defaultLifeCycle", { enumerable: true, get: function () { return default_life_cycle_1.defaultLifeCycle; } });
|
|
30
|
-
var detect_resource_type_1 = require("./detect-resource-type");
|
|
31
|
-
Object.defineProperty(exports, "detectResourceType", { enumerable: true, get: function () { return detect_resource_type_1.detectResourceType; } });
|
|
32
|
-
var download_resource_1 = require("./download-resource");
|
|
33
|
-
Object.defineProperty(exports, "beforeRetryHook", { enumerable: true, get: function () { return download_resource_1.beforeRetryHook; } });
|
|
34
|
-
Object.defineProperty(exports, "getRetry", { enumerable: true, get: function () { return download_resource_1.getRetry; } });
|
|
35
|
-
Object.defineProperty(exports, "requestForResource", { enumerable: true, get: function () { return download_resource_1.requestForResource; } });
|
|
36
|
-
Object.defineProperty(exports, "downloadResource", { enumerable: true, get: function () { return download_resource_1.downloadResource; } });
|
|
37
|
-
var download_streaming_resource_1 = require("./download-streaming-resource");
|
|
38
|
-
Object.defineProperty(exports, "streamingDownloadToFile", { enumerable: true, get: function () { return download_streaming_resource_1.streamingDownloadToFile; } });
|
|
39
|
-
Object.defineProperty(exports, "downloadStreamingResource", { enumerable: true, get: function () { return download_streaming_resource_1.downloadStreamingResource; } });
|
|
40
|
-
Object.defineProperty(exports, "downloadStreamingResourceWithHook", { enumerable: true, get: function () { return download_streaming_resource_1.downloadStreamingResourceWithHook; } });
|
|
41
|
-
var process_css_1 = require("./process-css");
|
|
42
|
-
Object.defineProperty(exports, "processCssText", { enumerable: true, get: function () { return process_css_1.processCssText; } });
|
|
43
|
-
Object.defineProperty(exports, "processCss", { enumerable: true, get: function () { return process_css_1.processCss; } });
|
|
44
|
-
var process_html_1 = require("./process-html");
|
|
45
|
-
Object.defineProperty(exports, "processHtml", { enumerable: true, get: function () { return process_html_1.processHtml; } });
|
|
46
|
-
var process_html_meta_1 = require("./process-html-meta");
|
|
47
|
-
Object.defineProperty(exports, "processHtmlMetaRefresh", { enumerable: true, get: function () { return process_html_meta_1.processHtmlMetaRefresh; } });
|
|
48
|
-
var process_site_map_1 = require("./process-site-map");
|
|
49
|
-
Object.defineProperty(exports, "processSiteMap", { enumerable: true, get: function () { return process_site_map_1.processSiteMap; } });
|
|
50
|
-
var process_svg_1 = require("./process-svg");
|
|
51
|
-
Object.defineProperty(exports, "processSvg", { enumerable: true, get: function () { return process_svg_1.processSvg; } });
|
|
52
|
-
var save_html_to_disk_1 = require("./save-html-to-disk");
|
|
53
|
-
Object.defineProperty(exports, "getResourceBodyFromHtml", { enumerable: true, get: function () { return save_html_to_disk_1.getResourceBodyFromHtml; } });
|
|
54
|
-
Object.defineProperty(exports, "saveHtmlToDisk", { enumerable: true, get: function () { return save_html_to_disk_1.saveHtmlToDisk; } });
|
|
55
|
-
var save_resource_to_disk_1 = require("./save-resource-to-disk");
|
|
56
|
-
Object.defineProperty(exports, "saveResourceToDisk", { enumerable: true, get: function () { return save_resource_to_disk_1.saveResourceToDisk; } });
|
|
57
|
-
var skip_links_1 = require("./skip-links");
|
|
58
|
-
Object.defineProperty(exports, "skipLinks", { enumerable: true, get: function () { return skip_links_1.skipLinks; } });
|
|
59
|
-
exports.types = __importStar(require("./types"));
|
|
1
|
+
export * as adapter from './adapters.js';
|
|
2
|
+
export { defaultLifeCycle } from './default-life-cycle.js';
|
|
3
|
+
export { detectResourceType } from './detect-resource-type.js';
|
|
4
|
+
export { beforeRetryHook, getRetry, requestForResource, downloadResource } from './download-resource.js';
|
|
5
|
+
export { streamingDownloadToFile, downloadStreamingResource, downloadStreamingResourceWithHook } from './download-streaming-resource.js';
|
|
6
|
+
export { processCssText, processCss } from './process-css.js';
|
|
7
|
+
export { processHtml } from './process-html.js';
|
|
8
|
+
export { processHtmlMetaRefresh } from './process-html-meta.js';
|
|
9
|
+
export { processSiteMap } from './process-site-map.js';
|
|
10
|
+
export { processSvg } from './process-svg.js';
|
|
11
|
+
export { getResourceBodyFromHtml, saveHtmlToDisk } from './save-html-to-disk.js';
|
|
12
|
+
export { saveResourceToDisk } from './save-resource-to-disk.js';
|
|
13
|
+
export { skipLinks } from './skip-links.js';
|
|
14
|
+
export * as types from './types.js';
|
|
60
15
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/life-cycle/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/life-cycle/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,eAAe,CAAC;AACzC,OAAO,EAAC,gBAAgB,EAAC,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAC,kBAAkB,EAAC,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EACL,eAAe,EAAE,QAAQ,EAAE,kBAAkB,EAAE,gBAAgB,EAChE,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,uBAAuB,EACvB,yBAAyB,EACzB,iCAAiC,EAClC,MAAM,kCAAkC,CAAC;AAE1C,OAAO,EAAC,cAAc,EAAE,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5D,OAAO,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAC;AAC9C,OAAO,EAAC,sBAAsB,EAAC,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,cAAc,EAAC,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAC,uBAAuB,EAAE,cAAc,EAAC,MAAM,wBAAwB,CAAC;AAC/E,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC1C,OAAO,KAAK,KAAK,MAAM,YAAY,CAAC"}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import type { Resource, ResourceEncoding, ResourceType } from '../resource';
|
|
2
|
-
import type { StaticDownloadOptions } from '../options';
|
|
3
|
-
import type { AsyncResult, DownloadResource, RequestOptions, SubmitResourceFunc } from './types';
|
|
4
|
-
import type { Cheerio } from '../types';
|
|
5
|
-
import type { DownloaderWithMeta } from '../downloader/types';
|
|
6
|
-
import type { WorkerInfo } from '../downloader/worker-pool';
|
|
1
|
+
import type { Resource, ResourceEncoding, ResourceType } from '../resource.js';
|
|
2
|
+
import type { StaticDownloadOptions } from '../options.js';
|
|
3
|
+
import type { AsyncResult, DownloadResource, RequestOptions, SubmitResourceFunc } from './types.js';
|
|
4
|
+
import type { Cheerio } from '../types.js';
|
|
5
|
+
import type { DownloaderWithMeta } from '../downloader/types.js';
|
|
6
|
+
import type { WorkerInfo } from '../downloader/worker-pool.js';
|
|
7
7
|
export interface PipelineExecutor {
|
|
8
8
|
/**
|
|
9
9
|
* @see InitLifeCycleFunc
|
|
@@ -36,3 +36,4 @@ export interface PipelineExecutor {
|
|
|
36
36
|
*/
|
|
37
37
|
dispose(pipeline: PipelineExecutor, downloader: DownloaderWithMeta, workerInfo?: WorkerInfo, workerExitCode?: number): AsyncResult<void>;
|
|
38
38
|
}
|
|
39
|
+
//# sourceMappingURL=pipeline-executor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline-executor.d.ts","sourceRoot":"","sources":["../../src/life-cycle/pipeline-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,QAAQ,EAAE,gBAAgB,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EACnB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,aAAa,CAAC;AACzC,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,wBAAwB,CAAC;AAC/D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,8BAA8B,CAAC;AAE7D,MAAM,WAAW,gBAAgB;IAC/B;;OAEG;IACH,IAAI,CACF,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,CAAC,EAAE,kBAAkB,GAC9B,WAAW,CAAC,IAAI,CAAC,CAAC;IAErB;;;;;;;OAOG;IACH,wBAAwB,CACtB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,YAAY,EACzB,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI,EAC3B,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,MAAM,EAAE,QAAQ,GACf,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;IAEhC,YAAY,CACV,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,MAAM,EAAE,QAAQ,GAAG,IAAI,GACtB,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAE9B,kBAAkB,CAChB,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,MAAM,EAAE,QAAQ,GAAG,IAAI,GACtB,WAAW,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAEpC,cAAc,CACZ,IAAI,EAAE,YAAY,EAClB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,SAAS,CAAC,EAAE,MAAM,EAClB,QAAQ,CAAC,EAAE,gBAAgB,EAC3B,WAAW,CAAC,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,YAAY,GACrB,WAAW,CAAC,QAAQ,CAAC,CAAC;IAEzB,qBAAqB,CACnB,GAAG,EAAE,QAAQ,EACb,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,MAAM,EAAE,QAAQ,GAAG,IAAI,EACvB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;IAEhC,QAAQ,CACN,GAAG,EAAE,QAAQ,EACb,cAAc,CAAC,EAAE,cAAc,EAC/B,OAAO,CAAC,EAAE,qBAAqB,GAC9B,WAAW,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAExC;;;;;OAKG;IACH,oBAAoB,CAClB,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,CAAC,EAAE,qBAAqB,GAC9B,WAAW,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAExC,UAAU,CACR,GAAG,EAAE,gBAAgB,EACrB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,WAAW,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC;IAExC;;OAEG;IACH,OAAO,CACL,QAAQ,EAAE,gBAAgB,EAC1B,UAAU,EAAE,kBAAkB,EAC9B,UAAU,CAAC,EAAE,UAAU,EACvB,cAAc,CAAC,EAAE,MAAM,GACtB,WAAW,CAAC,IAAI,CAAC,CAAC;CAEtB"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type { DownloadResource, SubmitResourceFunc } from './types';
|
|
2
|
-
import type { StaticDownloadOptions } from '../options';
|
|
3
|
-
import { Resource } from '../resource';
|
|
4
|
-
import type { PipelineExecutor } from './pipeline-executor';
|
|
1
|
+
import type { DownloadResource, SubmitResourceFunc } from './types.js';
|
|
2
|
+
import type { StaticDownloadOptions } from '../options.js';
|
|
3
|
+
import type { Resource } from '../resource.js';
|
|
4
|
+
import type { PipelineExecutor } from './pipeline-executor.js';
|
|
5
5
|
export declare function processCssText(cssText: string, res: DownloadResource, options: StaticDownloadOptions, pipeline: PipelineExecutor, depth: number, resources: Resource[]): Promise<string>;
|
|
6
6
|
export declare function processCss(res: DownloadResource, submit: SubmitResourceFunc, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource | void>;
|
|
7
|
+
//# sourceMappingURL=process-css.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process-css.d.ts","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAE,kBAAkB,EAAC,MAAM,YAAY,CAAC;AACrE,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAG7C,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAexC;AAED,wBAAsB,UAAU,CAC9B,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAa9D"}
|
|
@@ -1,19 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
exports.processCss = exports.processCssText = void 0;
|
|
7
|
-
const css_url_parser_1 = __importDefault(require("css-url-parser"));
|
|
8
|
-
const resource_1 = require("../resource");
|
|
9
|
-
const util_1 = require("../util");
|
|
10
|
-
async function processCssText(cssText, res, options, pipeline, depth, resources) {
|
|
11
|
-
const cssUrls = (0, css_url_parser_1.default)(cssText);
|
|
1
|
+
import parseCssUrls from 'css-url-parser';
|
|
2
|
+
import { ResourceType } from '../resource.js';
|
|
3
|
+
import { toString } from '../util.js';
|
|
4
|
+
export async function processCssText(cssText, res, options, pipeline, depth, resources) {
|
|
5
|
+
const cssUrls = parseCssUrls(cssText);
|
|
12
6
|
let rawUrl, r;
|
|
13
7
|
// noinspection DuplicatedCode
|
|
14
8
|
for (let i = 0, l = cssUrls.length; i < l; i++) {
|
|
15
9
|
rawUrl = cssUrls[i];
|
|
16
|
-
r = await pipeline.createAndProcessResource(rawUrl,
|
|
10
|
+
r = await pipeline.createAndProcessResource(rawUrl, ResourceType.Binary, depth, null, res);
|
|
17
11
|
if (!r)
|
|
18
12
|
continue;
|
|
19
13
|
if (!r.shouldBeDiscardedFromDownload) {
|
|
@@ -23,20 +17,18 @@ async function processCssText(cssText, res, options, pipeline, depth, resources)
|
|
|
23
17
|
}
|
|
24
18
|
return cssText;
|
|
25
19
|
}
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
if (res.type !== resource_1.ResourceType.Css) {
|
|
20
|
+
export async function processCss(res, submit, options, pipeline) {
|
|
21
|
+
if (res.type !== ResourceType.Css) {
|
|
29
22
|
return res;
|
|
30
23
|
}
|
|
31
24
|
const depth = res.depth + 1;
|
|
32
25
|
const resources = [];
|
|
33
|
-
let cssText =
|
|
34
|
-
options.encoding[
|
|
26
|
+
let cssText = toString(res.body, res.encoding ||
|
|
27
|
+
options.encoding[ResourceType.Css]);
|
|
35
28
|
cssText = await processCssText(cssText, res, options, pipeline, depth, resources);
|
|
36
29
|
res.body = cssText;
|
|
37
30
|
res.meta.cssProcessed = 1;
|
|
38
31
|
await submit(resources);
|
|
39
32
|
return res;
|
|
40
33
|
}
|
|
41
|
-
exports.processCss = processCss;
|
|
42
34
|
//# sourceMappingURL=process-css.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-css.js","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"process-css.js","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,gBAAgB,CAAC;AAI1C,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AAGpC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,GAAqB,EACrB,OAA8B,EAC9B,QAA0B,EAC1B,KAAa,EACb,SAAqB;IACrB,MAAM,OAAO,GAAa,YAAY,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,MAAc,EAAE,CAAkB,CAAC;IACvC,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC,GAAG,MAAM,QAAQ,CAAC,wBAAwB,CACzC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,GAAG,EAAE,CAAC;QAClC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAW,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;IACpC,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,OAAO,GAAW,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ;QACnD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAClF,GAAG,CAAC,IAAI,GAAG,OAAO,CAAC;IACnB,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1B,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACxB,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type { DownloadResource, SubmitResourceFunc } from './types';
|
|
2
|
-
import type { StaticDownloadOptions } from '../options';
|
|
3
|
-
import type { PipelineExecutor } from './pipeline-executor';
|
|
1
|
+
import type { DownloadResource, SubmitResourceFunc } from './types.js';
|
|
2
|
+
import type { StaticDownloadOptions } from '../options.js';
|
|
3
|
+
import type { PipelineExecutor } from './pipeline-executor.js';
|
|
4
4
|
export declare function processHtmlMetaRefresh(res: DownloadResource, submit: SubmitResourceFunc, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource>;
|
|
5
|
+
//# sourceMappingURL=process-html-meta.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"process-html-meta.d.ts","sourceRoot":"","sources":["../../src/life-cycle/process-html-meta.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAE,kBAAkB,EAAC,MAAM,YAAY,CAAC;AACrE,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAc7D,wBAAsB,sBAAsB,CAC1C,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAkE3B"}
|
|
@@ -1,21 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const resource_1 = require("../resource");
|
|
5
|
-
const adapters_1 = require("./adapters");
|
|
6
|
-
const logger_1 = require("../logger/logger");
|
|
1
|
+
import { ResourceType } from '../resource.js';
|
|
2
|
+
import { parseHtml } from './adapters.js';
|
|
3
|
+
import { skip } from '../logger/logger.js';
|
|
7
4
|
/**
|
|
8
5
|
* Originally create by https://github.com/stevenvachon at
|
|
9
6
|
* https://github.com/stevenvachon/http-equiv-refresh
|
|
10
7
|
* MIT license
|
|
11
8
|
*/
|
|
12
9
|
const META_REFRESH_PATTERN = /^\s*(\d+)(?:\s*;(?:\s*url\s*=)?\s*(?:["']\s*(.*?)\s*['"]|(.*?)))?\s*$/i;
|
|
13
|
-
async function processHtmlMetaRefresh(res, submit, options, pipeline) {
|
|
14
|
-
if (res.type !==
|
|
10
|
+
export async function processHtmlMetaRefresh(res, submit, options, pipeline) {
|
|
11
|
+
if (res.type !== ResourceType.Html) {
|
|
15
12
|
return res;
|
|
16
13
|
}
|
|
17
14
|
if (!res.meta.doc) {
|
|
18
|
-
res.meta.doc =
|
|
15
|
+
res.meta.doc = parseHtml(res, options);
|
|
19
16
|
}
|
|
20
17
|
const $ = res.meta.doc;
|
|
21
18
|
const metaLinks = $('meta[http-equiv="refresh"][content]');
|
|
@@ -41,18 +38,18 @@ async function processHtmlMetaRefresh(res, submit, options, pipeline) {
|
|
|
41
38
|
if (!link) {
|
|
42
39
|
continue;
|
|
43
40
|
}
|
|
44
|
-
const linkType = await pipeline.detectResourceType(link,
|
|
41
|
+
const linkType = await pipeline.detectResourceType(link, ResourceType.Html, elem, res);
|
|
45
42
|
if (!linkType) {
|
|
46
|
-
if (
|
|
47
|
-
|
|
43
|
+
if (skip.isTraceEnabled()) {
|
|
44
|
+
skip.trace('skip detectResourceType', originalLink, link, refUrl);
|
|
48
45
|
}
|
|
49
46
|
continue;
|
|
50
47
|
}
|
|
51
48
|
let resource = await pipeline.createResource(linkType, depth, link, refUrl, res.localRoot, options.encoding[linkType], savePath, res.type);
|
|
52
49
|
resource = await pipeline.processBeforeDownload(resource, elem, res, options);
|
|
53
50
|
if (!resource) {
|
|
54
|
-
if (
|
|
55
|
-
|
|
51
|
+
if (skip.isTraceEnabled()) {
|
|
52
|
+
skip.trace('skip processBeforeDownload', originalLink, link, linkType, refUrl);
|
|
56
53
|
}
|
|
57
54
|
continue;
|
|
58
55
|
}
|
|
@@ -64,5 +61,4 @@ async function processHtmlMetaRefresh(res, submit, options, pipeline) {
|
|
|
64
61
|
}
|
|
65
62
|
return res;
|
|
66
63
|
}
|
|
67
|
-
exports.processHtmlMetaRefresh = processHtmlMetaRefresh;
|
|
68
64
|
//# sourceMappingURL=process-html-meta.js.map
|