website-scrap-engine 0.8.4 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/downloader/worker-pool.d.ts.map +1 -1
- package/lib/downloader/worker-pool.js +72 -17
- package/lib/downloader/worker-pool.js.map +1 -1
- package/lib/downloader/worker.js +1 -0
- package/lib/downloader/worker.js.map +1 -1
- package/lib/life-cycle/adapters.d.ts +1 -1
- package/lib/life-cycle/adapters.d.ts.map +1 -1
- package/lib/life-cycle/adapters.js.map +1 -1
- package/lib/life-cycle/download-resource.d.ts.map +1 -1
- package/lib/life-cycle/download-resource.js +18 -7
- package/lib/life-cycle/download-resource.js.map +1 -1
- package/lib/life-cycle/download-streaming-resource.js +2 -2
- package/lib/life-cycle/download-streaming-resource.js.map +1 -1
- package/lib/life-cycle/process-css.d.ts.map +1 -1
- package/lib/life-cycle/process-css.js +34 -3
- package/lib/life-cycle/process-css.js.map +1 -1
- package/lib/life-cycle/process-site-map.js +1 -1
- package/lib/life-cycle/process-site-map.js.map +1 -1
- package/lib/life-cycle/read-or-copy-local-resource.d.ts.map +1 -1
- package/lib/life-cycle/read-or-copy-local-resource.js +2 -0
- package/lib/life-cycle/read-or-copy-local-resource.js.map +1 -1
- package/lib/life-cycle/save-html-to-disk.d.ts +1 -1
- package/lib/life-cycle/save-html-to-disk.d.ts.map +1 -1
- package/lib/life-cycle/save-html-to-disk.js +3 -2
- package/lib/life-cycle/save-html-to-disk.js.map +1 -1
- package/lib/life-cycle/save-resource-to-disk.js +1 -1
- package/lib/life-cycle/save-resource-to-disk.js.map +1 -1
- package/lib/options.js +1 -1
- package/package.json +18 -14
- package/src/downloader/worker-pool.ts +71 -21
- package/src/downloader/worker.ts +1 -0
- package/src/life-cycle/adapters.ts +1 -1
- package/src/life-cycle/download-resource.ts +16 -6
- package/src/life-cycle/download-streaming-resource.ts +2 -2
- package/src/life-cycle/process-css.ts +30 -3
- package/src/life-cycle/process-site-map.ts +1 -1
- package/src/life-cycle/read-or-copy-local-resource.ts +2 -0
- package/src/life-cycle/save-html-to-disk.ts +4 -3
- package/src/life-cycle/save-resource-to-disk.ts +1 -1
- package/src/options.ts +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAE,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAC3C,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,UAAU,CAAC;AAElC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EACV,cAAc,EACd,sBAAsB,EACtB,aAAa,EACd,MAAM,YAAY,CAAC;AAGpB,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,cAAe,YAAW,UAAU;IAI5B,MAAM,EAAE,MAAM;IAHjC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,SAAK;gBAEU,MAAM,EAAE,MAAM;CAGlC;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC3D;AAOD,qBAAa,UAAU,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,SAAS,aAAa,GAAG,aAAa;IAQjE,QAAQ,EAAE,MAAM;IAChB,YAAY,EAAE,MAAM;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACnC,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,aAAa;IAX/B,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,CAAM;IACpC,QAAQ,CAAC,YAAY,EAAE,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAM;IACxD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAM;IAC3D,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,aAAa,SAAK;gBAGT,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,OAAO,GAAE,MAAW,EACpB,OAAO,GAAE,aAAoC;IAgBtD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,GAAG,IAAI;IAIjD,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,WAAW,EAAE,aAAa,EAAC,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAC3C,OAAO,KAAK,EAAC,GAAG,EAAC,MAAM,UAAU,CAAC;AAElC,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EACV,cAAc,EACd,sBAAsB,EACtB,aAAa,EACd,MAAM,YAAY,CAAC;AAGpB,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,cAAe,YAAW,UAAU;IAI5B,MAAM,EAAE,MAAM;IAHjC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,IAAI,SAAK;gBAEU,MAAM,EAAE,MAAM;CAGlC;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,MAAM,CAAC;CAC3D;AAOD,qBAAa,UAAU,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,SAAS,aAAa,GAAG,aAAa;IAQjE,QAAQ,EAAE,MAAM;IAChB,YAAY,EAAE,MAAM;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACnC,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,aAAa;IAX/B,QAAQ,CAAC,OAAO,EAAE,UAAU,EAAE,CAAM;IACpC,QAAQ,CAAC,YAAY,EAAE,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAM;IACxD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAM;IAC3D,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B,aAAa,SAAK;gBAGT,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACnC,OAAO,GAAE,MAAW,EACpB,OAAO,GAAE,aAAoC;IAgBtD,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,GAAG,IAAI;IAIjD,SAAS,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI;IAQzD,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAa1D,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,GAAG,IAAI;IASxD,UAAU,CACR,QAAQ,EAAE,CAAC,EACX,YAAY,CAAC,EAAE,KAAK,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;IAc9D,QAAQ,IAAI,IAAI;IAmFV,OAAO,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;CAenC"}
|
|
@@ -38,7 +38,9 @@ export class WorkerPool {
|
|
|
38
38
|
if (message.type === WorkerMessageType.Complete) {
|
|
39
39
|
this.complete(info, message);
|
|
40
40
|
}
|
|
41
|
-
|
|
41
|
+
else {
|
|
42
|
+
this.takeLog(info, message);
|
|
43
|
+
}
|
|
42
44
|
}
|
|
43
45
|
takeLog(info, message) {
|
|
44
46
|
var _a, _b, _c, _d, _e;
|
|
@@ -80,38 +82,91 @@ export class WorkerPool {
|
|
|
80
82
|
if (!this.pendingTasks.length) {
|
|
81
83
|
return;
|
|
82
84
|
}
|
|
83
|
-
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
85
|
+
const sorted = this.workers.slice().sort((a, b) => a.load - b.load);
|
|
86
|
+
const n = sorted.length;
|
|
87
|
+
let remaining = this.pendingTasks.length;
|
|
88
|
+
// Cap by maxLoad capacity
|
|
89
|
+
if (this.maxLoad > 0) {
|
|
90
|
+
let capacity = 0;
|
|
91
|
+
for (let i = 0; i < n; i++) {
|
|
92
|
+
capacity += Math.max(0, this.maxLoad - sorted[i].load);
|
|
93
|
+
}
|
|
94
|
+
remaining = Math.min(remaining, capacity);
|
|
95
|
+
}
|
|
96
|
+
if (remaining <= 0) {
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
// Pass 1: water-fill to calculate balanced task assignments
|
|
100
|
+
const assign = new Array(n).fill(0);
|
|
101
|
+
let level = sorted[0].load;
|
|
102
|
+
for (let i = 0; i < n - 1 && remaining > 0; i++) {
|
|
103
|
+
let gap = sorted[i + 1].load - level;
|
|
104
|
+
if (this.maxLoad > 0) {
|
|
105
|
+
gap = Math.min(gap, this.maxLoad - level);
|
|
106
|
+
}
|
|
107
|
+
if (gap <= 0)
|
|
108
|
+
continue;
|
|
109
|
+
const width = i + 1;
|
|
110
|
+
const cost = gap * width;
|
|
111
|
+
if (cost <= remaining) {
|
|
112
|
+
for (let j = 0; j <= i; j++)
|
|
113
|
+
assign[j] += gap;
|
|
114
|
+
remaining -= cost;
|
|
115
|
+
level += gap;
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
const each = (remaining / width) | 0;
|
|
119
|
+
let extra = remaining % width;
|
|
120
|
+
for (let j = 0; j <= i; j++) {
|
|
121
|
+
assign[j] += each + (extra > 0 ? 1 : 0);
|
|
122
|
+
if (extra > 0)
|
|
123
|
+
extra--;
|
|
93
124
|
}
|
|
125
|
+
remaining = 0;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
// Distribute remaining evenly across all workers
|
|
129
|
+
if (remaining > 0) {
|
|
130
|
+
const each = (remaining / n) | 0;
|
|
131
|
+
let extra = remaining % n;
|
|
132
|
+
for (let j = 0; j < n; j++) {
|
|
133
|
+
assign[j] += each + (extra > 0 ? 1 : 0);
|
|
134
|
+
if (extra > 0)
|
|
135
|
+
extra--;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
// Pass 2: dispatch tasks to workers
|
|
139
|
+
let dispatched = 0;
|
|
140
|
+
for (let i = 0; i < n; i++) {
|
|
141
|
+
for (let j = 0; j < assign[i]; j++) {
|
|
142
|
+
const task = this.pendingTasks[dispatched];
|
|
143
|
+
if (!task)
|
|
144
|
+
break;
|
|
145
|
+
dispatched++;
|
|
94
146
|
try {
|
|
95
|
-
|
|
147
|
+
sorted[i].worker.postMessage({
|
|
96
148
|
taskId: task.taskId,
|
|
97
149
|
body: task.body
|
|
98
150
|
}, task.transferList);
|
|
99
151
|
this.workingTasks[task.taskId] = task;
|
|
152
|
+
++sorted[i].load;
|
|
100
153
|
}
|
|
101
154
|
catch (e) {
|
|
102
155
|
delete this.workingTasks[task.taskId];
|
|
103
156
|
task.reject(e);
|
|
104
|
-
continue;
|
|
105
157
|
}
|
|
106
|
-
++sorted[i].load;
|
|
107
|
-
}
|
|
108
|
-
if (!this.pendingTasks.length) {
|
|
109
|
-
break;
|
|
110
158
|
}
|
|
111
159
|
}
|
|
160
|
+
if (dispatched > 0) {
|
|
161
|
+
this.pendingTasks.splice(0, dispatched);
|
|
162
|
+
}
|
|
112
163
|
}
|
|
113
164
|
async dispose() {
|
|
114
165
|
const numbers = await Promise.all(this.workers.map(info => info.worker.terminate()));
|
|
166
|
+
for (const task of this.pendingTasks) {
|
|
167
|
+
task.reject(new Error('disposed'));
|
|
168
|
+
}
|
|
169
|
+
this.pendingTasks.length = 0;
|
|
115
170
|
for (const taskId in this.workingTasks) {
|
|
116
171
|
// noinspection JSUnfilteredForInLoop
|
|
117
172
|
this.workingTasks[taskId].reject(new Error('disposed'));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAE3C,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAO9C,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAQ7C,MAAM,OAAO,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,UAAU;IAOrB,YACS,QAAgB,EAChB,YAAoB,EACpB,UAAmC,EACnC,UAAkB,CAAC,CAAC,EACpB,UAAyB,oBAAoB;QAJ7C,aAAQ,GAAR,QAAQ,CAAQ;QAChB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAyB;QACnC,YAAO,GAAP,OAAO,CAAa;QACpB,YAAO,GAAP,OAAO,CAAsC;QAX7C,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,
|
|
1
|
+
{"version":3,"file":"worker-pool.js","sourceRoot":"","sources":["../../src/downloader/worker-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAE3C,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAO9C,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAQ7C,MAAM,OAAO,cAAc;IAIzB,YAAmB,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAFjC,SAAI,GAAG,CAAC,CAAC;QAGP,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC5B,CAAC;CACF;AAMD,SAAS,oBAAoB,CAC3B,QAAsB,EAAE,OAAuB;IAC/C,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,UAAU;IAOrB,YACS,QAAgB,EAChB,YAAoB,EACpB,UAAmC,EACnC,UAAkB,CAAC,CAAC,EACpB,UAAyB,oBAAoB;QAJ7C,aAAQ,GAAR,QAAQ,CAAQ;QAChB,iBAAY,GAAZ,YAAY,CAAQ;QACpB,eAAU,GAAV,UAAU,CAAyB;QACnC,YAAO,GAAP,OAAO,CAAa;QACpB,YAAO,GAAP,OAAO,CAAsC;QAX7C,YAAO,GAAiB,EAAE,CAAC;QAC3B,iBAAY,GAAgC,EAAE,CAAC;QAC/C,iBAAY,GAAmC,EAAE,CAAC;QAE3D,kBAAa,GAAG,CAAC,CAAC;QAShB,MAAM,KAAK,GAAoB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,cAAc,CAClC,OAAO,CAAC,YAAY,EAAE,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,EAC1C,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EACxC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAY,CAAC,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAC/B,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,aAAa,CAAC,IAAgB,EAAE,GAAU;QACxC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACnD,CAAC;IAED,SAAS,CAAC,IAAgB,EAAE,OAAsB;QAChD,IAAI,OAAO,CAAC,IAAI,KAAK,iBAAiB,CAAC,QAAQ,EAAE,CAAC;YAChD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,OAA2B,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAgB,EAAE,OAAyB;;QACjD,IAAI,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,CAAA,EAAE,CAAC;YACnB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,IAAI,0CAAE,OAAO,CAAC;QACvC,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,MAAM,EAAE,CAAC;YACpB,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC;QAC7E,CAAC;aAAM,CAAC;YACN,MAAA,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,mDAAG,IAAI,CAAC,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,IAAgB,EAAE,OAAsB;QAC/C,EAAE,IAAI,CAAC,IAAI,CAAC;QACZ,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpC,MAAM,OAAO,GAA+B,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,UAAU,CACR,QAAW,EACX,YAA+C;QAC/C,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,IAAI,GAA8B;gBACtC,MAAM,EAAE,EAAE,IAAI,CAAC,aAAa;gBAC5B,OAAO;gBACP,MAAM;gBACN,IAAI,EAAE,QAAQ;gBACd,YAAY;aACb,CAAC;YACF,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7B,YAAY,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,IAAI,CACtC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,0BAA0B;QAC1B,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,QAAQ,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACzD,CAAC;YACD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,4DAA4D;QAC5D,MAAM,MAAM,GAAa,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,IAAI,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC;YACrC,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACrB,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;gBAAE,SAAS;YACvB,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC;YACzB,IAAI,IAAI,IAAI,SAAS,EAAE,CAAC;gBACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE;oBAAE,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;gBAC9C,SAAS,IAAI,IAAI,CAAC;gBAClB,KAAK,IAAI,GAAG,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,CAAC,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC;gBAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC5B,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACxC,IAAI,KAAK,GAAG,CAAC;wBAAE,KAAK,EAAE,CAAC;gBACzB,CAAC;gBACD,SAAS,GAAG,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QACD,iDAAiD;QACjD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC;YAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,IAAI,KAAK,GAAG,CAAC;oBAAE,KAAK,EAAE,CAAC;YACzB,CAAC;QACH,CAAC;QAED,oCAAoC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,IAAI,GACR,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;gBAChC,IAAI,CAAC,IAAI;oBAAE,MAAM;gBACjB,UAAU,EAAE,CAAC;gBACb,IAAI,CAAC;oBACH,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC;wBAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;wBACnB,IAAI,EAAE,IAAI,CAAC,IAAI;qBAChB,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;oBACtB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,IAAsB,CAAC;oBACxD,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBACnB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACrD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACrC,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACvC,qCAAqC;YACrC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;YACxD,qCAAqC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
package/lib/downloader/worker.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,UAAU,EAAC,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAMnD,OAAO,EAAC,iBAAiB,EAAE,uBAAuB,EAAC,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAC,IAAI,EAAC,MAAM,qBAAqB,CAAC;AACzC,OAAO,EAAC,qBAAqB,EAAC,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAKjE,MAAM,EAAC,aAAa,EAAE,eAAe,EAAC,GAGlC,UAAU,CAAC;AAEf,MAAM,YAAY,GAA6B,qBAAqB,CAAC,aAAa,CAAC,CAAC;AAEpF,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;IAChD,OAAO,GAAG,oBAAoB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAEzD,MAAM,QAAQ,GACZ,IAAI,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,IAAI,IAAK,IAAsB,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,GAAmC,EAAE,EAAE;IAC/E,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAC5C,IAAI,KAA6B,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;QACrB,MAAM,gBAAgB,GAAqB,iBAAiB,CAAC,GAAG,CAAqB,CAAC;QACtF,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,iBAAiB,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,iBAAiB,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC;QACF,MAAM,iBAAiB,GACrB,MAAM,QAAQ,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;aAAM,IAAI,MAAM,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,IAAI,CAAC,+BAA+B,EACvC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;YACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE,CAAC;YAC5D,aAAa,GAAG,iBAAiB,CAAC,aAAa,CAAC;QAClD,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,4CAA4C;QAC5C,mEAAmE;QACnE,IAAI,CAAC;YACH,mBAAmB;YACnB,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,uCAAuC;gBACvC,KAAK,GAAG,CAAC,CAAC;YACZ,CAAC;QACH,CAAC;QAAC,WAAM,CAAC;YACP,uDAAuD;YACvD,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAA2B,EAAE,CAAC;gBACzC,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAE,CAA6B,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,CAAC;
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/downloader/worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,UAAU,EAAC,MAAM,qBAAqB,CAAC;AAE3D,OAAO,EAAC,oBAAoB,EAAC,MAAM,eAAe,CAAC;AAMnD,OAAO,EAAC,iBAAiB,EAAE,uBAAuB,EAAC,MAAM,gBAAgB,CAAC;AAC1E,OAAO,EAAC,IAAI,EAAC,MAAM,qBAAqB,CAAC;AACzC,OAAO,EAAC,qBAAqB,EAAC,MAAM,YAAY,CAAC;AAEjD,OAAO,EAAC,iBAAiB,EAAC,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAC,oBAAoB,EAAC,MAAM,6BAA6B,CAAC;AAKjE,MAAM,EAAC,aAAa,EAAE,eAAe,EAAC,GAGlC,UAAU,CAAC;AAEf,MAAM,YAAY,GAA6B,qBAAqB,CAAC,aAAa,CAAC,CAAC;AAEpF,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;IAChD,OAAO,GAAG,oBAAoB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAEzD,MAAM,QAAQ,GACZ,IAAI,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAE1D,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC;IAEpE,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrC,IAAI,IAAI,IAAK,IAAsB,CAAC,IAAI,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,WAAW,CAAC,SAAS,EAAE,KAAK,EAAE,GAAmC,EAAE,EAAE;IAC/E,MAAM,iBAAiB,GAAkB,EAAE,CAAC;IAC5C,IAAI,KAA6B,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC;QACrC,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC;QACrB,MAAM,gBAAgB,GAAqB,iBAAiB,CAAC,GAAG,CAAqB,CAAC;QACtF,MAAM,MAAM,GAAuB,CAAC,SAAgC,EAAE,EAAE;YACtE,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC1C,iBAAiB,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChE,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,iBAAiB,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC,CAAC;QACF,MAAM,iBAAiB,GACrB,MAAM,QAAQ,CAAC,oBAAoB,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;QAChE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;aAAM,IAAI,MAAM,QAAQ,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,IAAI,CAAC,+BAA+B,EACvC,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;QAED,IAAI,iBAAiB,IAAI,iBAAiB,CAAC,aAAa;YACtD,iBAAiB,CAAC,aAAa,KAAK,iBAAiB,CAAC,GAAG,EAAE,CAAC;YAC5D,aAAa,GAAG,iBAAiB,CAAC,aAAa,CAAC;QAClD,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,4CAA4C;QAC5C,mEAAmE;QACnE,IAAI,CAAC;YACH,mBAAmB;YACnB,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE,CAAC;gBAC1C,KAAK,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,uCAAuC;gBACvC,KAAK,GAAG,CAAC,CAAC;YACZ,CAAC;QACH,CAAC;QAAC,WAAM,CAAC;YACP,uDAAuD;YACvD,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC/B,MAAM,KAAK,GAA2B,EAAE,CAAC;gBACzC,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAClB,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAE,CAA6B,CAAC,CAAC,CAAC,CAAC,CAAC;gBACvD,CAAC;gBACD,KAAK,GAAG,KAAK,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,GAA0B;YACrC,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,IAAI,EAAE,iBAAiB,CAAC,QAAQ;YAChC,IAAI,EAAE,iBAAiB;YACvB,KAAK;YACL,aAAa;SACd,CAAC;QACF,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;AAEH,CAAC,CAAC,CAAC"}
|
|
@@ -28,7 +28,7 @@ export interface HtmlProcessFunc {
|
|
|
28
28
|
}): CheerioStatic;
|
|
29
29
|
}
|
|
30
30
|
export declare const parseHtml: (res: DownloadResource & {
|
|
31
|
-
type: ResourceType.Html;
|
|
31
|
+
type: ResourceType.Html | ResourceType.Svg;
|
|
32
32
|
}, options: StaticDownloadOptions) => CheerioStatic;
|
|
33
33
|
export declare const processHtml: (fn: HtmlProcessFunc) => ProcessResourceAfterDownloadFunc;
|
|
34
34
|
export interface AsyncHtmlProcessFunc {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,QAAQ,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,gCAAgC,EAChC,iCAAiC,EACjC,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAC,OAAO,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAExD,MAAM,WAAW,eAAe;IAC9B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;CAC1E;AAED,eAAO,MAAM,WAAW,GAAI,IAAI,eAAe,KAAG,gBACoB,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC;CAC1B;AAED,eAAO,MAAM,YAAY,GAAI,IAAI,gBAAgB,KAAG,iCAMjD,CAAC;AAGJ,MAAM,WAAW,sBAAsB;IACrC,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;CACtF;AAED,eAAO,MAAM,UAAU,GAAI,IAAI,sBAAsB,KAAG,iCAIrD,CAAC;AAEJ,MAAM,WAAW,mBAAmB;IAClC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,eAAO,MAAM,eAAe,GAAI,IAAI,mBAAmB,KAAG,iCAUvD,CAAC;AAEJ,eAAO,MAAM,cAAc,GAAI,IAAI,mBAAmB,KAAG,gCAMtD,CAAC;AAEJ,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAclC;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,QAAQ,GAAG;QAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;KAAE,GAAG,aAAa,CAAC;CAChF;AAED,eAAO,MAAM,SAAS,GACpB,KAAK,gBAAgB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;CAAE,
|
|
1
|
+
{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,QAAQ,EAAmB,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,KAAK,EACV,WAAW,EACX,gBAAgB,EAChB,gBAAgB,EAChB,gCAAgC,EAChC,iCAAiC,EACjC,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAEpB,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAC,OAAO,EAAE,aAAa,EAAC,MAAM,aAAa,CAAC;AAExD,MAAM,WAAW,eAAe;IAC9B,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;CAC1E;AAED,eAAO,MAAM,WAAW,GAAI,IAAI,eAAe,KAAG,gBACoB,CAAC;AAEvE,MAAM,WAAW,gBAAgB;IAC/B,CAAC,GAAG,EAAE,QAAQ,GAAG,OAAO,CAAC;CAC1B;AAED,eAAO,MAAM,YAAY,GAAI,IAAI,gBAAgB,KAAG,iCAMjD,CAAC;AAGJ,MAAM,WAAW,sBAAsB;IACrC,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;CACtF;AAED,eAAO,MAAM,UAAU,GAAI,IAAI,sBAAsB,KAAG,iCAIrD,CAAC;AAEJ,MAAM,WAAW,mBAAmB;IAClC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAC;CAC7C;AAED,eAAO,MAAM,eAAe,GAAI,IAAI,mBAAmB,KAAG,iCAUvD,CAAC;AAEJ,eAAO,MAAM,cAAc,GAAI,IAAI,mBAAmB,KAAG,gCAMtD,CAAC;AAEJ,wBAAsB,oBAAoB,CACxC,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAclC;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,QAAQ,GAAG;QAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;KAAE,GAAG,aAAa,CAAC;CAChF;AAED,eAAO,MAAM,SAAS,GACpB,KAAK,gBAAgB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC,IAAI,GAAG,YAAY,CAAC,GAAG,CAAA;CAAE,EACtE,SAAS,qBAAqB,KAC7B,aAOF,CAAC;AAEF,eAAO,MAAM,WAAW,GAAI,IAAI,eAAe,KAAG,gCAS/C,CAAC;AAGJ,MAAM,WAAW,oBAAoB;IACnC,CAAC,CAAC,EAAE,aAAa,EAAE,GAAG,EAAE,QAAQ,GAAG;QAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;KAAE,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;CAC7F;AAED,eAAO,MAAM,gBAAgB,GAAI,IAAI,oBAAoB,KAAG,gCASzD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapters.js","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAE7B,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAS5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AASpC,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAmB,EAAoB,EAAE,CACnE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;AAMvE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAoB,EAAqC,EAAE,CACtF,GAAG,CAAC,EAAE;IACJ,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACZ,GAAG,CAAC,6BAA6B,GAAG,IAAI,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAOJ,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAA0B,EAAqC,EAAE,CAC1F,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;IACvB,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAMJ,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAuB,EAAqC,EAAE,CAC5F,GAAG,CAAC,EAAE;IACJ,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,MAAM,YAAY,GAAkB,EAAE,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;QAC3E,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEJ,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAuB,EAAoC,EAAE,CAC1F,GAAG,CAAC,EAAE;IACJ,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;QACtB,GAAG,CAAC,aAAa,GAAG,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;IAC9D,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEJ,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAE1B,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QACvD,MAAM,aAAa,GAAoB,MAAM,QAAQ,CAAC,wBAAwB,CAC5E,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC;YACtC,mEAAmE;YACnE,aAAa;YACb,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;gBAC3B,GAAG,CAAC,kBAAkB,GAAG,aAAa,CAAC,QAAQ,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAMD,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,
|
|
1
|
+
{"version":3,"file":"adapters.js","sourceRoot":"","sources":["../../src/life-cycle/adapters.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAE7B,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAS5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AASpC,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAmB,EAAoB,EAAE,CACnE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;AAMvE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAoB,EAAqC,EAAE,CACtF,GAAG,CAAC,EAAE;IACJ,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;QACZ,GAAG,CAAC,6BAA6B,GAAG,IAAI,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAOJ,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAA0B,EAAqC,EAAE,CAC1F,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE;IACvB,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAMJ,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAuB,EAAqC,EAAE,CAC5F,GAAG,CAAC,EAAE;IACJ,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QACrB,MAAM,YAAY,GAAkB,EAAE,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;QAC3E,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,YAAY,GAAG,YAAY,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEJ,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAuB,EAAoC,EAAE,CAC1F,GAAG,CAAC,EAAE;IACJ,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;QACtB,GAAG,CAAC,aAAa,GAAG,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC;IAC9D,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEJ,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAE1B,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QACvD,MAAM,aAAa,GAAoB,MAAM,QAAQ,CAAC,wBAAwB,CAC5E,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACrD,IAAI,aAAa,EAAE,CAAC;YAClB,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC;YACtC,mEAAmE;YACnE,aAAa;YACb,IAAI,aAAa,CAAC,QAAQ,EAAE,CAAC;gBAC3B,GAAG,CAAC,kBAAkB,GAAG,aAAa,CAAC,QAAQ,CAAC;YAClD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAMD,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,GAAsE,EACtE,OAA8B,EACf,EAAE;IACjB,MAAM,QAAQ,GACZ,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC;IACvD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,EAAmB,EAAoC,EAAE,CACnF,CAAC,GAAqB,EAAE,MAA0B,EAAE,OAA8B,EAAE,EAAE;IACpF,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAClB,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAOJ,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,EAAwB,EAAoC,EAAE,CAC7F,KAAK,EAAE,GAAqB,EAAE,MAA0B,EAAE,OAA8B,EAAE,EAAE;IAC1F,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YAClB,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-resource.d.ts","sourceRoot":"","sources":["../../src/life-cycle/download-resource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAE,WAAW,EAAgB,QAAQ,EAAC,MAAM,KAAK,CAAC;AAE9E,OAAO,KAAK,EAAC,gBAAgB,EAAE,cAAc,EAAC,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE7C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AAKzD,6BAA6B;AAC7B,eAAO,MAAM,eAAe,EAAE,eAqB7B,CAAC;AAEF,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,KAAK,CAAC;IACnD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"download-resource.d.ts","sourceRoot":"","sources":["../../src/life-cycle/download-resource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAE,WAAW,EAAgB,QAAQ,EAAC,MAAM,KAAK,CAAC;AAE9E,OAAO,KAAK,EAAC,gBAAgB,EAAE,cAAc,EAAC,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE7C,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AAKzD,6BAA6B;AAC7B,eAAO,MAAM,eAAe,EAAE,eAqB7B,CAAC;AAEF,MAAM,WAAW,aAAc,SAAQ,OAAO,CAAC,KAAK,CAAC;IACnD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,CA+C3C;AAED,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,QAAQ,GAAG;IAAE,sBAAsB,EAAE,MAAM,CAAA;CAAE,EAClD,cAAc,EAAE,cAAc,EAC9B,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,gBAAgB,GAAG,QAAQ,GAAG,IAAI,CAAC,CA8C7C;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,QAAQ,EACb,cAAc,EAAE,cAAc,EAC9B,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,gBAAgB,GAAG,QAAQ,GAAG,IAAI,CAAC,CAsE7C"}
|
|
@@ -48,6 +48,9 @@ export async function getRetry(url, options) {
|
|
|
48
48
|
// force cast for typescript 4.4
|
|
49
49
|
err = e;
|
|
50
50
|
if (err && err.message === 'premature close') {
|
|
51
|
+
if (err.retryLimitExceeded) {
|
|
52
|
+
throw e;
|
|
53
|
+
}
|
|
51
54
|
logger.retry.warn(i, url, 'manually retry on premature close', err.name, err.code, err.event, err.message);
|
|
52
55
|
await sleep(i * 200);
|
|
53
56
|
continue;
|
|
@@ -73,7 +76,13 @@ export async function getRetry(url, options) {
|
|
|
73
76
|
return res;
|
|
74
77
|
}
|
|
75
78
|
export async function requestForResource(res, requestOptions, options) {
|
|
76
|
-
|
|
79
|
+
let downloadLink;
|
|
80
|
+
try {
|
|
81
|
+
downloadLink = encodeURI(decodeURI(res.downloadLink));
|
|
82
|
+
}
|
|
83
|
+
catch (_a) {
|
|
84
|
+
downloadLink = res.downloadLink;
|
|
85
|
+
}
|
|
77
86
|
const reqOptions = Object.assign({}, requestOptions);
|
|
78
87
|
reqOptions.responseType = 'buffer';
|
|
79
88
|
if (res.refUrl && res.refUrl !== downloadLink) {
|
|
@@ -138,9 +147,9 @@ export async function downloadResource(res, requestOptions, options) {
|
|
|
138
147
|
else if (Array.isArray(contentType)) {
|
|
139
148
|
nonHtml = true;
|
|
140
149
|
for (const header of contentType) {
|
|
141
|
-
if (
|
|
142
|
-
|
|
143
|
-
|
|
150
|
+
if (header.includes('/html') ||
|
|
151
|
+
header.includes('/xml') ||
|
|
152
|
+
header.includes('application/xhtml+xml')) {
|
|
144
153
|
nonHtml = false;
|
|
145
154
|
break;
|
|
146
155
|
}
|
|
@@ -156,11 +165,13 @@ export async function downloadResource(res, requestOptions, options) {
|
|
|
156
165
|
Buffer.isBuffer(downloadedResource.body))) {
|
|
157
166
|
if (!downloadedResource.body.includes(options.meta.detectIncompleteHtml)) {
|
|
158
167
|
logger.error.info('Detected incomplete html, try again', downloadedResource.downloadLink);
|
|
159
|
-
downloadedResource = await requestForResource(res, requestOptions);
|
|
168
|
+
downloadedResource = await requestForResource(res, requestOptions, options);
|
|
160
169
|
}
|
|
161
170
|
// probably more retries here?
|
|
162
|
-
if (!downloadedResource ||
|
|
163
|
-
|
|
171
|
+
if (!downloadedResource ||
|
|
172
|
+
(typeof downloadedResource.body === 'string' ||
|
|
173
|
+
Buffer.isBuffer(downloadedResource.body)) &&
|
|
174
|
+
!downloadedResource.body.includes(options.meta.detectIncompleteHtml)) {
|
|
164
175
|
logger.error.warn('Detected incomplete html twice', res.downloadLink);
|
|
165
176
|
return downloadedResource;
|
|
166
177
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-resource.js","sourceRoot":"","sources":["../../src/life-cycle/download-resource.ts"],"names":[],"mappings":"AACA,OAAO,GAAG,EAAE,EAAC,YAAY,EAAC,MAAM,KAAK,CAAC;AAGtC,OAAO,EAAC,gBAAgB,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAE9D,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAC,SAAS,EAAE,KAAK,EAAC,MAAM,YAAY,CAAC;AAC5C,OAAO,GAAG,MAAM,OAAO,CAAC;AAExB,6BAA6B;AAC7B,MAAM,CAAC,MAAM,eAAe,GAAoB,CAC9C,KAAmB,EACnB,UAA8B,EAC9B,EAAE;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACnE,CAAC,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EACzD,KAAK,CAAC,OAAO,EAAG,KAAsB,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,CAAC,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChF,CAAC;AACH,CAAC,CAAC;AAQF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAoB;IAEpB,IAAI,GAAG,GAAqC,KAAK,CAAC,CAAC;IACnD,IAAI,GAAG,GAAyB,KAAK,CAAC,EAAE,YAAyB,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,GAAG,GAAG,KAAK,CAAC,CAAC;QACb,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC1C,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAA8B,CAAC;YAClE,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,0CAA0C,EAClE,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnB,SAAS;YACX,CAAC;YACD,MAAM;QACR,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,gCAAgC;YAChC,GAAG,GAAG,CAAyB,CAAC;YAChC,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,KAAK,iBAAiB,EAAE,CAAC;gBAC7C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAC3D,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,+CAA+C;YAC/C,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB;gBAChC,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;gBAC5D,4DAA4D;gBAC5D,4EAA4E;gBAC5E,2BAA2B;gBAC3B,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;gBACvD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,qBAAqB,GAAG,CAAC,KAAK,UAAU,EAChE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACnC,MAAM,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,IAAI,GAAG,EAAE,CAAC;QACR,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,+CAA+C,EACrE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAkD,EAClD,cAA8B,EAC9B,OAA+B;IAE/B,
|
|
1
|
+
{"version":3,"file":"download-resource.js","sourceRoot":"","sources":["../../src/life-cycle/download-resource.ts"],"names":[],"mappings":"AACA,OAAO,GAAG,EAAE,EAAC,YAAY,EAAC,MAAM,KAAK,CAAC;AAGtC,OAAO,EAAC,gBAAgB,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAE9D,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAC,SAAS,EAAE,KAAK,EAAC,MAAM,YAAY,CAAC;AAC5C,OAAO,GAAG,MAAM,OAAO,CAAC;AAExB,6BAA6B;AAC7B,MAAM,CAAC,MAAM,eAAe,GAAoB,CAC9C,KAAmB,EACnB,UAA8B,EAC9B,EAAE;IACF,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;QACnE,CAAC,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EACzD,KAAK,CAAC,OAAO,EAAG,KAAsB,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,CAAC,UAAU,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;aACnE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAChF,CAAC;AACH,CAAC,CAAC;AAQF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,OAAoB;IAEpB,IAAI,GAAG,GAAqC,KAAK,CAAC,CAAC;IACnD,IAAI,GAAG,GAAyB,KAAK,CAAC,EAAE,YAAyB,CAAC;IAClE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,GAAG,GAAG,KAAK,CAAC,CAAC;QACb,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC1C,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAA8B,CAAC;YAClE,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC1C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,0CAA0C,EAClE,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;gBACnB,SAAS;YACX,CAAC;YACD,MAAM;QACR,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,gCAAgC;YAChC,GAAG,GAAG,CAAyB,CAAC;YAChC,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,KAAK,iBAAiB,EAAE,CAAC;gBAC7C,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;oBAC3B,MAAM,CAAC,CAAC;gBACV,CAAC;gBACD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,mCAAmC,EAC3D,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,+CAA+C;YAC/C,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,kBAAkB;gBAChC,CAAC,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;gBAC5D,4DAA4D;gBAC5D,4EAA4E;gBAC5E,2BAA2B;gBAC3B,CAAC,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;gBACvD,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,qBAAqB,GAAG,CAAC,KAAK,UAAU,EAChE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACnC,MAAM,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,IAAI,GAAG,EAAE,CAAC;QACR,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,+CAA+C,EACrE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAkD,EAClD,cAA8B,EAC9B,OAA+B;IAE/B,IAAI,YAAoB,CAAC;IACzB,IAAI,CAAC;QACH,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;IACxD,CAAC;IAAC,WAAM,CAAC;QACP,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC;IAClC,CAAC;IACD,MAAM,UAAU,GAAgB,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;IAClE,UAAU,CAAC,YAAY,GAAG,QAAQ,CAAC;IACnC,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,CAAC,OAAO,CAAC,CAAC;QACtD,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC;QAC7B,UAAU,CAAC,OAAO,GAAG,OAAO,CAAC;IAC/B,CAAC;IACD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,CAAC,MAAM,EACnD,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC1B,MAAM,QAAQ,GACZ,MAAM,QAAQ,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;IAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,GAAe,CAAC;QACjC,OAAO,QAAQ,CAAC,sBAAsB,CAAC;QACvC,OAAO,QAAQ,CAAC,QAAQ,CAAC;QACzB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;QAClE,OAAO,GAAe,CAAC;IACzB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;IAEpC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,GAAG,EACpE,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACpD,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,sBAAsB,CAAC;IACpE,GAAG,CAAC,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC;IACjC,mEAAmE;IACnE,aAAa;IACb,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QAClC,GAAG,CAAC,kBAAkB,GAAG,gBAAgB,CACvC,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,EACtB,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAC9B,CAAC,CAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,sBAAsB,CAAA,EAChC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,YAAY,CAAC,CAAC;IAC3B,CAAC;IACD,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IACzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,GAAa,EACb,cAA8B,EAC9B,OAA8B;IAE9B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,GAAuB,CAAC;IACjC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;QAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;IAClE,CAAC;IACD,IAAI,kBAAkB,GAAuC,MAAM,kBAAkB,CACnF,GAAsD,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;IACnF,IAAI,CAAC,kBAAkB,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,CAAC;QACpD,OAAO,kBAAkB,CAAC;IAC5B,CAAC;IACD,IAAI,kBAAkB,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QAClD,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC;YAChD,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,WAAW,GACf,OAAO,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,cAAc,CAAC,CAAC;gBACrD,IAAI,OAAO,GAAG,KAAK,CAAC;gBACpB,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE,CAAC;oBACpC,OAAO,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;wBACtC,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC;wBAC7B,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;gBACnD,CAAC;qBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;oBACtC,OAAO,GAAG,IAAI,CAAC;oBACf,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;wBACjC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;4BAC1B,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;4BACvB,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;4BAC3C,OAAO,GAAG,KAAK,CAAC;4BAChB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,gCAAgC,EAChD,kBAAkB,CAAC,YAAY,EAAE,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,CAAC,oBAAoB;YACnC,CAAC,OAAO,kBAAkB,CAAC,IAAI,KAAK,QAAQ;gBAC1C,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAC9C,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACzE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,EACrD,kBAAkB,CAAC,YAAY,CAAC,CAAC;gBACnC,kBAAkB,GAAG,MAAM,kBAAkB,CAC3C,GAAsD,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;YACrF,CAAC;YACD,8BAA8B;YAC9B,IAAI,CAAC,kBAAkB;gBACrB,CAAC,OAAO,kBAAkB,CAAC,IAAI,KAAK,QAAQ;oBAC1C,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;oBAC3C,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,gCAAgC,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC;gBACtE,OAAO,kBAAkB,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,kBAAkB,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChD,kBAAkB,CAAC,YAAY;YAC7B,kBAAkB,CAAC,eAAe,GAAG,GAAG,CAAC,sBAAsB,CAAC;IACpE,CAAC;IACD,OAAO,kBAAkB,CAAC;AAC5B,CAAC"}
|
|
@@ -170,7 +170,7 @@ export async function streamingDownloadToFile(res, requestOptions) {
|
|
|
170
170
|
retry = retryCount * 300;
|
|
171
171
|
}
|
|
172
172
|
if (retry > 0) {
|
|
173
|
-
makeRequest(retryCount + 1);
|
|
173
|
+
setTimeout(() => makeRequest(retryCount + 1), retry);
|
|
174
174
|
return;
|
|
175
175
|
}
|
|
176
176
|
}
|
|
@@ -191,7 +191,7 @@ export async function optionallySetLastModifiedTime(res, options) {
|
|
|
191
191
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
192
192
|
let mtime = void 0;
|
|
193
193
|
if (options.preferRemoteLastModifiedTime && ((_b = (_a = res.meta) === null || _a === void 0 ? void 0 : _a.headers) === null || _b === void 0 ? void 0 : _b['last-modified'])) {
|
|
194
|
-
mtime = Date.parse((_c = res.meta.headers) === null || _c === void 0 ? void 0 : _c['last-modified']);
|
|
194
|
+
mtime = Date.parse((_c = res.meta.headers) === null || _c === void 0 ? void 0 : _c['last-modified']) / 1000;
|
|
195
195
|
}
|
|
196
196
|
// void and NaN check
|
|
197
197
|
if (mtime) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download-streaming-resource.js","sourceRoot":"","sources":["../../src/life-cycle/download-streaming-resource.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAC,SAAS,EAAE,iBAAiB,EAAE,QAAQ,IAAI,EAAE,EAAC,MAAM,SAAS,CAAC;AACrE,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AACrC,OAAO,EAAC,SAAS,EAAC,MAAM,WAAW,CAAC;AAEpC,OAAO,GAAG,EAAE,EAAC,SAAS,EAAC,MAAM,KAAK,CAAC;AAEnC,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAO5C,OAAO,EAAC,UAAU,EAAC,MAAM,UAAU,CAAC;AACpC,OAAO,EAAC,KAAK,IAAI,WAAW,EAAE,KAAK,IAAI,WAAW,EAAC,MAAM,qBAAqB,CAAC;AAG/E,OAAO,EAAC,SAAS,EAAC,MAAM,YAAY,CAAC;AAErC,MAAM,iBAAiB,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,WAAoB;IAClD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,YAAqB;IACxE,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAkD,EAClD,cAA8B;IAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,gCAAgC;QAChC,IAAI,CAAC,IAAK,CAA4B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,4BAA4B;IAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE;QAChD,QAAQ,EAAE,IAAI;KACf,CAEA,CAAC;IACF,IAAI,eAAmC,CAAC;IAExC,OAAO,IAAI,OAAO,CAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC/C,IAAI,gBAAgC,CAAC;QACrC,IAAI,UAAyB,CAAC;QAC9B,MAAM,WAAW,GAAG,CAAC,UAAkB,EAAQ,EAAE;YAC/C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,gBAAgB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzC,UAAU,GAAG,SAAS,CAAC;gBACvB,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,UAAU,IAAI,gBAAgB,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,SAAS,UAAU,GAAG,CAAC;gBAC/C,eAAe,GAAG,iBAAiB,CAAC,QAAQ,EAAE;oBAC5C,KAAK,EAAE,GAAG;oBACV,KAAK,EAAE,UAAU;iBAClB,CAAC,CAAC;YACL,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;YAEhC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,QAAkB,EAAE,EAAE;gBACpD,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;gBACpC,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;oBACnC,IAAI,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;wBACvD,gBAAgB,GAAG,IAAI,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBACD,IAAI,gBAAgB,IAAI,eAAe,IAAI,UAAU;oBACnD,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG;wBAC1B,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtE,WAAW,CAAC,IAAI,CAAC,+BAA+B,EAC9C,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;oBACtE,gBAAgB,GAAG,KAAK,CAAC;oBACzB,eAAe,CAAC,OAAO,EAAE,CAAC;oBAC1B,eAAe,GAAG,SAAS,CAAC;oBAC5B,UAAU,GAAG,SAAS,CAAC;gBACzB,CAAC;gBAED,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;oBAC/B,6BAA6B;oBAC7B,sDAAsD;oBACtD,OAAO;gBACT,CAAC;gBACD,gBAAgB;gBAChB,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,iBAAiB,CAAC,QAAQ,EAAE;wBAC5C,KAAK,EAAE,GAAG;qBACX,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC;oBACH,iCAAiC;oBACjC,MAAM,iBAAiB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;gBACpD,CAAC;gBAAC,WAAM,CAAC;oBACP,kCAAkC;oBAClC,4BAA4B;oBAC5B,OAAO;gBACT,CAAC;gBAED,iCAAiC;gBACjC,YAAY;gBACZ,IAAI;gBAEJ,OAAO,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,GAAG,EAAE;gBACzB,IAAI,eAAe,EAAE,CAAC;oBACpB,IAAI,gBAAgB,EAAE,CAAC;wBACrB,IAAI,UAAU,EAAE,CAAC;4BACf,UAAU,IAAI,eAAe,CAAC,YAAY,CAAC;wBAC7C,CAAC;6BAAM,CAAC;4BACN,UAAU,GAAG,eAAe,CAAC,YAAY,CAAC;wBAC5C,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,UAAU,GAAG,SAAS,CAAC;oBACzB,CAAC;oBACD,eAAe,CAAC,OAAO,EAAE,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACN,UAAU,GAAG,SAAS,CAAC;gBACzB,CAAC;gBACD,eAAe,GAAG,SAAS,CAAC;YAC9B,CAAC,CAAC;YAEF,MAAM,OAAO,GAAG,CAAC,KAAmB,EAAE,EAAE;gBACtC,4DAA4D;gBAC5D,yDAAyD;gBACzD,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACpE,WAAW,CAAC,IAAI,CAAC,+BAA+B,EAC9C,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EACnD,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;oBAC7B,gBAAgB,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,EAAC,OAAO,EAAC,GAAG,OAAO,CAAC;gBAE1B,IAAI,KAAK,YAAY,SAAS,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC3D,MAAM,EAAC,QAAQ,EAAC,GAAG,KAAK,CAAC;oBACzB,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAClB,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;oBACf,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,iBAAiB,EAAE,CAAC;wBACjD,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAC3C,mCAAmC;wBACnC,8DAA8D;wBAC9D,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAG,KAAa,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC/D,KAAK,GAAG,UAAU,GAAG,GAAG,CAAC;oBAC3B,CAAC;oBACD,+CAA+C;oBAC/C,8DAA8D;oBAC9D,IAAI,KAAK,IAAI,CAAE,KAAa,CAAC,kBAAkB;wBAC7C,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;wBAChE,4DAA4D;wBAC5D,4EAA4E;wBAC5E,2BAA2B;wBAC3B,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;wBAC3D,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY;wBAC3C,8DAA8D;wBAC9D,qBAAsB,KAAa,CAAC,KAAK,UAAU,EACnD,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;wBACzC,KAAK,GAAG,UAAU,GAAG,GAAG,CAAC;oBAC3B,CAAC;oBACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;wBACd,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"download-streaming-resource.js","sourceRoot":"","sources":["../../src/life-cycle/download-streaming-resource.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAC,SAAS,EAAE,iBAAiB,EAAE,QAAQ,IAAI,EAAE,EAAC,MAAM,SAAS,CAAC;AACrE,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAC;AACrC,OAAO,EAAC,SAAS,EAAC,MAAM,WAAW,CAAC;AAEpC,OAAO,GAAG,EAAE,EAAC,SAAS,EAAC,MAAM,KAAK,CAAC;AAEnC,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAO5C,OAAO,EAAC,UAAU,EAAC,MAAM,UAAU,CAAC;AACpC,OAAO,EAAC,KAAK,IAAI,WAAW,EAAE,KAAK,IAAI,WAAW,EAAC,MAAM,qBAAqB,CAAC;AAG/E,OAAO,EAAC,SAAS,EAAC,MAAM,YAAY,CAAC;AAErC,MAAM,iBAAiB,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,WAAoB;IAClD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,UAAkB,EAAE,YAAqB;IACxE,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,UAAU,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAkD,EAClD,cAA8B;IAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,gCAAgC;QAChC,IAAI,CAAC,IAAK,CAA4B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzD,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,4BAA4B;IAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE;QAChD,QAAQ,EAAE,IAAI;KACf,CAEA,CAAC;IACF,IAAI,eAAmC,CAAC;IAExC,OAAO,IAAI,OAAO,CAAW,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC/C,IAAI,gBAAgC,CAAC;QACrC,IAAI,UAAyB,CAAC;QAC9B,MAAM,WAAW,GAAG,CAAC,UAAkB,EAAQ,EAAE;YAC/C,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC,gBAAgB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;gBACzC,UAAU,GAAG,SAAS,CAAC;gBACvB,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC;YAC/B,CAAC;iBAAM,IAAI,UAAU,IAAI,gBAAgB,EAAE,CAAC;gBAC1C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACrB,OAAO,CAAC,OAAO,GAAG,EAAE,CAAC;gBACvB,CAAC;gBACD,OAAO,CAAC,OAAO,CAAC,KAAK,GAAG,SAAS,UAAU,GAAG,CAAC;gBAC/C,eAAe,GAAG,iBAAiB,CAAC,QAAQ,EAAE;oBAC5C,KAAK,EAAE,GAAG;oBACV,KAAK,EAAE,UAAU;iBAClB,CAAC,CAAC;YACL,CAAC;YACD,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO,CAAC,UAAU,GAAG,UAAU,CAAC;YAEhC,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,QAAkB,EAAE,EAAE;gBACpD,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;gBACpC,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;oBACnC,IAAI,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;wBACvD,gBAAgB,GAAG,IAAI,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBACD,IAAI,gBAAgB,IAAI,eAAe,IAAI,UAAU;oBACnD,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG;wBAC1B,CAAC,gBAAgB,CAAC,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtE,WAAW,CAAC,IAAI,CAAC,+BAA+B,EAC9C,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;oBACtE,gBAAgB,GAAG,KAAK,CAAC;oBACzB,eAAe,CAAC,OAAO,EAAE,CAAC;oBAC1B,eAAe,GAAG,SAAS,CAAC;oBAC5B,UAAU,GAAG,SAAS,CAAC;gBACzB,CAAC;gBAED,IAAI,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;oBAC/B,6BAA6B;oBAC7B,sDAAsD;oBACtD,OAAO;gBACT,CAAC;gBACD,gBAAgB;gBAChB,IAAI,CAAC,eAAe,EAAE,CAAC;oBACrB,eAAe,GAAG,iBAAiB,CAAC,QAAQ,EAAE;wBAC5C,KAAK,EAAE,GAAG;qBACX,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC;oBACH,iCAAiC;oBACjC,MAAM,iBAAiB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;gBACpD,CAAC;gBAAC,WAAM,CAAC;oBACP,kCAAkC;oBAClC,4BAA4B;oBAC5B,OAAO;gBACT,CAAC;gBAED,iCAAiC;gBACjC,YAAY;gBACZ,IAAI;gBAEJ,OAAO,CAAC,QAAQ,CAAC,CAAC;YACpB,CAAC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,GAAG,EAAE;gBACzB,IAAI,eAAe,EAAE,CAAC;oBACpB,IAAI,gBAAgB,EAAE,CAAC;wBACrB,IAAI,UAAU,EAAE,CAAC;4BACf,UAAU,IAAI,eAAe,CAAC,YAAY,CAAC;wBAC7C,CAAC;6BAAM,CAAC;4BACN,UAAU,GAAG,eAAe,CAAC,YAAY,CAAC;wBAC5C,CAAC;oBACH,CAAC;yBAAM,CAAC;wBACN,UAAU,GAAG,SAAS,CAAC;oBACzB,CAAC;oBACD,eAAe,CAAC,OAAO,EAAE,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACN,UAAU,GAAG,SAAS,CAAC;gBACzB,CAAC;gBACD,eAAe,GAAG,SAAS,CAAC;YAC9B,CAAC,CAAC;YAEF,MAAM,OAAO,GAAG,CAAC,KAAmB,EAAE,EAAE;gBACtC,4DAA4D;gBAC5D,yDAAyD;gBACzD,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBACpE,WAAW,CAAC,IAAI,CAAC,+BAA+B,EAC9C,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EACnD,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;oBAC7B,gBAAgB,GAAG,KAAK,CAAC;gBAC3B,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,EAAC,OAAO,EAAC,GAAG,OAAO,CAAC;gBAE1B,IAAI,KAAK,YAAY,SAAS,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;oBAC3D,MAAM,EAAC,QAAQ,EAAC,GAAG,KAAK,CAAC;oBACzB,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAClB,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;oBACf,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,KAAK,iBAAiB,EAAE,CAAC;wBACjD,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY,EAC3C,mCAAmC;wBACnC,8DAA8D;wBAC9D,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAG,KAAa,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;wBAC/D,KAAK,GAAG,UAAU,GAAG,GAAG,CAAC;oBAC3B,CAAC;oBACD,+CAA+C;oBAC/C,8DAA8D;oBAC9D,IAAI,KAAK,IAAI,CAAE,KAAa,CAAC,kBAAkB;wBAC7C,CAAC,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;wBAChE,4DAA4D;wBAC5D,4EAA4E;wBAC5E,2BAA2B;wBAC3B,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;wBAC3D,WAAW,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,YAAY;wBAC3C,8DAA8D;wBAC9D,qBAAsB,KAAa,CAAC,KAAK,UAAU,EACnD,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;wBACzC,KAAK,GAAG,UAAU,GAAG,GAAG,CAAC;oBAC3B,CAAC;oBACD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;wBACd,UAAU,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;wBACrD,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC;YAEF,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAE/B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,aAAqB,EAAE,EAAE;gBAC9C,aAAa,EAAE,CAAC;gBAChB,OAAO,GAAG,IAAI,CAAC;gBACf,WAAW,CAAC,aAAa,CAAC,CAAC;YAC7B,CAAC,CAAC,CAAC;QAEL,CAAC,CAAC;QAEF,WAAW,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,GAAa,EAAE,OAA8B;;IAE7C,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC;IACjE,CAAC;IAED,qBAAqB;IACrB,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC;YACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,WAAW,CAAC,IAAI,CAAC,kBAAkB,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,GAAa,EACb,cAA8B,EAC9B,OAA8B;IAE9B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,GAAuB,CAAC;IACjC,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;QAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;IAClE,CAAC;IACD,MAAM,uBAAuB,CAC3B,GAAsD,EAAE,cAAc,CAAC,CAAC;IAE1E,MAAM,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAClD,qBAAqB;IACrB,oCAAoC;IACpC,qBAAqB;IACrB,sDAAsD;IACtD,OAAO;AACT,CAAC;AA+CD,MAAM,UAAU,iCAAiC,CAC/C,cAA4C,EAC5C,aAA0C,EAC1C,aAA0C;IAE1C,IAAI,CAAC,cAAc,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa,EAAE,CAAC;QACxD,OAAO,yBAAyB,CAAC;IACnC,CAAC;IACD,OAAO,KAAK,EACV,GAAa,EACb,cAA8B,EAC9B,OAA8B,EAC9B,QAA0B,EAC1B,EAAE;QACF,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACb,OAAO,GAAuB,CAAC;QACjC,CAAC;QACD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,eAAe,EAAE,CAAC;YAC9C,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YACjC,OAAO,GAAG,CAAC;QACb,CAAC;QACD,IAAI,QAAQ,GAAuC,GAAG,CAAC;QACvD,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACxE,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO;YACT,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClB,OAAO,QAA4B,CAAC;YACtC,CAAC;YACD,IAAI,QAAQ,CAAC,6BAA6B,EAAE,CAAC;gBAC3C,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;YAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;QAClE,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,uBAAuB,CAC3B,GAAsD,EAAE,cAAc,CAAC,CAAC;YAC1E,MAAM,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,uBAAuB,CAC3B,GAAsD,EAAE,cAAc,CAAC,CAAC;gBAC1E,MAAM,6BAA6B,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,aAAa,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QACD,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,GAAG,CAAC,YAAY;YACd,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,sBAAsB,CAAC;QACnD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,aAAa,CAAC,GAAG,EAAE,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO;IACT,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-css.d.ts","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAE,kBAAkB,EAAC,MAAM,YAAY,CAAC;AACrE,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAG7C,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"process-css.d.ts","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,gBAAgB,EAAE,kBAAkB,EAAC,MAAM,YAAY,CAAC;AACrE,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAG7C,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAsB,cAAc,CAClC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CA0CxC;AAED,wBAAsB,UAAU,CAC9B,GAAG,EAAE,gBAAgB,EACrB,MAAM,EAAE,kBAAkB,EAC1B,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAa9D"}
|
|
@@ -3,6 +3,10 @@ import { ResourceType } from '../resource.js';
|
|
|
3
3
|
import { toString } from '../util.js';
|
|
4
4
|
export async function processCssText(cssText, res, options, pipeline, depth, resources) {
|
|
5
5
|
const cssUrls = parseCssUrls(cssText);
|
|
6
|
+
if (!cssUrls.length)
|
|
7
|
+
return cssText;
|
|
8
|
+
// Phase 1: process URLs and collect replacements
|
|
9
|
+
const replacements = [];
|
|
6
10
|
let rawUrl, r;
|
|
7
11
|
// noinspection DuplicatedCode
|
|
8
12
|
for (let i = 0, l = cssUrls.length; i < l; i++) {
|
|
@@ -13,9 +17,36 @@ export async function processCssText(cssText, res, options, pipeline, depth, res
|
|
|
13
17
|
if (!r.shouldBeDiscardedFromDownload) {
|
|
14
18
|
resources.push(r);
|
|
15
19
|
}
|
|
16
|
-
|
|
20
|
+
if (rawUrl !== r.replacePath) {
|
|
21
|
+
replacements.push([rawUrl, r.replacePath]);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
if (!replacements.length)
|
|
25
|
+
return cssText;
|
|
26
|
+
// Phase 2: single-pass positional replacement to avoid
|
|
27
|
+
// corrupting already-replaced paths
|
|
28
|
+
const occurrences = [];
|
|
29
|
+
for (const [url, replacePath] of replacements) {
|
|
30
|
+
let from = 0, pos;
|
|
31
|
+
while ((pos = cssText.indexOf(url, from)) !== -1) {
|
|
32
|
+
occurrences.push([pos, url.length, replacePath]);
|
|
33
|
+
from = pos + url.length;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
if (!occurrences.length)
|
|
37
|
+
return cssText;
|
|
38
|
+
occurrences.sort((a, b) => a[0] - b[0]);
|
|
39
|
+
const parts = [];
|
|
40
|
+
let lastEnd = 0;
|
|
41
|
+
for (const [pos, len, replacePath] of occurrences) {
|
|
42
|
+
if (pos < lastEnd)
|
|
43
|
+
continue;
|
|
44
|
+
parts.push(cssText.slice(lastEnd, pos));
|
|
45
|
+
parts.push(replacePath);
|
|
46
|
+
lastEnd = pos + len;
|
|
17
47
|
}
|
|
18
|
-
|
|
48
|
+
parts.push(cssText.slice(lastEnd));
|
|
49
|
+
return parts.join('');
|
|
19
50
|
}
|
|
20
51
|
export async function processCss(res, submit, options, pipeline) {
|
|
21
52
|
if (res.type !== ResourceType.Css) {
|
|
@@ -28,7 +59,7 @@ export async function processCss(res, submit, options, pipeline) {
|
|
|
28
59
|
cssText = await processCssText(cssText, res, options, pipeline, depth, resources);
|
|
29
60
|
res.body = cssText;
|
|
30
61
|
res.meta.cssProcessed = 1;
|
|
31
|
-
|
|
62
|
+
submit(resources);
|
|
32
63
|
return res;
|
|
33
64
|
}
|
|
34
65
|
//# sourceMappingURL=process-css.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-css.js","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,gBAAgB,CAAC;AAI1C,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AAGpC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,GAAqB,EACrB,OAA8B,EAC9B,QAA0B,EAC1B,KAAa,EACb,SAAqB;IACrB,MAAM,OAAO,GAAa,YAAY,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,MAAc,EAAE,CAAkB,CAAC;IACvC,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC,GAAG,MAAM,QAAQ,CAAC,wBAAwB,CACzC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"process-css.js","sourceRoot":"","sources":["../../src/life-cycle/process-css.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,gBAAgB,CAAC;AAI1C,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AAGpC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAe,EACf,GAAqB,EACrB,OAA8B,EAC9B,QAA0B,EAC1B,KAAa,EACb,SAAqB;IACrB,MAAM,OAAO,GAAa,YAAY,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IACpC,iDAAiD;IACjD,MAAM,YAAY,GAA4B,EAAE,CAAC;IACjD,IAAI,MAAc,EAAE,CAAkB,CAAC;IACvC,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC,GAAG,MAAM,QAAQ,CAAC,wBAAwB,CACzC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;QACD,IAAI,MAAM,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IACzC,uDAAuD;IACvD,oCAAoC;IACpC,MAAM,WAAW,GAAoC,EAAE,CAAC;IACxD,KAAK,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,YAAY,EAAE,CAAC;QAC9C,IAAI,IAAI,GAAG,CAAC,EAAE,GAAW,CAAC;QAC1B,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACjD,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;YACjD,IAAI,GAAG,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IACxC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,WAAW,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,IAAI,GAAG,GAAG,OAAO;YAAE,SAAS;QAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACxB,OAAO,GAAG,GAAG,GAAG,GAAG,CAAC;IACtB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IACnC,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,GAAG,EAAE,CAAC;QAClC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAW,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;IACpC,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,OAAO,GAAW,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ;QACnD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACtC,OAAO,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAClF,GAAG,CAAC,IAAI,GAAG,OAAO,CAAC;IACnB,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;IAC1B,MAAM,CAAC,SAAS,CAAC,CAAC;IAClB,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"process-site-map.js","sourceRoot":"","sources":["../../src/life-cycle/process-site-map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAI7B,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AAIpC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,OAAO,EAAE,CAAC;QACtC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,CAAC,GAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAC7C,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,MAAM,GAAgB,IAAI,GAAG,EAAE,CAAC;IACtC,MAAM,KAAK,GAAW,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;IACpC,0CAA0C;IAC1C,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACtC,IAAI,GAAG,GAAW,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,GAAkB,EAAE,CAAkB,CAAC;IAC3C,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACd,CAAC,GAAG,MAAM,QAAQ,CAAC,wBAAwB,CACzC,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,MAAM,
|
|
1
|
+
{"version":3,"file":"process-site-map.js","sourceRoot":"","sources":["../../src/life-cycle/process-site-map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,IAAI,EAAC,MAAM,SAAS,CAAC;AAI7B,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,YAAY,CAAC;AAIpC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAqB,EACrB,MAA0B,EAC1B,OAA8B,EAC9B,QAA0B;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,OAAO,EAAE,CAAC;QACtC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,CAAC,GAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAC7C,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC;IACrE,MAAM,MAAM,GAAgB,IAAI,GAAG,EAAE,CAAC;IACtC,MAAM,KAAK,GAAW,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC;IACpC,0CAA0C;IAC1C,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QACtC,IAAI,GAAG,GAAW,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,GAAa,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,GAAkB,EAAE,CAAkB,CAAC;IAC3C,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACd,CAAC,GAAG,MAAM,QAAQ,CAAC,wBAAwB,CACzC,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,IAAI,CAAC,CAAC,CAAC,6BAA6B,EAAE,CAAC;YACrC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,MAAM,CAAC,SAAS,CAAC,CAAC;IAClB,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read-or-copy-local-resource.d.ts","sourceRoot":"","sources":["../../src/life-cycle/read-or-copy-local-resource.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE7C,OAAO,KAAK,EAAC,gBAAgB,EAAE,cAAc,EAAC,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"read-or-copy-local-resource.d.ts","sourceRoot":"","sources":["../../src/life-cycle/read-or-copy-local-resource.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAC;AAE7C,OAAO,KAAK,EAAC,gBAAgB,EAAE,cAAc,EAAC,MAAM,YAAY,CAAC;AACjE,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AAMzD,wBAAsB,uBAAuB,CAC3C,GAAG,EAAE,QAAQ,EACb,cAAc,EAAE,cAAc,EAC9B,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,gBAAgB,GAAG,QAAQ,GAAG,IAAI,CAAC,CA2D7C"}
|
|
@@ -2,6 +2,7 @@ import * as path from 'node:path';
|
|
|
2
2
|
import { promises } from 'node:fs';
|
|
3
3
|
import { ResourceType } from '../resource.js';
|
|
4
4
|
import { error as errorLogger } from '../logger/logger.js';
|
|
5
|
+
import { mkdirRetry } from '../io.js';
|
|
5
6
|
const FILE_PREFIX = 'file://';
|
|
6
7
|
export async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
7
8
|
var _a;
|
|
@@ -35,6 +36,7 @@ export async function readOrCopyLocalResource(res, requestOptions, options) {
|
|
|
35
36
|
}
|
|
36
37
|
if (res.type === ResourceType.StreamingBinary) {
|
|
37
38
|
const fileDestPath = path.join((_a = res.localRoot) !== null && _a !== void 0 ? _a : options.localRoot, res.savePath);
|
|
39
|
+
await mkdirRetry(path.dirname(fileDestPath));
|
|
38
40
|
await promises.copyFile(fileSrcPath, fileDestPath);
|
|
39
41
|
}
|
|
40
42
|
else {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"read-or-copy-local-resource.js","sourceRoot":"","sources":["../../src/life-cycle/read-or-copy-local-resource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;AAEjC,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG5C,OAAO,EAAC,KAAK,IAAI,WAAW,EAAC,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"read-or-copy-local-resource.js","sourceRoot":"","sources":["../../src/life-cycle/read-or-copy-local-resource.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAElC,OAAO,EAAC,QAAQ,EAAC,MAAM,SAAS,CAAC;AAEjC,OAAO,EAAC,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAG5C,OAAO,EAAC,KAAK,IAAI,WAAW,EAAC,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAC,UAAU,EAAC,MAAM,UAAU,CAAC;AAEpC,MAAM,WAAW,GAAG,SAAS,CAAC;AAE9B,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAa,EACb,cAA8B,EAC9B,OAA8B;;IAE9B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;QACb,OAAO,GAAuB,CAAC;IACjC,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC;QAChC,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,eAAe,CAAC;IAClE,CAAC;IACD,IAAI,WAAW,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC7D,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IACD,sBAAsB;IACtB,IAAI,KAAK,GAAiB,KAAK,CAAC,CAAC;IACjC,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QACnC,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;gBAChD,IAAI,MAAM,QAAQ,CAAC,MAAM,CAAC,WAAW,GAAG,GAAG,GAAG,KAAK,CAAC;qBACjD,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;oBACvC,WAAW,IAAI,GAAG,GAAG,KAAK,CAAC;oBAC3B,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAM,YAAY,CAAC,eAAe,EAAE,CAAC;QAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QACjF,MAAM,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;QAC7C,MAAM,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACrD,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,WAAW,EAAE;YAC9C,QAAQ,EAAE,GAAG,CAAC,QAAQ;SACvB,CAAC,CAAC;IACL,CAAC;IACD,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG;gBACjB,eAAe,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;gBAC1C,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE;aACxC,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,WAAW,CAAC,IAAI,CAAC,OAAO,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,GAAG,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,GAAG,CAAC,YAAY;QACd,GAAG,CAAC,eAAe,GAAG,GAAG,CAAC,sBAAsB,CAAC;IAEnD,IAAI,GAAG,CAAC,IAAI,KAAM,YAAY,CAAC,eAAe,EAAE,CAAC;QAC/C,OAAO;IACT,CAAC;IACD,OAAO,GAAuB,CAAC;AACjC,CAAC"}
|
|
@@ -4,7 +4,7 @@ import type { ResourceBody, ResourceEncoding } from '../resource.js';
|
|
|
4
4
|
import { ResourceType } from '../resource.js';
|
|
5
5
|
import type { PipelineExecutor } from './pipeline-executor.js';
|
|
6
6
|
export declare function getResourceBodyFromHtml(res: DownloadResource & {
|
|
7
|
-
type: ResourceType.Html;
|
|
7
|
+
type: ResourceType.Html | ResourceType.Svg;
|
|
8
8
|
}, options: StaticDownloadOptions): ResourceBody;
|
|
9
9
|
export declare function redirectHtml(relativePath: string, encoding?: ResourceEncoding): string;
|
|
10
10
|
export declare function saveHtmlToDisk(res: DownloadResource, options: StaticDownloadOptions, pipeline: PipelineExecutor): Promise<DownloadResource | void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"save-html-to-disk.d.ts","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAC,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AAG3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAgB,uBAAuB,CACrC,GAAG,EAAE,gBAAgB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAA;CAAE,
|
|
1
|
+
{"version":3,"file":"save-html-to-disk.d.ts","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAC,qBAAqB,EAAC,MAAM,eAAe,CAAC;AACzD,OAAO,KAAK,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAC,YAAY,EAAgB,MAAM,gBAAgB,CAAC;AAG3D,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,wBAAwB,CAAC;AAE7D,wBAAgB,uBAAuB,CACrC,GAAG,EAAE,gBAAgB,GAAG;IAAE,IAAI,EAAE,YAAY,CAAC,IAAI,GAAG,YAAY,CAAC,GAAG,CAAA;CAAE,EACtE,OAAO,EAAE,qBAAqB,GAC7B,YAAY,CAQd;AAED,wBAAgB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAWtF;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,gBAAgB,EACrB,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAiD9D"}
|
|
@@ -13,12 +13,13 @@ export function getResourceBodyFromHtml(res, options) {
|
|
|
13
13
|
return res.meta.doc.html();
|
|
14
14
|
}
|
|
15
15
|
export function redirectHtml(relativePath, encoding) {
|
|
16
|
+
const jsPath = relativePath.replace(/'/g, '\\\'');
|
|
16
17
|
// language=HTML
|
|
17
18
|
return `<html lang="en">
|
|
18
19
|
<head>
|
|
19
20
|
<meta charset="${encoding || 'utf8'}">
|
|
20
21
|
<meta http-equiv="refresh" content="0; url=${relativePath}">
|
|
21
|
-
<script>location.replace('${
|
|
22
|
+
<script>location.replace('${jsPath}' + location.hash);</script>
|
|
22
23
|
<title>Redirecting</title>
|
|
23
24
|
</head>
|
|
24
25
|
</html>`;
|
|
@@ -32,7 +33,7 @@ export async function saveHtmlToDisk(res, options, pipeline) {
|
|
|
32
33
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
33
34
|
let mtime = void 0;
|
|
34
35
|
if (options.preferRemoteLastModifiedTime && ((_c = (_b = res.meta) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['last-modified'])) {
|
|
35
|
-
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']);
|
|
36
|
+
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']) / 1000;
|
|
36
37
|
}
|
|
37
38
|
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
|
|
38
39
|
if (res.redirectedSavePath) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"save-html-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,GAAG,MAAM,OAAO,CAAC;AAIxB,OAAO,EAAC,YAAY,EAAE,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAC,UAAU,EAAC,MAAM,YAAY,CAAC;AACtC,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AAGnC,MAAM,UAAU,uBAAuB,CACrC,
|
|
1
|
+
{"version":3,"file":"save-html-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-html-to-disk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,GAAG,MAAM,OAAO,CAAC;AAIxB,OAAO,EAAC,YAAY,EAAE,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAC,UAAU,EAAC,MAAM,YAAY,CAAC;AACtC,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AAGnC,MAAM,UAAU,uBAAuB,CACrC,GAAsE,EACtE,OAA8B;IAE9B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAClB,OAAO,GAAG,CAAC,IAAI,CAAC;IAClB,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;AAC7B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,YAAoB,EAAE,QAA2B;IAC5E,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAClD,gBAAgB;IAChB,OAAO;;iBAEQ,QAAQ,IAAI,MAAM;6CACU,YAAY;4BAC7B,MAAM;;;QAG1B,CAAC;AACT,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAqB,EACrB,OAA8B,EAC9B,QAA0B;;IAC1B,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,SAAS,GAAW,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,CAAC;IAC7D,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC;IACjE,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QACvD,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAC5C,MAAM,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;qBAC1D,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC3C,MAAM,YAAY,GAAW,UAAU,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC/D,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAC3D,YAAY,CAAC,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACjE,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACjE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,EACrE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBACjE,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACvD,CAAC;YACD,OAAO;QACT,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EACtE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,EAAE,SAAS,EAChD,SAAS,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAC9B,IAAI,gBAAgB,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,YAAY,GAAW,UAAU,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACtE,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAC5C,YAAY,CAAC,YAAY,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjE,MAAM,kBAAkB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EACxE,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EACvD,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;YAC9C,MAAM,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAClE,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACjE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACrF,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAiB,uBAAuB,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACrD,OAAO;AACT,CAAC"}
|
|
@@ -6,7 +6,7 @@ export async function saveResourceToDisk(res, options, pipeline) {
|
|
|
6
6
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
7
7
|
let mtime = void 0;
|
|
8
8
|
if (options.preferRemoteLastModifiedTime && ((_c = (_b = res.meta) === null || _b === void 0 ? void 0 : _b.headers) === null || _c === void 0 ? void 0 : _c['last-modified'])) {
|
|
9
|
-
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']);
|
|
9
|
+
mtime = Date.parse((_d = res.meta.headers) === null || _d === void 0 ? void 0 : _d['last-modified']) / 1000;
|
|
10
10
|
}
|
|
11
11
|
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
|
|
12
12
|
if (res.redirectedSavePath) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"save-resource-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-resource-to-disk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AAGnC,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAqB,EACrB,OAA8B,EAC9B,QAA0B;;IAC1B,MAAM,SAAS,GAAW,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,CAAC;IAC7D,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"save-resource-to-disk.js","sourceRoot":"","sources":["../../src/life-cycle/save-resource-to-disk.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAC,SAAS,EAAC,MAAM,UAAU,CAAC;AAGnC,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,GAAqB,EACrB,OAA8B,EAC9B,QAA0B;;IAC1B,MAAM,SAAS,GAAW,MAAA,GAAG,CAAC,SAAS,mCAAI,OAAO,CAAC,SAAS,CAAC;IAC7D,mEAAmE;IACnE,IAAI,KAAK,GAAkB,KAAK,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,4BAA4B,KAAI,MAAA,MAAA,GAAG,CAAC,IAAI,0CAAE,OAAO,0CAAG,eAAe,CAAC,CAAA,EAAE,CAAC;QACjF,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAA,GAAG,CAAC,IAAI,CAAC,OAAO,0CAAG,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC;IACjE,CAAC;IACD,IAAI,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,aAAa,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;QACvD,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,IAAI,GAAG,CAAC,kBAAkB,KAAK,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAC5C,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBAC7D,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE,GAAG,CAAC,IAAI,EAChE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/E,OAAO;QACT,CAAC;QACD,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAC7D,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,aAAa,EAAE,SAAS,EAChD,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,oDAAoD;QACpD,IAAI,gBAAgB,CAAC,WAAW,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC/E,MAAM,kBAAkB,GAAG,MAAM,QAAQ,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,EAC/D,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EACvD,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,kBAAkB,GAAG,SAAS,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YAClE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,kBAAkB,CAAC,EAAE,GAAG,CAAC,IAAI,EAChE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACvB,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAW,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzD,OAAO;AACT,CAAC"}
|
package/lib/options.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "website-scrap-engine",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.6",
|
|
4
4
|
"description": "Configurable website scraper in typescript",
|
|
5
5
|
"main": "lib",
|
|
6
6
|
"types": "lib",
|
|
@@ -16,28 +16,32 @@
|
|
|
16
16
|
"build": "npm run lint && npm run tsc && npm run copy",
|
|
17
17
|
"copy": "node copy-src.js",
|
|
18
18
|
"prepack": "npm run clean && npm run build",
|
|
19
|
-
"postshrinkwrap": "node package-lock-resolved.js"
|
|
19
|
+
"postshrinkwrap": "node package-lock-resolved.js",
|
|
20
|
+
"postinstall": "npm run postshrinkwrap && node -e \"require('fs').copyFileSync('fake-undici.d.ts', 'node_modules/cheerio/node_modules/undici/index.d.ts')\""
|
|
20
21
|
},
|
|
21
22
|
"dependencies": {
|
|
22
|
-
"cheerio": "^1.
|
|
23
|
+
"cheerio": "^1.2.0",
|
|
23
24
|
"css-url-parser": "^1.1.4",
|
|
24
25
|
"got": "^13.0.0",
|
|
25
26
|
"log4js": "^6.9.1",
|
|
26
27
|
"mkdirp": "^3.0.1",
|
|
27
|
-
"p-queue": "^8.1.
|
|
28
|
-
"srcset": "^5.0.
|
|
28
|
+
"p-queue": "^8.1.1",
|
|
29
|
+
"srcset": "^5.0.3",
|
|
29
30
|
"urijs": "^1.19.11"
|
|
30
31
|
},
|
|
31
32
|
"devDependencies": {
|
|
32
|
-
"@jest/globals": "^30.
|
|
33
|
-
"@types/node": "^
|
|
34
|
-
"@types/urijs": "^1.19.
|
|
35
|
-
"@typescript-eslint/eslint-plugin": "^8.
|
|
36
|
-
"@typescript-eslint/parser": "^8.
|
|
37
|
-
"eslint": "^9.
|
|
38
|
-
"jest": "^30.0
|
|
39
|
-
"ts-jest": "^29.4.
|
|
40
|
-
"typescript": "^5.9.
|
|
33
|
+
"@jest/globals": "^30.1.1",
|
|
34
|
+
"@types/node": "^25.5.0",
|
|
35
|
+
"@types/urijs": "^1.19.26",
|
|
36
|
+
"@typescript-eslint/eslint-plugin": "^8.57.1",
|
|
37
|
+
"@typescript-eslint/parser": "^8.57.1",
|
|
38
|
+
"eslint": "^9.39.2",
|
|
39
|
+
"jest": "^30.3.0",
|
|
40
|
+
"ts-jest": "^29.4.6",
|
|
41
|
+
"typescript": "^5.9.3"
|
|
42
|
+
},
|
|
43
|
+
"overrides": {
|
|
44
|
+
"undici": "npm:@favware/skip-dependency@1.2.2"
|
|
41
45
|
},
|
|
42
46
|
"files": [
|
|
43
47
|
".editorconfig",
|
|
@@ -55,7 +55,7 @@ export class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMessage> {
|
|
|
55
55
|
this.workers[i].worker.addListener('message',
|
|
56
56
|
msg => this.onMessage(this.workers[i], msg));
|
|
57
57
|
this.workers[i].worker.addListener('error',
|
|
58
|
-
err => this.workerOnError(this.workers[i], err));
|
|
58
|
+
err => this.workerOnError(this.workers[i], err as Error));
|
|
59
59
|
ready.push(new Promise(resolve =>
|
|
60
60
|
this.workers[i].worker.addListener('online',resolve)));
|
|
61
61
|
}
|
|
@@ -69,8 +69,9 @@ export class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMessage> {
|
|
|
69
69
|
onMessage(info: WorkerInfo, message: WorkerMessage): void {
|
|
70
70
|
if (message.type === WorkerMessageType.Complete) {
|
|
71
71
|
this.complete(info, message);
|
|
72
|
+
} else {
|
|
73
|
+
this.takeLog(info, message as LogWorkerMessage);
|
|
72
74
|
}
|
|
73
|
-
this.takeLog(info, message as LogWorkerMessage);
|
|
74
75
|
}
|
|
75
76
|
|
|
76
77
|
takeLog(info: WorkerInfo, message: LogWorkerMessage): void {
|
|
@@ -115,43 +116,92 @@ export class WorkerPool<T = unknown, R extends WorkerMessage = WorkerMessage> {
|
|
|
115
116
|
if (!this.pendingTasks.length) {
|
|
116
117
|
return;
|
|
117
118
|
}
|
|
118
|
-
|
|
119
|
-
const sorted = this.workers.sort(
|
|
119
|
+
const sorted = this.workers.slice().sort(
|
|
120
120
|
(a, b) => a.load - b.load);
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
) {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
121
|
+
const n = sorted.length;
|
|
122
|
+
let remaining = this.pendingTasks.length;
|
|
123
|
+
|
|
124
|
+
// Cap by maxLoad capacity
|
|
125
|
+
if (this.maxLoad > 0) {
|
|
126
|
+
let capacity = 0;
|
|
127
|
+
for (let i = 0; i < n; i++) {
|
|
128
|
+
capacity += Math.max(0, this.maxLoad - sorted[i].load);
|
|
129
|
+
}
|
|
130
|
+
remaining = Math.min(remaining, capacity);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
if (remaining <= 0) {
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Pass 1: water-fill to calculate balanced task assignments
|
|
138
|
+
const assign: number[] = new Array(n).fill(0);
|
|
139
|
+
let level = sorted[0].load;
|
|
140
|
+
for (let i = 0; i < n - 1 && remaining > 0; i++) {
|
|
141
|
+
let gap = sorted[i + 1].load - level;
|
|
142
|
+
if (this.maxLoad > 0) {
|
|
143
|
+
gap = Math.min(gap, this.maxLoad - level);
|
|
144
|
+
}
|
|
145
|
+
if (gap <= 0) continue;
|
|
146
|
+
const width = i + 1;
|
|
147
|
+
const cost = gap * width;
|
|
148
|
+
if (cost <= remaining) {
|
|
149
|
+
for (let j = 0; j <= i; j++) assign[j] += gap;
|
|
150
|
+
remaining -= cost;
|
|
151
|
+
level += gap;
|
|
152
|
+
} else {
|
|
153
|
+
const each = (remaining / width) | 0;
|
|
154
|
+
let extra = remaining % width;
|
|
155
|
+
for (let j = 0; j <= i; j++) {
|
|
156
|
+
assign[j] += each + (extra > 0 ? 1 : 0);
|
|
157
|
+
if (extra > 0) extra--;
|
|
132
158
|
}
|
|
159
|
+
remaining = 0;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
// Distribute remaining evenly across all workers
|
|
163
|
+
if (remaining > 0) {
|
|
164
|
+
const each = (remaining / n) | 0;
|
|
165
|
+
let extra = remaining % n;
|
|
166
|
+
for (let j = 0; j < n; j++) {
|
|
167
|
+
assign[j] += each + (extra > 0 ? 1 : 0);
|
|
168
|
+
if (extra > 0) extra--;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// Pass 2: dispatch tasks to workers
|
|
173
|
+
let dispatched = 0;
|
|
174
|
+
for (let i = 0; i < n; i++) {
|
|
175
|
+
for (let j = 0; j < assign[i]; j++) {
|
|
176
|
+
const task: PendingPromiseWithBody<R> | undefined =
|
|
177
|
+
this.pendingTasks[dispatched];
|
|
178
|
+
if (!task) break;
|
|
179
|
+
dispatched++;
|
|
133
180
|
try {
|
|
134
|
-
|
|
181
|
+
sorted[i].worker.postMessage({
|
|
135
182
|
taskId: task.taskId,
|
|
136
183
|
body: task.body
|
|
137
184
|
}, task.transferList);
|
|
138
185
|
this.workingTasks[task.taskId] = task as PendingPromise;
|
|
186
|
+
++sorted[i].load;
|
|
139
187
|
} catch (e) {
|
|
140
188
|
delete this.workingTasks[task.taskId];
|
|
141
189
|
task.reject(e);
|
|
142
|
-
continue;
|
|
143
190
|
}
|
|
144
|
-
++sorted[i].load;
|
|
145
|
-
}
|
|
146
|
-
if (!this.pendingTasks.length) {
|
|
147
|
-
break;
|
|
148
191
|
}
|
|
149
192
|
}
|
|
193
|
+
if (dispatched > 0) {
|
|
194
|
+
this.pendingTasks.splice(0, dispatched);
|
|
195
|
+
}
|
|
150
196
|
}
|
|
151
197
|
|
|
152
198
|
async dispose(): Promise<number[]> {
|
|
153
199
|
const numbers = await Promise.all(
|
|
154
200
|
this.workers.map(info => info.worker.terminate()));
|
|
201
|
+
for (const task of this.pendingTasks) {
|
|
202
|
+
task.reject(new Error('disposed'));
|
|
203
|
+
}
|
|
204
|
+
this.pendingTasks.length = 0;
|
|
155
205
|
for (const taskId in this.workingTasks) {
|
|
156
206
|
// noinspection JSUnfilteredForInLoop
|
|
157
207
|
this.workingTasks[taskId].reject(new Error('disposed'));
|
package/src/downloader/worker.ts
CHANGED
|
@@ -94,7 +94,7 @@ export interface HtmlProcessFunc {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
export const parseHtml = (
|
|
97
|
-
res: DownloadResource & { type: ResourceType.Html },
|
|
97
|
+
res: DownloadResource & { type: ResourceType.Html | ResourceType.Svg },
|
|
98
98
|
options: StaticDownloadOptions
|
|
99
99
|
): CheerioStatic => {
|
|
100
100
|
const encoding: ResourceEncoding =
|
|
@@ -66,6 +66,9 @@ export async function getRetry(
|
|
|
66
66
|
// force cast for typescript 4.4
|
|
67
67
|
err = e as DownloadError | void;
|
|
68
68
|
if (err && err.message === 'premature close') {
|
|
69
|
+
if (err.retryLimitExceeded) {
|
|
70
|
+
throw e;
|
|
71
|
+
}
|
|
69
72
|
logger.retry.warn(i, url, 'manually retry on premature close',
|
|
70
73
|
err.name, err.code, err.event, err.message);
|
|
71
74
|
await sleep(i * 200);
|
|
@@ -99,7 +102,12 @@ export async function requestForResource(
|
|
|
99
102
|
requestOptions: RequestOptions,
|
|
100
103
|
options?: StaticDownloadOptions
|
|
101
104
|
): Promise<DownloadResource | Resource | void> {
|
|
102
|
-
|
|
105
|
+
let downloadLink: string;
|
|
106
|
+
try {
|
|
107
|
+
downloadLink = encodeURI(decodeURI(res.downloadLink));
|
|
108
|
+
} catch {
|
|
109
|
+
downloadLink = res.downloadLink;
|
|
110
|
+
}
|
|
103
111
|
const reqOptions: OptionsInit = Object.assign({}, requestOptions);
|
|
104
112
|
reqOptions.responseType = 'buffer';
|
|
105
113
|
if (res.refUrl && res.refUrl !== downloadLink) {
|
|
@@ -178,9 +186,9 @@ export async function downloadResource(
|
|
|
178
186
|
} else if (Array.isArray(contentType)) {
|
|
179
187
|
nonHtml = true;
|
|
180
188
|
for (const header of contentType) {
|
|
181
|
-
if (
|
|
182
|
-
|
|
183
|
-
|
|
189
|
+
if (header.includes('/html') ||
|
|
190
|
+
header.includes('/xml') ||
|
|
191
|
+
header.includes('application/xhtml+xml')) {
|
|
184
192
|
nonHtml = false;
|
|
185
193
|
break;
|
|
186
194
|
}
|
|
@@ -199,10 +207,12 @@ export async function downloadResource(
|
|
|
199
207
|
logger.error.info('Detected incomplete html, try again',
|
|
200
208
|
downloadedResource.downloadLink);
|
|
201
209
|
downloadedResource = await requestForResource(
|
|
202
|
-
res as (Resource & { downloadStartTimestamp: number }), requestOptions);
|
|
210
|
+
res as (Resource & { downloadStartTimestamp: number }), requestOptions, options);
|
|
203
211
|
}
|
|
204
212
|
// probably more retries here?
|
|
205
|
-
if (!downloadedResource ||
|
|
213
|
+
if (!downloadedResource ||
|
|
214
|
+
(typeof downloadedResource.body === 'string' ||
|
|
215
|
+
Buffer.isBuffer(downloadedResource.body)) &&
|
|
206
216
|
!downloadedResource.body.includes(options.meta.detectIncompleteHtml)) {
|
|
207
217
|
logger.error.warn('Detected incomplete html twice', res.downloadLink);
|
|
208
218
|
return downloadedResource;
|
|
@@ -198,7 +198,7 @@ export async function streamingDownloadToFile(
|
|
|
198
198
|
retry = retryCount * 300;
|
|
199
199
|
}
|
|
200
200
|
if (retry > 0) {
|
|
201
|
-
makeRequest(retryCount + 1);
|
|
201
|
+
setTimeout(() => makeRequest(retryCount + 1), retry);
|
|
202
202
|
return;
|
|
203
203
|
}
|
|
204
204
|
}
|
|
@@ -226,7 +226,7 @@ export async function optionallySetLastModifiedTime(
|
|
|
226
226
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
227
227
|
let mtime: number | void = void 0;
|
|
228
228
|
if (options.preferRemoteLastModifiedTime && res.meta?.headers?.['last-modified']) {
|
|
229
|
-
mtime = Date.parse(res.meta.headers?.['last-modified']);
|
|
229
|
+
mtime = Date.parse(res.meta.headers?.['last-modified']) / 1000;
|
|
230
230
|
}
|
|
231
231
|
|
|
232
232
|
// void and NaN check
|
|
@@ -14,6 +14,9 @@ export async function processCssText(
|
|
|
14
14
|
depth: number,
|
|
15
15
|
resources: Resource[]): Promise<string> {
|
|
16
16
|
const cssUrls: string[] = parseCssUrls(cssText);
|
|
17
|
+
if (!cssUrls.length) return cssText;
|
|
18
|
+
// Phase 1: process URLs and collect replacements
|
|
19
|
+
const replacements: Array<[string, string]> = [];
|
|
17
20
|
let rawUrl: string, r: Resource | void;
|
|
18
21
|
// noinspection DuplicatedCode
|
|
19
22
|
for (let i = 0, l = cssUrls.length; i < l; i++) {
|
|
@@ -24,9 +27,33 @@ export async function processCssText(
|
|
|
24
27
|
if (!r.shouldBeDiscardedFromDownload) {
|
|
25
28
|
resources.push(r);
|
|
26
29
|
}
|
|
27
|
-
|
|
30
|
+
if (rawUrl !== r.replacePath) {
|
|
31
|
+
replacements.push([rawUrl, r.replacePath]);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (!replacements.length) return cssText;
|
|
35
|
+
// Phase 2: single-pass positional replacement to avoid
|
|
36
|
+
// corrupting already-replaced paths
|
|
37
|
+
const occurrences: Array<[number, number, string]> = [];
|
|
38
|
+
for (const [url, replacePath] of replacements) {
|
|
39
|
+
let from = 0, pos: number;
|
|
40
|
+
while ((pos = cssText.indexOf(url, from)) !== -1) {
|
|
41
|
+
occurrences.push([pos, url.length, replacePath]);
|
|
42
|
+
from = pos + url.length;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (!occurrences.length) return cssText;
|
|
46
|
+
occurrences.sort((a, b) => a[0] - b[0]);
|
|
47
|
+
const parts: string[] = [];
|
|
48
|
+
let lastEnd = 0;
|
|
49
|
+
for (const [pos, len, replacePath] of occurrences) {
|
|
50
|
+
if (pos < lastEnd) continue;
|
|
51
|
+
parts.push(cssText.slice(lastEnd, pos));
|
|
52
|
+
parts.push(replacePath);
|
|
53
|
+
lastEnd = pos + len;
|
|
28
54
|
}
|
|
29
|
-
|
|
55
|
+
parts.push(cssText.slice(lastEnd));
|
|
56
|
+
return parts.join('');
|
|
30
57
|
}
|
|
31
58
|
|
|
32
59
|
export async function processCss(
|
|
@@ -44,6 +71,6 @@ export async function processCss(
|
|
|
44
71
|
cssText = await processCssText(cssText, res, options, pipeline, depth, resources);
|
|
45
72
|
res.body = cssText;
|
|
46
73
|
res.meta.cssProcessed = 1;
|
|
47
|
-
|
|
74
|
+
submit(resources);
|
|
48
75
|
return res;
|
|
49
76
|
}
|
|
@@ -6,6 +6,7 @@ import {ResourceType} from '../resource.js';
|
|
|
6
6
|
import type {DownloadResource, RequestOptions} from './types.js';
|
|
7
7
|
import type {StaticDownloadOptions} from '../options.js';
|
|
8
8
|
import {error as errorLogger} from '../logger/logger.js';
|
|
9
|
+
import {mkdirRetry} from '../io.js';
|
|
9
10
|
|
|
10
11
|
const FILE_PREFIX = 'file://';
|
|
11
12
|
|
|
@@ -44,6 +45,7 @@ export async function readOrCopyLocalResource(
|
|
|
44
45
|
}
|
|
45
46
|
if (res.type === ResourceType.StreamingBinary) {
|
|
46
47
|
const fileDestPath = path.join(res.localRoot ?? options.localRoot, res.savePath);
|
|
48
|
+
await mkdirRetry(path.dirname(fileDestPath));
|
|
47
49
|
await promises.copyFile(fileSrcPath, fileDestPath);
|
|
48
50
|
} else {
|
|
49
51
|
res.body = await promises.readFile(fileSrcPath, {
|
|
@@ -9,7 +9,7 @@ import {writeFile} from '../io.js';
|
|
|
9
9
|
import type {PipelineExecutor} from './pipeline-executor.js';
|
|
10
10
|
|
|
11
11
|
export function getResourceBodyFromHtml(
|
|
12
|
-
res: DownloadResource & { type: ResourceType.Html },
|
|
12
|
+
res: DownloadResource & { type: ResourceType.Html | ResourceType.Svg },
|
|
13
13
|
options: StaticDownloadOptions
|
|
14
14
|
): ResourceBody {
|
|
15
15
|
if (!res.meta.doc) {
|
|
@@ -22,12 +22,13 @@ export function getResourceBodyFromHtml(
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
export function redirectHtml(relativePath: string, encoding?: ResourceEncoding): string {
|
|
25
|
+
const jsPath = relativePath.replace(/'/g, '\\\'');
|
|
25
26
|
// language=HTML
|
|
26
27
|
return `<html lang="en">
|
|
27
28
|
<head>
|
|
28
29
|
<meta charset="${encoding || 'utf8'}">
|
|
29
30
|
<meta http-equiv="refresh" content="0; url=${relativePath}">
|
|
30
|
-
<script>location.replace('${
|
|
31
|
+
<script>location.replace('${jsPath}' + location.hash);</script>
|
|
31
32
|
<title>Redirecting</title>
|
|
32
33
|
</head>
|
|
33
34
|
</html>`;
|
|
@@ -44,7 +45,7 @@ export async function saveHtmlToDisk(
|
|
|
44
45
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
45
46
|
let mtime: number | void = void 0;
|
|
46
47
|
if (options.preferRemoteLastModifiedTime && res.meta?.headers?.['last-modified']) {
|
|
47
|
-
mtime = Date.parse(res.meta.headers?.['last-modified']);
|
|
48
|
+
mtime = Date.parse(res.meta.headers?.['last-modified']) / 1000;
|
|
48
49
|
}
|
|
49
50
|
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
|
|
50
51
|
if (res.redirectedSavePath) {
|
|
@@ -12,7 +12,7 @@ export async function saveResourceToDisk(
|
|
|
12
12
|
// https://github.com/website-local/website-scrap-engine/issues/174
|
|
13
13
|
let mtime: number | void = void 0;
|
|
14
14
|
if (options.preferRemoteLastModifiedTime && res.meta?.headers?.['last-modified']) {
|
|
15
|
-
mtime = Date.parse(res.meta.headers?.['last-modified']);
|
|
15
|
+
mtime = Date.parse(res.meta.headers?.['last-modified']) / 1000;
|
|
16
16
|
}
|
|
17
17
|
if (res.redirectedUrl && res.redirectedUrl !== res.url) {
|
|
18
18
|
if (res.redirectedSavePath) {
|
package/src/options.ts
CHANGED
|
@@ -258,7 +258,7 @@ export const calculateFastDelay: RetryFunction = (retryObject: RetryObject): num
|
|
|
258
258
|
retryAfter = 1;
|
|
259
259
|
}
|
|
260
260
|
if (retryOptions.maxRetryAfter) {
|
|
261
|
-
if (retryAfter
|
|
261
|
+
if (retryAfter <= retryOptions.maxRetryAfter) {
|
|
262
262
|
delay = retryAfter;
|
|
263
263
|
}
|
|
264
264
|
} else {
|