rezo 1.0.66 → 1.0.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +5 -0
- package/dist/adapters/entries/fetch.d.ts +5 -0
- package/dist/adapters/entries/http.d.ts +5 -0
- package/dist/adapters/entries/http2.d.ts +5 -0
- package/dist/adapters/entries/react-native.d.ts +5 -0
- package/dist/adapters/entries/xhr.d.ts +5 -0
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +9 -9
- package/dist/crawler/crawler.cjs +26 -5
- package/dist/crawler/crawler.js +26 -5
- package/dist/crawler/index.cjs +40 -40
- package/dist/crawler.d.ts +10 -0
- package/dist/entries/crawler.cjs +4 -4
- package/dist/index.cjs +27 -27
- package/dist/index.d.ts +5 -0
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/platform/browser.d.ts +5 -0
- package/dist/platform/bun.d.ts +5 -0
- package/dist/platform/deno.d.ts +5 -0
- package/dist/platform/node.d.ts +5 -0
- package/dist/platform/react-native.d.ts +5 -0
- package/dist/platform/worker.d.ts +5 -0
- package/dist/proxy/index.cjs +4 -4
- package/dist/proxy/manager.cjs +1 -1
- package/dist/proxy/manager.js +1 -1
- package/dist/queue/index.cjs +8 -8
- package/dist/queue/queue.cjs +3 -1
- package/dist/queue/queue.js +3 -1
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/wget/asset-extractor.cjs +556 -0
- package/dist/wget/asset-extractor.js +553 -0
- package/dist/wget/asset-organizer.cjs +230 -0
- package/dist/wget/asset-organizer.js +227 -0
- package/dist/wget/download-cache.cjs +221 -0
- package/dist/wget/download-cache.js +218 -0
- package/dist/wget/downloader.cjs +607 -0
- package/dist/wget/downloader.js +604 -0
- package/dist/wget/file-writer.cjs +349 -0
- package/dist/wget/file-writer.js +346 -0
- package/dist/wget/filter-lists.cjs +1330 -0
- package/dist/wget/filter-lists.js +1330 -0
- package/dist/wget/index.cjs +633 -0
- package/dist/wget/index.d.ts +8486 -0
- package/dist/wget/index.js +614 -0
- package/dist/wget/link-converter.cjs +297 -0
- package/dist/wget/link-converter.js +294 -0
- package/dist/wget/progress.cjs +271 -0
- package/dist/wget/progress.js +266 -0
- package/dist/wget/resume.cjs +166 -0
- package/dist/wget/resume.js +163 -0
- package/dist/wget/robots.cjs +303 -0
- package/dist/wget/robots.js +300 -0
- package/dist/wget/types.cjs +200 -0
- package/dist/wget/types.js +197 -0
- package/dist/wget/url-filter.cjs +351 -0
- package/dist/wget/url-filter.js +348 -0
- package/package.json +6 -1
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
class ProgressTracker {
|
|
2
|
+
url;
|
|
3
|
+
filename;
|
|
4
|
+
totalBytes = null;
|
|
5
|
+
bytesDownloaded = 0;
|
|
6
|
+
startTime = 0;
|
|
7
|
+
lastUpdateTime = 0;
|
|
8
|
+
lastBytes = 0;
|
|
9
|
+
speed = 0;
|
|
10
|
+
contentType = null;
|
|
11
|
+
speedSamples = [];
|
|
12
|
+
maxSamples = 10;
|
|
13
|
+
constructor(url, filename) {
|
|
14
|
+
this.url = url;
|
|
15
|
+
this.filename = filename;
|
|
16
|
+
}
|
|
17
|
+
start(totalBytes, contentType) {
|
|
18
|
+
this.totalBytes = totalBytes;
|
|
19
|
+
this.bytesDownloaded = 0;
|
|
20
|
+
this.startTime = Date.now();
|
|
21
|
+
this.lastUpdateTime = this.startTime;
|
|
22
|
+
this.lastBytes = 0;
|
|
23
|
+
this.speed = 0;
|
|
24
|
+
this.speedSamples = [];
|
|
25
|
+
this.contentType = contentType || null;
|
|
26
|
+
}
|
|
27
|
+
update(bytesDownloaded) {
|
|
28
|
+
const now = Date.now();
|
|
29
|
+
const timeDiff = now - this.lastUpdateTime;
|
|
30
|
+
if (timeDiff > 0) {
|
|
31
|
+
const bytesDiff = bytesDownloaded - this.lastBytes;
|
|
32
|
+
const instantSpeed = bytesDiff / timeDiff * 1000;
|
|
33
|
+
this.speedSamples.push(instantSpeed);
|
|
34
|
+
if (this.speedSamples.length > this.maxSamples) {
|
|
35
|
+
this.speedSamples.shift();
|
|
36
|
+
}
|
|
37
|
+
this.speed = this.speedSamples.reduce((a, b) => a + b, 0) / this.speedSamples.length;
|
|
38
|
+
}
|
|
39
|
+
this.bytesDownloaded = bytesDownloaded;
|
|
40
|
+
this.lastBytes = bytesDownloaded;
|
|
41
|
+
this.lastUpdateTime = now;
|
|
42
|
+
}
|
|
43
|
+
getProgress() {
|
|
44
|
+
let percent = null;
|
|
45
|
+
let eta = null;
|
|
46
|
+
if (this.totalBytes !== null && this.totalBytes > 0) {
|
|
47
|
+
percent = Math.min(100, this.bytesDownloaded / this.totalBytes * 100);
|
|
48
|
+
if (this.speed > 0) {
|
|
49
|
+
const remaining = this.totalBytes - this.bytesDownloaded;
|
|
50
|
+
eta = remaining / this.speed;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
url: this.url,
|
|
55
|
+
filename: this.filename,
|
|
56
|
+
bytesDownloaded: this.bytesDownloaded,
|
|
57
|
+
totalBytes: this.totalBytes,
|
|
58
|
+
percent,
|
|
59
|
+
speed: this.speed,
|
|
60
|
+
eta,
|
|
61
|
+
contentType: this.contentType
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
getElapsed() {
|
|
65
|
+
return Date.now() - this.startTime;
|
|
66
|
+
}
|
|
67
|
+
getAverageSpeed() {
|
|
68
|
+
const elapsed = this.getElapsed();
|
|
69
|
+
if (elapsed === 0)
|
|
70
|
+
return 0;
|
|
71
|
+
return this.bytesDownloaded / elapsed * 1000;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
class ProgressReporter {
|
|
76
|
+
options;
|
|
77
|
+
stats;
|
|
78
|
+
activeDownloads = new Map;
|
|
79
|
+
progressCallback;
|
|
80
|
+
constructor(options) {
|
|
81
|
+
this.options = options;
|
|
82
|
+
this.stats = {
|
|
83
|
+
urlsDownloaded: 0,
|
|
84
|
+
urlsFailed: 0,
|
|
85
|
+
urlsSkipped: 0,
|
|
86
|
+
bytesDownloaded: 0,
|
|
87
|
+
filesWritten: 0,
|
|
88
|
+
startTime: Date.now()
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
onProgress(callback) {
|
|
92
|
+
this.progressCallback = callback;
|
|
93
|
+
}
|
|
94
|
+
createTracker(url, filename) {
|
|
95
|
+
const tracker = new ProgressTracker(url, filename);
|
|
96
|
+
this.activeDownloads.set(url, tracker);
|
|
97
|
+
return tracker;
|
|
98
|
+
}
|
|
99
|
+
reportProgress(tracker) {
|
|
100
|
+
const progress = tracker.getProgress();
|
|
101
|
+
if (this.progressCallback) {
|
|
102
|
+
this.progressCallback(progress);
|
|
103
|
+
}
|
|
104
|
+
if (!this.options.quiet && this.options.progress !== "none") {
|
|
105
|
+
this.displayProgress(progress);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
reportComplete(url, size) {
|
|
109
|
+
this.stats.urlsDownloaded++;
|
|
110
|
+
this.stats.bytesDownloaded += size;
|
|
111
|
+
this.stats.filesWritten++;
|
|
112
|
+
this.activeDownloads.delete(url);
|
|
113
|
+
}
|
|
114
|
+
reportFailed(url) {
|
|
115
|
+
this.stats.urlsFailed++;
|
|
116
|
+
this.activeDownloads.delete(url);
|
|
117
|
+
}
|
|
118
|
+
reportSkipped(url, reason) {
|
|
119
|
+
this.stats.urlsSkipped++;
|
|
120
|
+
if (this.options.verbose && !this.options.quiet) {
|
|
121
|
+
this.log(`Skipped: ${url} (${reason})`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
displayProgress(progress) {
|
|
125
|
+
const style = this.options.progress || "bar";
|
|
126
|
+
if (style === "bar") {
|
|
127
|
+
this.displayBar(progress);
|
|
128
|
+
} else if (style === "dot") {
|
|
129
|
+
this.displayDot(progress);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
displayBar(progress) {
|
|
133
|
+
const width = 40;
|
|
134
|
+
const percent = progress.percent ?? 0;
|
|
135
|
+
const filled = Math.round(width * percent / 100);
|
|
136
|
+
const empty = width - filled;
|
|
137
|
+
const bar = "█".repeat(filled) + "░".repeat(empty);
|
|
138
|
+
const speedStr = this.formatSpeed(progress.speed);
|
|
139
|
+
const etaStr = progress.eta !== null ? this.formatTime(progress.eta) : "--:--";
|
|
140
|
+
const sizeStr = this.formatBytes(progress.bytesDownloaded);
|
|
141
|
+
const totalStr = progress.totalBytes !== null ? "/" + this.formatBytes(progress.totalBytes) : "";
|
|
142
|
+
const line = `[${bar}] ${percent.toFixed(1)}% ${sizeStr}${totalStr} ${speedStr} ETA: ${etaStr}`;
|
|
143
|
+
if (this.options.showProgress || process.stdout.isTTY) {
|
|
144
|
+
process.stdout.write("\r" + line);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
displayDot(progress) {
|
|
148
|
+
const kb = Math.floor(progress.bytesDownloaded / 1024);
|
|
149
|
+
const dots = kb - this.getDotCount(progress.url);
|
|
150
|
+
if (dots > 0) {
|
|
151
|
+
process.stdout.write(".".repeat(Math.min(dots, 50)));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
dotCounts = new Map;
|
|
155
|
+
getDotCount(url) {
|
|
156
|
+
return this.dotCounts.get(url) || 0;
|
|
157
|
+
}
|
|
158
|
+
log(message) {
|
|
159
|
+
if (!this.options.quiet) {
|
|
160
|
+
console.log(message);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
debug(message) {
|
|
164
|
+
if (this.options.debug && !this.options.quiet) {
|
|
165
|
+
console.log(`[DEBUG] ${message}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
verbose(message) {
|
|
169
|
+
if (this.options.verbose && !this.options.quiet) {
|
|
170
|
+
console.log(message);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
formatBytes(bytes) {
|
|
174
|
+
if (bytes < 1024)
|
|
175
|
+
return `${bytes} B`;
|
|
176
|
+
if (bytes < 1024 * 1024)
|
|
177
|
+
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
178
|
+
if (bytes < 1024 * 1024 * 1024)
|
|
179
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
180
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
|
181
|
+
}
|
|
182
|
+
formatSpeed(bytesPerSecond) {
|
|
183
|
+
const unit = this.options.reportSpeed || "bytes";
|
|
184
|
+
if (unit === "bits") {
|
|
185
|
+
const bps = bytesPerSecond * 8;
|
|
186
|
+
if (bps < 1024)
|
|
187
|
+
return `${bps.toFixed(0)} bps`;
|
|
188
|
+
if (bps < 1024 * 1024)
|
|
189
|
+
return `${(bps / 1024).toFixed(1)} Kbps`;
|
|
190
|
+
if (bps < 1024 * 1024 * 1024)
|
|
191
|
+
return `${(bps / (1024 * 1024)).toFixed(1)} Mbps`;
|
|
192
|
+
return `${(bps / (1024 * 1024 * 1024)).toFixed(2)} Gbps`;
|
|
193
|
+
}
|
|
194
|
+
if (bytesPerSecond < 1024)
|
|
195
|
+
return `${bytesPerSecond.toFixed(0)} B/s`;
|
|
196
|
+
if (bytesPerSecond < 1024 * 1024)
|
|
197
|
+
return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`;
|
|
198
|
+
if (bytesPerSecond < 1024 * 1024 * 1024)
|
|
199
|
+
return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MB/s`;
|
|
200
|
+
return `${(bytesPerSecond / (1024 * 1024 * 1024)).toFixed(2)} GB/s`;
|
|
201
|
+
}
|
|
202
|
+
formatTime(seconds) {
|
|
203
|
+
if (seconds < 0)
|
|
204
|
+
return "--:--";
|
|
205
|
+
const hours = Math.floor(seconds / 3600);
|
|
206
|
+
const minutes = Math.floor(seconds % 3600 / 60);
|
|
207
|
+
const secs = Math.floor(seconds % 60);
|
|
208
|
+
if (hours > 0) {
|
|
209
|
+
return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
|
|
210
|
+
}
|
|
211
|
+
return `${minutes}:${secs.toString().padStart(2, "0")}`;
|
|
212
|
+
}
|
|
213
|
+
finish() {
|
|
214
|
+
this.stats.endTime = Date.now();
|
|
215
|
+
this.stats.duration = this.stats.endTime - this.stats.startTime;
|
|
216
|
+
return this.stats;
|
|
217
|
+
}
|
|
218
|
+
getStats() {
|
|
219
|
+
return { ...this.stats };
|
|
220
|
+
}
|
|
221
|
+
formatStats() {
|
|
222
|
+
const stats = this.getStats();
|
|
223
|
+
const duration = stats.duration || Date.now() - stats.startTime;
|
|
224
|
+
const avgSpeed = duration > 0 ? stats.bytesDownloaded / duration * 1000 : 0;
|
|
225
|
+
const lines = [
|
|
226
|
+
"",
|
|
227
|
+
`FINISHED --${new Date().toISOString()}--`,
|
|
228
|
+
`Total: ${stats.urlsDownloaded} file(s), ${this.formatBytes(stats.bytesDownloaded)}`,
|
|
229
|
+
`Downloaded: ${stats.filesWritten} file(s) in ${this.formatTime(duration / 1000)}`,
|
|
230
|
+
`Average speed: ${this.formatSpeed(avgSpeed)}`
|
|
231
|
+
];
|
|
232
|
+
if (stats.urlsFailed > 0) {
|
|
233
|
+
lines.push(`Failed: ${stats.urlsFailed} URL(s)`);
|
|
234
|
+
}
|
|
235
|
+
if (stats.urlsSkipped > 0) {
|
|
236
|
+
lines.push(`Skipped: ${stats.urlsSkipped} URL(s)`);
|
|
237
|
+
}
|
|
238
|
+
return lines.join(`
|
|
239
|
+
`);
|
|
240
|
+
}
|
|
241
|
+
newline() {
|
|
242
|
+
if (!this.options.quiet && this.options.progress !== "none") {
|
|
243
|
+
console.log("");
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
function parseSize(size) {
|
|
248
|
+
if (typeof size === "number")
|
|
249
|
+
return size;
|
|
250
|
+
const match = size.match(/^(\d+(?:\.\d+)?)\s*([kmgKMG])?$/);
|
|
251
|
+
if (!match)
|
|
252
|
+
return parseInt(size, 10) || 0;
|
|
253
|
+
const value = parseFloat(match[1]);
|
|
254
|
+
const unit = (match[2] || "").toLowerCase();
|
|
255
|
+
switch (unit) {
|
|
256
|
+
case "k":
|
|
257
|
+
return value * 1024;
|
|
258
|
+
case "m":
|
|
259
|
+
return value * 1024 * 1024;
|
|
260
|
+
case "g":
|
|
261
|
+
return value * 1024 * 1024 * 1024;
|
|
262
|
+
default:
|
|
263
|
+
return value;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
exports.ProgressTracker = ProgressTracker;
|
|
268
|
+
exports.ProgressReporter = ProgressReporter;
|
|
269
|
+
exports.parseSize = parseSize;
|
|
270
|
+
exports.default = ProgressReporter;
|
|
271
|
+
module.exports = Object.assign(ProgressReporter, exports);
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
export class ProgressTracker {
|
|
2
|
+
url;
|
|
3
|
+
filename;
|
|
4
|
+
totalBytes = null;
|
|
5
|
+
bytesDownloaded = 0;
|
|
6
|
+
startTime = 0;
|
|
7
|
+
lastUpdateTime = 0;
|
|
8
|
+
lastBytes = 0;
|
|
9
|
+
speed = 0;
|
|
10
|
+
contentType = null;
|
|
11
|
+
speedSamples = [];
|
|
12
|
+
maxSamples = 10;
|
|
13
|
+
constructor(url, filename) {
|
|
14
|
+
this.url = url;
|
|
15
|
+
this.filename = filename;
|
|
16
|
+
}
|
|
17
|
+
start(totalBytes, contentType) {
|
|
18
|
+
this.totalBytes = totalBytes;
|
|
19
|
+
this.bytesDownloaded = 0;
|
|
20
|
+
this.startTime = Date.now();
|
|
21
|
+
this.lastUpdateTime = this.startTime;
|
|
22
|
+
this.lastBytes = 0;
|
|
23
|
+
this.speed = 0;
|
|
24
|
+
this.speedSamples = [];
|
|
25
|
+
this.contentType = contentType || null;
|
|
26
|
+
}
|
|
27
|
+
update(bytesDownloaded) {
|
|
28
|
+
const now = Date.now();
|
|
29
|
+
const timeDiff = now - this.lastUpdateTime;
|
|
30
|
+
if (timeDiff > 0) {
|
|
31
|
+
const bytesDiff = bytesDownloaded - this.lastBytes;
|
|
32
|
+
const instantSpeed = bytesDiff / timeDiff * 1000;
|
|
33
|
+
this.speedSamples.push(instantSpeed);
|
|
34
|
+
if (this.speedSamples.length > this.maxSamples) {
|
|
35
|
+
this.speedSamples.shift();
|
|
36
|
+
}
|
|
37
|
+
this.speed = this.speedSamples.reduce((a, b) => a + b, 0) / this.speedSamples.length;
|
|
38
|
+
}
|
|
39
|
+
this.bytesDownloaded = bytesDownloaded;
|
|
40
|
+
this.lastBytes = bytesDownloaded;
|
|
41
|
+
this.lastUpdateTime = now;
|
|
42
|
+
}
|
|
43
|
+
getProgress() {
|
|
44
|
+
let percent = null;
|
|
45
|
+
let eta = null;
|
|
46
|
+
if (this.totalBytes !== null && this.totalBytes > 0) {
|
|
47
|
+
percent = Math.min(100, this.bytesDownloaded / this.totalBytes * 100);
|
|
48
|
+
if (this.speed > 0) {
|
|
49
|
+
const remaining = this.totalBytes - this.bytesDownloaded;
|
|
50
|
+
eta = remaining / this.speed;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
url: this.url,
|
|
55
|
+
filename: this.filename,
|
|
56
|
+
bytesDownloaded: this.bytesDownloaded,
|
|
57
|
+
totalBytes: this.totalBytes,
|
|
58
|
+
percent,
|
|
59
|
+
speed: this.speed,
|
|
60
|
+
eta,
|
|
61
|
+
contentType: this.contentType
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
getElapsed() {
|
|
65
|
+
return Date.now() - this.startTime;
|
|
66
|
+
}
|
|
67
|
+
getAverageSpeed() {
|
|
68
|
+
const elapsed = this.getElapsed();
|
|
69
|
+
if (elapsed === 0)
|
|
70
|
+
return 0;
|
|
71
|
+
return this.bytesDownloaded / elapsed * 1000;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export class ProgressReporter {
|
|
76
|
+
options;
|
|
77
|
+
stats;
|
|
78
|
+
activeDownloads = new Map;
|
|
79
|
+
progressCallback;
|
|
80
|
+
constructor(options) {
|
|
81
|
+
this.options = options;
|
|
82
|
+
this.stats = {
|
|
83
|
+
urlsDownloaded: 0,
|
|
84
|
+
urlsFailed: 0,
|
|
85
|
+
urlsSkipped: 0,
|
|
86
|
+
bytesDownloaded: 0,
|
|
87
|
+
filesWritten: 0,
|
|
88
|
+
startTime: Date.now()
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
onProgress(callback) {
|
|
92
|
+
this.progressCallback = callback;
|
|
93
|
+
}
|
|
94
|
+
createTracker(url, filename) {
|
|
95
|
+
const tracker = new ProgressTracker(url, filename);
|
|
96
|
+
this.activeDownloads.set(url, tracker);
|
|
97
|
+
return tracker;
|
|
98
|
+
}
|
|
99
|
+
reportProgress(tracker) {
|
|
100
|
+
const progress = tracker.getProgress();
|
|
101
|
+
if (this.progressCallback) {
|
|
102
|
+
this.progressCallback(progress);
|
|
103
|
+
}
|
|
104
|
+
if (!this.options.quiet && this.options.progress !== "none") {
|
|
105
|
+
this.displayProgress(progress);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
reportComplete(url, size) {
|
|
109
|
+
this.stats.urlsDownloaded++;
|
|
110
|
+
this.stats.bytesDownloaded += size;
|
|
111
|
+
this.stats.filesWritten++;
|
|
112
|
+
this.activeDownloads.delete(url);
|
|
113
|
+
}
|
|
114
|
+
reportFailed(url) {
|
|
115
|
+
this.stats.urlsFailed++;
|
|
116
|
+
this.activeDownloads.delete(url);
|
|
117
|
+
}
|
|
118
|
+
reportSkipped(url, reason) {
|
|
119
|
+
this.stats.urlsSkipped++;
|
|
120
|
+
if (this.options.verbose && !this.options.quiet) {
|
|
121
|
+
this.log(`Skipped: ${url} (${reason})`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
displayProgress(progress) {
|
|
125
|
+
const style = this.options.progress || "bar";
|
|
126
|
+
if (style === "bar") {
|
|
127
|
+
this.displayBar(progress);
|
|
128
|
+
} else if (style === "dot") {
|
|
129
|
+
this.displayDot(progress);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
displayBar(progress) {
|
|
133
|
+
const width = 40;
|
|
134
|
+
const percent = progress.percent ?? 0;
|
|
135
|
+
const filled = Math.round(width * percent / 100);
|
|
136
|
+
const empty = width - filled;
|
|
137
|
+
const bar = "█".repeat(filled) + "░".repeat(empty);
|
|
138
|
+
const speedStr = this.formatSpeed(progress.speed);
|
|
139
|
+
const etaStr = progress.eta !== null ? this.formatTime(progress.eta) : "--:--";
|
|
140
|
+
const sizeStr = this.formatBytes(progress.bytesDownloaded);
|
|
141
|
+
const totalStr = progress.totalBytes !== null ? "/" + this.formatBytes(progress.totalBytes) : "";
|
|
142
|
+
const line = `[${bar}] ${percent.toFixed(1)}% ${sizeStr}${totalStr} ${speedStr} ETA: ${etaStr}`;
|
|
143
|
+
if (this.options.showProgress || process.stdout.isTTY) {
|
|
144
|
+
process.stdout.write("\r" + line);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
displayDot(progress) {
|
|
148
|
+
const kb = Math.floor(progress.bytesDownloaded / 1024);
|
|
149
|
+
const dots = kb - this.getDotCount(progress.url);
|
|
150
|
+
if (dots > 0) {
|
|
151
|
+
process.stdout.write(".".repeat(Math.min(dots, 50)));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
dotCounts = new Map;
|
|
155
|
+
getDotCount(url) {
|
|
156
|
+
return this.dotCounts.get(url) || 0;
|
|
157
|
+
}
|
|
158
|
+
log(message) {
|
|
159
|
+
if (!this.options.quiet) {
|
|
160
|
+
console.log(message);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
debug(message) {
|
|
164
|
+
if (this.options.debug && !this.options.quiet) {
|
|
165
|
+
console.log(`[DEBUG] ${message}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
verbose(message) {
|
|
169
|
+
if (this.options.verbose && !this.options.quiet) {
|
|
170
|
+
console.log(message);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
formatBytes(bytes) {
|
|
174
|
+
if (bytes < 1024)
|
|
175
|
+
return `${bytes} B`;
|
|
176
|
+
if (bytes < 1024 * 1024)
|
|
177
|
+
return `${(bytes / 1024).toFixed(1)} KB`;
|
|
178
|
+
if (bytes < 1024 * 1024 * 1024)
|
|
179
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
180
|
+
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
|
181
|
+
}
|
|
182
|
+
formatSpeed(bytesPerSecond) {
|
|
183
|
+
const unit = this.options.reportSpeed || "bytes";
|
|
184
|
+
if (unit === "bits") {
|
|
185
|
+
const bps = bytesPerSecond * 8;
|
|
186
|
+
if (bps < 1024)
|
|
187
|
+
return `${bps.toFixed(0)} bps`;
|
|
188
|
+
if (bps < 1024 * 1024)
|
|
189
|
+
return `${(bps / 1024).toFixed(1)} Kbps`;
|
|
190
|
+
if (bps < 1024 * 1024 * 1024)
|
|
191
|
+
return `${(bps / (1024 * 1024)).toFixed(1)} Mbps`;
|
|
192
|
+
return `${(bps / (1024 * 1024 * 1024)).toFixed(2)} Gbps`;
|
|
193
|
+
}
|
|
194
|
+
if (bytesPerSecond < 1024)
|
|
195
|
+
return `${bytesPerSecond.toFixed(0)} B/s`;
|
|
196
|
+
if (bytesPerSecond < 1024 * 1024)
|
|
197
|
+
return `${(bytesPerSecond / 1024).toFixed(1)} KB/s`;
|
|
198
|
+
if (bytesPerSecond < 1024 * 1024 * 1024)
|
|
199
|
+
return `${(bytesPerSecond / (1024 * 1024)).toFixed(1)} MB/s`;
|
|
200
|
+
return `${(bytesPerSecond / (1024 * 1024 * 1024)).toFixed(2)} GB/s`;
|
|
201
|
+
}
|
|
202
|
+
formatTime(seconds) {
|
|
203
|
+
if (seconds < 0)
|
|
204
|
+
return "--:--";
|
|
205
|
+
const hours = Math.floor(seconds / 3600);
|
|
206
|
+
const minutes = Math.floor(seconds % 3600 / 60);
|
|
207
|
+
const secs = Math.floor(seconds % 60);
|
|
208
|
+
if (hours > 0) {
|
|
209
|
+
return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
|
|
210
|
+
}
|
|
211
|
+
return `${minutes}:${secs.toString().padStart(2, "0")}`;
|
|
212
|
+
}
|
|
213
|
+
finish() {
|
|
214
|
+
this.stats.endTime = Date.now();
|
|
215
|
+
this.stats.duration = this.stats.endTime - this.stats.startTime;
|
|
216
|
+
return this.stats;
|
|
217
|
+
}
|
|
218
|
+
getStats() {
|
|
219
|
+
return { ...this.stats };
|
|
220
|
+
}
|
|
221
|
+
formatStats() {
|
|
222
|
+
const stats = this.getStats();
|
|
223
|
+
const duration = stats.duration || Date.now() - stats.startTime;
|
|
224
|
+
const avgSpeed = duration > 0 ? stats.bytesDownloaded / duration * 1000 : 0;
|
|
225
|
+
const lines = [
|
|
226
|
+
"",
|
|
227
|
+
`FINISHED --${new Date().toISOString()}--`,
|
|
228
|
+
`Total: ${stats.urlsDownloaded} file(s), ${this.formatBytes(stats.bytesDownloaded)}`,
|
|
229
|
+
`Downloaded: ${stats.filesWritten} file(s) in ${this.formatTime(duration / 1000)}`,
|
|
230
|
+
`Average speed: ${this.formatSpeed(avgSpeed)}`
|
|
231
|
+
];
|
|
232
|
+
if (stats.urlsFailed > 0) {
|
|
233
|
+
lines.push(`Failed: ${stats.urlsFailed} URL(s)`);
|
|
234
|
+
}
|
|
235
|
+
if (stats.urlsSkipped > 0) {
|
|
236
|
+
lines.push(`Skipped: ${stats.urlsSkipped} URL(s)`);
|
|
237
|
+
}
|
|
238
|
+
return lines.join(`
|
|
239
|
+
`);
|
|
240
|
+
}
|
|
241
|
+
newline() {
|
|
242
|
+
if (!this.options.quiet && this.options.progress !== "none") {
|
|
243
|
+
console.log("");
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
export function parseSize(size) {
|
|
248
|
+
if (typeof size === "number")
|
|
249
|
+
return size;
|
|
250
|
+
const match = size.match(/^(\d+(?:\.\d+)?)\s*([kmgKMG])?$/);
|
|
251
|
+
if (!match)
|
|
252
|
+
return parseInt(size, 10) || 0;
|
|
253
|
+
const value = parseFloat(match[1]);
|
|
254
|
+
const unit = (match[2] || "").toLowerCase();
|
|
255
|
+
switch (unit) {
|
|
256
|
+
case "k":
|
|
257
|
+
return value * 1024;
|
|
258
|
+
case "m":
|
|
259
|
+
return value * 1024 * 1024;
|
|
260
|
+
case "g":
|
|
261
|
+
return value * 1024 * 1024 * 1024;
|
|
262
|
+
default:
|
|
263
|
+
return value;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
export default ProgressReporter;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
const { promises: fs } = require("node:fs");
|
|
2
|
+
|
|
3
|
+
class ResumeHandler {
|
|
4
|
+
options;
|
|
5
|
+
constructor(options) {
|
|
6
|
+
this.options = options;
|
|
7
|
+
}
|
|
8
|
+
async getResumeInfo(localPath) {
|
|
9
|
+
try {
|
|
10
|
+
const stats = await fs.stat(localPath);
|
|
11
|
+
return {
|
|
12
|
+
path: localPath,
|
|
13
|
+
bytesDownloaded: stats.size,
|
|
14
|
+
mtime: stats.mtime,
|
|
15
|
+
exists: true,
|
|
16
|
+
canResume: this.options.continueDownload === true && stats.size > 0
|
|
17
|
+
};
|
|
18
|
+
} catch {
|
|
19
|
+
return {
|
|
20
|
+
path: localPath,
|
|
21
|
+
bytesDownloaded: 0,
|
|
22
|
+
mtime: new Date(0),
|
|
23
|
+
exists: false,
|
|
24
|
+
canResume: false
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
getResumeHeaders(info) {
|
|
29
|
+
if (!info.canResume) {
|
|
30
|
+
return {};
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
Range: `bytes=${info.bytesDownloaded}-`,
|
|
34
|
+
"If-Range": info.mtime.toUTCString()
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
async getTimestampHeaders(localPath) {
|
|
38
|
+
if (!this.options.timestamping) {
|
|
39
|
+
return {};
|
|
40
|
+
}
|
|
41
|
+
try {
|
|
42
|
+
const stats = await fs.stat(localPath);
|
|
43
|
+
return {
|
|
44
|
+
"If-Modified-Since": stats.mtime.toUTCString()
|
|
45
|
+
};
|
|
46
|
+
} catch {
|
|
47
|
+
return {};
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async checkTimestamp(localPath, remoteMtime) {
|
|
51
|
+
if (!this.options.timestamping) {
|
|
52
|
+
return {
|
|
53
|
+
shouldDownload: true,
|
|
54
|
+
reason: "no-timestamp"
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
if (!remoteMtime) {
|
|
58
|
+
return {
|
|
59
|
+
shouldDownload: true,
|
|
60
|
+
reason: "no-timestamp"
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
try {
|
|
64
|
+
const stats = await fs.stat(localPath);
|
|
65
|
+
const localMtime = stats.mtime;
|
|
66
|
+
const timeDiff = remoteMtime.getTime() - localMtime.getTime();
|
|
67
|
+
if (timeDiff > 1000) {
|
|
68
|
+
return {
|
|
69
|
+
shouldDownload: true,
|
|
70
|
+
reason: "newer",
|
|
71
|
+
localMtime,
|
|
72
|
+
remoteMtime
|
|
73
|
+
};
|
|
74
|
+
} else if (timeDiff < -1000) {
|
|
75
|
+
return {
|
|
76
|
+
shouldDownload: false,
|
|
77
|
+
reason: "older",
|
|
78
|
+
localMtime,
|
|
79
|
+
remoteMtime
|
|
80
|
+
};
|
|
81
|
+
} else {
|
|
82
|
+
return {
|
|
83
|
+
shouldDownload: false,
|
|
84
|
+
reason: "same",
|
|
85
|
+
localMtime,
|
|
86
|
+
remoteMtime
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
} catch {
|
|
90
|
+
return {
|
|
91
|
+
shouldDownload: true,
|
|
92
|
+
reason: "not-found",
|
|
93
|
+
remoteMtime
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
isValidPartialResponse(statusCode, contentRange, expectedStart) {
|
|
98
|
+
if (statusCode !== 206) {
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
101
|
+
if (!contentRange) {
|
|
102
|
+
return false;
|
|
103
|
+
}
|
|
104
|
+
const match = contentRange.match(/bytes\s+(\d+)-(\d+)\/(\d+|\*)/i);
|
|
105
|
+
if (!match) {
|
|
106
|
+
return false;
|
|
107
|
+
}
|
|
108
|
+
const start = parseInt(match[1], 10);
|
|
109
|
+
if (start !== expectedStart) {
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
parseContentRange(contentRange) {
|
|
115
|
+
if (!contentRange)
|
|
116
|
+
return null;
|
|
117
|
+
const match = contentRange.match(/bytes\s+(\d+)-(\d+)\/(\d+|\*)/i);
|
|
118
|
+
if (!match)
|
|
119
|
+
return null;
|
|
120
|
+
return {
|
|
121
|
+
start: parseInt(match[1], 10),
|
|
122
|
+
end: parseInt(match[2], 10),
|
|
123
|
+
total: match[3] === "*" ? null : parseInt(match[3], 10)
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
supportsRanges(acceptRanges) {
|
|
127
|
+
if (!acceptRanges)
|
|
128
|
+
return true;
|
|
129
|
+
return acceptRanges.toLowerCase() !== "none";
|
|
130
|
+
}
|
|
131
|
+
parseLastModified(lastModified) {
|
|
132
|
+
if (!lastModified)
|
|
133
|
+
return null;
|
|
134
|
+
try {
|
|
135
|
+
const date = new Date(lastModified);
|
|
136
|
+
return isNaN(date.getTime()) ? null : date;
|
|
137
|
+
} catch {
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
determineAction(statusCode, contentRange, resumeInfo) {
|
|
142
|
+
if (statusCode === 304) {
|
|
143
|
+
return "skip";
|
|
144
|
+
}
|
|
145
|
+
if (statusCode === 206) {
|
|
146
|
+
if (this.isValidPartialResponse(statusCode, contentRange, resumeInfo.bytesDownloaded)) {
|
|
147
|
+
return "resume";
|
|
148
|
+
}
|
|
149
|
+
return "restart";
|
|
150
|
+
}
|
|
151
|
+
if (statusCode === 200) {
|
|
152
|
+
return "restart";
|
|
153
|
+
}
|
|
154
|
+
if (statusCode === 416) {
|
|
155
|
+
return "skip";
|
|
156
|
+
}
|
|
157
|
+
return "restart";
|
|
158
|
+
}
|
|
159
|
+
updateOptions(options) {
|
|
160
|
+
this.options = { ...this.options, ...options };
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
exports.ResumeHandler = ResumeHandler;
|
|
165
|
+
exports.default = ResumeHandler;
|
|
166
|
+
module.exports = Object.assign(ResumeHandler, exports);
|