rezo 1.0.19 → 1.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +39 -34
- package/dist/adapters/curl.js +39 -34
- package/dist/adapters/entries/curl.d.ts +19 -18
- package/dist/adapters/entries/fetch.d.ts +19 -18
- package/dist/adapters/entries/http.d.ts +19 -18
- package/dist/adapters/entries/http2.d.ts +19 -18
- package/dist/adapters/entries/react-native.d.ts +19 -18
- package/dist/adapters/entries/xhr.d.ts +19 -18
- package/dist/adapters/fetch.cjs +42 -41
- package/dist/adapters/fetch.js +42 -41
- package/dist/adapters/http.cjs +65 -46
- package/dist/adapters/http.js +65 -46
- package/dist/adapters/http2.cjs +41 -36
- package/dist/adapters/http2.js +41 -36
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/react-native.cjs +41 -27
- package/dist/adapters/react-native.js +41 -27
- package/dist/adapters/xhr.cjs +43 -38
- package/dist/adapters/xhr.js +43 -38
- package/dist/cache/index.cjs +13 -13
- package/dist/crawler.d.ts +19 -18
- package/dist/entries/crawler.cjs +5 -5
- package/dist/errors/rezo-error.cjs +139 -21
- package/dist/errors/rezo-error.js +138 -21
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +19 -18
- package/dist/platform/browser.d.ts +19 -18
- package/dist/platform/bun.d.ts +19 -18
- package/dist/platform/deno.d.ts +19 -18
- package/dist/platform/node.d.ts +19 -18
- package/dist/platform/react-native.d.ts +19 -18
- package/dist/platform/worker.d.ts +19 -18
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/utils/http-config.cjs +6 -3
- package/dist/utils/http-config.js +6 -3
- package/dist/utils/timing.cjs +90 -0
- package/dist/utils/timing.js +78 -0
- package/package.json +1 -1
|
@@ -17,6 +17,30 @@ const Environment = {
|
|
|
17
17
|
hasFormData: typeof FormData !== "undefined",
|
|
18
18
|
hasAbortController: typeof AbortController !== "undefined"
|
|
19
19
|
};
|
|
20
|
+
function updateTiming(config, timing, bodySize) {
|
|
21
|
+
const now = performance.now();
|
|
22
|
+
config.timing.domainLookupStart = config.timing.startTime;
|
|
23
|
+
config.timing.domainLookupEnd = config.timing.startTime;
|
|
24
|
+
config.timing.connectStart = config.timing.startTime;
|
|
25
|
+
config.timing.secureConnectionStart = 0;
|
|
26
|
+
config.timing.connectEnd = config.timing.startTime;
|
|
27
|
+
config.timing.requestStart = config.timing.startTime;
|
|
28
|
+
config.timing.responseStart = timing.firstByteTime || config.timing.startTime;
|
|
29
|
+
config.timing.responseEnd = now;
|
|
30
|
+
config.transfer.bodySize = bodySize;
|
|
31
|
+
config.transfer.responseSize = bodySize;
|
|
32
|
+
}
|
|
33
|
+
function getTimingDurations(config) {
|
|
34
|
+
const t = config.timing;
|
|
35
|
+
return {
|
|
36
|
+
total: t.responseEnd - t.startTime,
|
|
37
|
+
dns: t.domainLookupEnd - t.domainLookupStart,
|
|
38
|
+
tcp: t.secureConnectionStart > 0 ? t.secureConnectionStart - t.connectStart : t.connectEnd - t.connectStart,
|
|
39
|
+
tls: t.secureConnectionStart > 0 ? t.connectEnd - t.secureConnectionStart : undefined,
|
|
40
|
+
firstByte: t.responseStart - t.startTime,
|
|
41
|
+
download: t.responseEnd - t.responseStart
|
|
42
|
+
};
|
|
43
|
+
}
|
|
20
44
|
const responseCacheInstances = new Map;
|
|
21
45
|
function getCacheConfigKey(option) {
|
|
22
46
|
if (option === true)
|
|
@@ -227,10 +251,11 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
227
251
|
const maxRetries = config?.retry?.maxRetries || 0;
|
|
228
252
|
const incrementDelay = config?.retry?.incrementDelay || false;
|
|
229
253
|
const statusCodes = config?.retry?.statusCodes;
|
|
254
|
+
const startTime = performance.now();
|
|
230
255
|
const timing = {
|
|
231
|
-
startTime
|
|
232
|
-
startTimestamp: Date.now()
|
|
256
|
+
startTime
|
|
233
257
|
};
|
|
258
|
+
config.timing.startTime = startTime;
|
|
234
259
|
const ABSOLUTE_MAX_ATTEMPTS = 50;
|
|
235
260
|
let totalAttempts = 0;
|
|
236
261
|
config.setSignal();
|
|
@@ -294,7 +319,6 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
294
319
|
config.isSecure = isSecure;
|
|
295
320
|
config.finalUrl = url;
|
|
296
321
|
config.network.protocol = isSecure ? "https" : "http";
|
|
297
|
-
config.timing.startTimestamp = timing.startTimestamp;
|
|
298
322
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
299
323
|
const headers = toFetchHeaders(reqHeaders);
|
|
300
324
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
@@ -347,9 +371,9 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
347
371
|
if (timeoutId)
|
|
348
372
|
clearTimeout(timeoutId);
|
|
349
373
|
}
|
|
350
|
-
if (!
|
|
374
|
+
if (!timing.firstByteTime) {
|
|
351
375
|
timing.firstByteTime = performance.now();
|
|
352
|
-
config.timing.
|
|
376
|
+
config.timing.responseStart = timing.firstByteTime;
|
|
353
377
|
}
|
|
354
378
|
const status = response.status;
|
|
355
379
|
const statusText = response.statusText;
|
|
@@ -373,8 +397,8 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
373
397
|
contentLength: contentLength ? parseInt(contentLength, 10) : undefined,
|
|
374
398
|
cookies: cookies.array,
|
|
375
399
|
timing: {
|
|
376
|
-
firstByte: config.timing.
|
|
377
|
-
total: performance.now() - timing.startTime
|
|
400
|
+
firstByte: config.timing.responseStart - config.timing.startTime,
|
|
401
|
+
total: performance.now() - config.timing.startTime
|
|
378
402
|
}
|
|
379
403
|
};
|
|
380
404
|
eventEmitter.emit("headers", headersEvent);
|
|
@@ -428,11 +452,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
428
452
|
bodySize = text.length;
|
|
429
453
|
}
|
|
430
454
|
}
|
|
431
|
-
config
|
|
432
|
-
config.timing.durationMs = performance.now() - timing.startTime;
|
|
433
|
-
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
434
|
-
config.transfer.bodySize = bodySize;
|
|
435
|
-
config.transfer.responseSize = bodySize;
|
|
455
|
+
updateTiming(config, timing, bodySize);
|
|
436
456
|
if (status >= 400) {
|
|
437
457
|
const error = builErrorFromResponse(`HTTP Error ${status}: ${statusText}`, {
|
|
438
458
|
status,
|
|
@@ -466,11 +486,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
466
486
|
finalUrl: url,
|
|
467
487
|
cookies,
|
|
468
488
|
urls: buildUrlTree(config, url),
|
|
469
|
-
timing:
|
|
470
|
-
total: config.timing.durationMs || 0,
|
|
471
|
-
firstByte: config.timing.ttfbMs,
|
|
472
|
-
download: config.timing.transferMs
|
|
473
|
-
},
|
|
489
|
+
timing: getTimingDurations(config),
|
|
474
490
|
config: sanitizeConfig(config)
|
|
475
491
|
};
|
|
476
492
|
streamResult.emit("finish", streamFinishEvent);
|
|
@@ -491,11 +507,10 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
491
507
|
fileName: config.fileName || "",
|
|
492
508
|
fileSize: bodySize,
|
|
493
509
|
timing: {
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
download: config.timing.transferMs || 0
|
|
510
|
+
...getTimingDurations(config),
|
|
511
|
+
download: getTimingDurations(config).download || 0
|
|
497
512
|
},
|
|
498
|
-
averageSpeed: config.
|
|
513
|
+
averageSpeed: getTimingDurations(config).download ? bodySize / getTimingDurations(config).download * 1000 : 0,
|
|
499
514
|
config: sanitizeConfig(config)
|
|
500
515
|
};
|
|
501
516
|
downloadResult.emit("finish", downloadFinishEvent);
|
|
@@ -517,13 +532,12 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
517
532
|
urls: buildUrlTree(config, url),
|
|
518
533
|
uploadSize: config.transfer.requestSize || 0,
|
|
519
534
|
timing: {
|
|
520
|
-
|
|
521
|
-
upload: config.
|
|
522
|
-
waiting: config.
|
|
523
|
-
download: config.timing.transferMs
|
|
535
|
+
...getTimingDurations(config),
|
|
536
|
+
upload: getTimingDurations(config).firstByte || 0,
|
|
537
|
+
waiting: getTimingDurations(config).download > 0 && getTimingDurations(config).firstByte > 0 ? getTimingDurations(config).download - getTimingDurations(config).firstByte : 0
|
|
524
538
|
},
|
|
525
|
-
averageUploadSpeed: config.
|
|
526
|
-
averageDownloadSpeed: config.
|
|
539
|
+
averageUploadSpeed: getTimingDurations(config).firstByte && config.transfer.requestSize ? config.transfer.requestSize / getTimingDurations(config).firstByte * 1000 : 0,
|
|
540
|
+
averageDownloadSpeed: getTimingDurations(config).download ? bodySize / getTimingDurations(config).download * 1000 : 0,
|
|
527
541
|
config: sanitizeConfig(config)
|
|
528
542
|
};
|
|
529
543
|
uploadResult.emit("finish", uploadFinishEvent);
|
package/dist/adapters/xhr.cjs
CHANGED
|
@@ -16,6 +16,30 @@ const Environment = {
|
|
|
16
16
|
hasFormData: typeof FormData !== "undefined",
|
|
17
17
|
hasBlob: typeof Blob !== "undefined"
|
|
18
18
|
};
|
|
19
|
+
function updateTiming(config, timing, bodySize) {
|
|
20
|
+
const now = performance.now();
|
|
21
|
+
config.timing.domainLookupStart = config.timing.startTime;
|
|
22
|
+
config.timing.domainLookupEnd = config.timing.startTime;
|
|
23
|
+
config.timing.connectStart = config.timing.startTime;
|
|
24
|
+
config.timing.secureConnectionStart = 0;
|
|
25
|
+
config.timing.connectEnd = config.timing.startTime;
|
|
26
|
+
config.timing.requestStart = config.timing.startTime;
|
|
27
|
+
config.timing.responseStart = timing.firstByteTime || config.timing.startTime;
|
|
28
|
+
config.timing.responseEnd = now;
|
|
29
|
+
config.transfer.bodySize = bodySize;
|
|
30
|
+
config.transfer.responseSize = bodySize;
|
|
31
|
+
}
|
|
32
|
+
function getTimingDurations(config) {
|
|
33
|
+
const t = config.timing;
|
|
34
|
+
return {
|
|
35
|
+
total: t.responseEnd - t.startTime,
|
|
36
|
+
dns: t.domainLookupEnd - t.domainLookupStart,
|
|
37
|
+
tcp: t.secureConnectionStart > 0 ? t.secureConnectionStart - t.connectStart : t.connectEnd - t.connectStart,
|
|
38
|
+
tls: t.secureConnectionStart > 0 ? t.connectEnd - t.secureConnectionStart : undefined,
|
|
39
|
+
firstByte: t.responseStart - t.startTime,
|
|
40
|
+
download: t.responseEnd - t.responseStart
|
|
41
|
+
};
|
|
42
|
+
}
|
|
19
43
|
const responseCacheInstances = new Map;
|
|
20
44
|
function getCacheConfigKey(option) {
|
|
21
45
|
if (option === true)
|
|
@@ -258,10 +282,11 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
|
|
|
258
282
|
const maxRetries = config?.retry?.maxRetries || 0;
|
|
259
283
|
const incrementDelay = config?.retry?.incrementDelay || false;
|
|
260
284
|
const statusCodes = config?.retry?.statusCodes;
|
|
285
|
+
const startTime = performance.now();
|
|
261
286
|
const timing = {
|
|
262
|
-
startTime
|
|
263
|
-
startTimestamp: Date.now()
|
|
287
|
+
startTime
|
|
264
288
|
};
|
|
289
|
+
config.timing.startTime = startTime;
|
|
265
290
|
const ABSOLUTE_MAX_ATTEMPTS = 50;
|
|
266
291
|
let totalAttempts = 0;
|
|
267
292
|
config.setSignal();
|
|
@@ -326,7 +351,6 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
326
351
|
config.isSecure = isSecure;
|
|
327
352
|
config.finalUrl = url;
|
|
328
353
|
config.network.protocol = isSecure ? "https" : "http";
|
|
329
|
-
config.timing.startTimestamp = timing.startTimestamp;
|
|
330
354
|
const xhr = new XMLHttpRequest;
|
|
331
355
|
xhr.open(fetchOptions.method.toUpperCase(), url, true);
|
|
332
356
|
const headers = toXHRHeaders(fetchOptions.headers);
|
|
@@ -370,9 +394,9 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
370
394
|
});
|
|
371
395
|
}
|
|
372
396
|
xhr.onprogress = (event) => {
|
|
373
|
-
if (!
|
|
397
|
+
if (!timing.firstByteTime) {
|
|
374
398
|
timing.firstByteTime = performance.now();
|
|
375
|
-
config.timing.
|
|
399
|
+
config.timing.responseStart = timing.firstByteTime;
|
|
376
400
|
}
|
|
377
401
|
if (eventEmitter) {
|
|
378
402
|
const progressEvent = {
|
|
@@ -404,13 +428,10 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
404
428
|
};
|
|
405
429
|
}
|
|
406
430
|
xhr.onload = () => {
|
|
407
|
-
if (!
|
|
431
|
+
if (!timing.firstByteTime) {
|
|
408
432
|
timing.firstByteTime = performance.now();
|
|
409
|
-
config.timing.
|
|
433
|
+
config.timing.responseStart = timing.firstByteTime;
|
|
410
434
|
}
|
|
411
|
-
config.timing.endTimestamp = Date.now();
|
|
412
|
-
config.timing.durationMs = performance.now() - timing.startTime;
|
|
413
|
-
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
414
435
|
const status = xhr.status;
|
|
415
436
|
const statusText = xhr.statusText;
|
|
416
437
|
const responseHeaders = parseXHRHeaders(xhr);
|
|
@@ -427,8 +448,8 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
427
448
|
contentLength: contentLength ? parseInt(contentLength, 10) : undefined,
|
|
428
449
|
cookies: cookies.array,
|
|
429
450
|
timing: {
|
|
430
|
-
firstByte: config.timing.
|
|
431
|
-
total: performance.now() - timing.startTime
|
|
451
|
+
firstByte: config.timing.responseStart - config.timing.startTime,
|
|
452
|
+
total: performance.now() - config.timing.startTime
|
|
432
453
|
}
|
|
433
454
|
};
|
|
434
455
|
eventEmitter.emit("headers", headersEvent);
|
|
@@ -472,8 +493,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
472
493
|
responseData = text;
|
|
473
494
|
}
|
|
474
495
|
}
|
|
475
|
-
config
|
|
476
|
-
config.transfer.responseSize = bodySize;
|
|
496
|
+
updateTiming(config, timing, bodySize);
|
|
477
497
|
if (status >= 400) {
|
|
478
498
|
const error = builErrorFromResponse(`HTTP Error ${status}: ${statusText}`, {
|
|
479
499
|
status,
|
|
@@ -509,14 +529,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
509
529
|
finalUrl: xhr.responseURL || url,
|
|
510
530
|
cookies,
|
|
511
531
|
urls: buildUrlTree(config, xhr.responseURL || url),
|
|
512
|
-
timing:
|
|
513
|
-
total: config.timing.durationMs || 0,
|
|
514
|
-
dns: config.timing.dnsMs,
|
|
515
|
-
tcp: config.timing.tcpMs,
|
|
516
|
-
tls: config.timing.tlsMs,
|
|
517
|
-
firstByte: config.timing.ttfbMs,
|
|
518
|
-
download: config.timing.transferMs
|
|
519
|
-
},
|
|
532
|
+
timing: getTimingDurations(config),
|
|
520
533
|
config: sanitizeConfig(config)
|
|
521
534
|
};
|
|
522
535
|
streamResult.emit("finish", streamFinishEvent);
|
|
@@ -537,14 +550,10 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
537
550
|
fileName: config.fileName || "",
|
|
538
551
|
fileSize: bodySize,
|
|
539
552
|
timing: {
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
tcp: config.timing.tcpMs,
|
|
543
|
-
tls: config.timing.tlsMs,
|
|
544
|
-
firstByte: config.timing.ttfbMs,
|
|
545
|
-
download: config.timing.transferMs || 0
|
|
553
|
+
...getTimingDurations(config),
|
|
554
|
+
download: getTimingDurations(config).download || 0
|
|
546
555
|
},
|
|
547
|
-
averageSpeed: config.
|
|
556
|
+
averageSpeed: getTimingDurations(config).download ? bodySize / getTimingDurations(config).download * 1000 : 0,
|
|
548
557
|
config: sanitizeConfig(config)
|
|
549
558
|
};
|
|
550
559
|
downloadResult.emit("finish", downloadFinishEvent);
|
|
@@ -566,16 +575,12 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
566
575
|
urls: buildUrlTree(config, xhr.responseURL || url),
|
|
567
576
|
uploadSize: config.transfer.requestSize || 0,
|
|
568
577
|
timing: {
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
tls: config.timing.tlsMs,
|
|
573
|
-
upload: config.timing.transferMs || 0,
|
|
574
|
-
waiting: config.timing.ttfbMs || 0,
|
|
575
|
-
download: config.timing.transferMs
|
|
578
|
+
...getTimingDurations(config),
|
|
579
|
+
upload: getTimingDurations(config).firstByte || 0,
|
|
580
|
+
waiting: getTimingDurations(config).download > 0 && getTimingDurations(config).firstByte > 0 ? getTimingDurations(config).download - getTimingDurations(config).firstByte : 0
|
|
576
581
|
},
|
|
577
|
-
averageUploadSpeed: config.
|
|
578
|
-
averageDownloadSpeed: config.
|
|
582
|
+
averageUploadSpeed: getTimingDurations(config).firstByte && config.transfer.requestSize ? config.transfer.requestSize / getTimingDurations(config).firstByte * 1000 : 0,
|
|
583
|
+
averageDownloadSpeed: getTimingDurations(config).download ? bodySize / getTimingDurations(config).download * 1000 : 0,
|
|
579
584
|
config: sanitizeConfig(config)
|
|
580
585
|
};
|
|
581
586
|
uploadResult.emit("finish", uploadFinishEvent);
|
package/dist/adapters/xhr.js
CHANGED
|
@@ -16,6 +16,30 @@ const Environment = {
|
|
|
16
16
|
hasFormData: typeof FormData !== "undefined",
|
|
17
17
|
hasBlob: typeof Blob !== "undefined"
|
|
18
18
|
};
|
|
19
|
+
function updateTiming(config, timing, bodySize) {
|
|
20
|
+
const now = performance.now();
|
|
21
|
+
config.timing.domainLookupStart = config.timing.startTime;
|
|
22
|
+
config.timing.domainLookupEnd = config.timing.startTime;
|
|
23
|
+
config.timing.connectStart = config.timing.startTime;
|
|
24
|
+
config.timing.secureConnectionStart = 0;
|
|
25
|
+
config.timing.connectEnd = config.timing.startTime;
|
|
26
|
+
config.timing.requestStart = config.timing.startTime;
|
|
27
|
+
config.timing.responseStart = timing.firstByteTime || config.timing.startTime;
|
|
28
|
+
config.timing.responseEnd = now;
|
|
29
|
+
config.transfer.bodySize = bodySize;
|
|
30
|
+
config.transfer.responseSize = bodySize;
|
|
31
|
+
}
|
|
32
|
+
function getTimingDurations(config) {
|
|
33
|
+
const t = config.timing;
|
|
34
|
+
return {
|
|
35
|
+
total: t.responseEnd - t.startTime,
|
|
36
|
+
dns: t.domainLookupEnd - t.domainLookupStart,
|
|
37
|
+
tcp: t.secureConnectionStart > 0 ? t.secureConnectionStart - t.connectStart : t.connectEnd - t.connectStart,
|
|
38
|
+
tls: t.secureConnectionStart > 0 ? t.connectEnd - t.secureConnectionStart : undefined,
|
|
39
|
+
firstByte: t.responseStart - t.startTime,
|
|
40
|
+
download: t.responseEnd - t.responseStart
|
|
41
|
+
};
|
|
42
|
+
}
|
|
19
43
|
const responseCacheInstances = new Map;
|
|
20
44
|
function getCacheConfigKey(option) {
|
|
21
45
|
if (option === true)
|
|
@@ -258,10 +282,11 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
|
|
|
258
282
|
const maxRetries = config?.retry?.maxRetries || 0;
|
|
259
283
|
const incrementDelay = config?.retry?.incrementDelay || false;
|
|
260
284
|
const statusCodes = config?.retry?.statusCodes;
|
|
285
|
+
const startTime = performance.now();
|
|
261
286
|
const timing = {
|
|
262
|
-
startTime
|
|
263
|
-
startTimestamp: Date.now()
|
|
287
|
+
startTime
|
|
264
288
|
};
|
|
289
|
+
config.timing.startTime = startTime;
|
|
265
290
|
const ABSOLUTE_MAX_ATTEMPTS = 50;
|
|
266
291
|
let totalAttempts = 0;
|
|
267
292
|
config.setSignal();
|
|
@@ -326,7 +351,6 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
326
351
|
config.isSecure = isSecure;
|
|
327
352
|
config.finalUrl = url;
|
|
328
353
|
config.network.protocol = isSecure ? "https" : "http";
|
|
329
|
-
config.timing.startTimestamp = timing.startTimestamp;
|
|
330
354
|
const xhr = new XMLHttpRequest;
|
|
331
355
|
xhr.open(fetchOptions.method.toUpperCase(), url, true);
|
|
332
356
|
const headers = toXHRHeaders(fetchOptions.headers);
|
|
@@ -370,9 +394,9 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
370
394
|
});
|
|
371
395
|
}
|
|
372
396
|
xhr.onprogress = (event) => {
|
|
373
|
-
if (!
|
|
397
|
+
if (!timing.firstByteTime) {
|
|
374
398
|
timing.firstByteTime = performance.now();
|
|
375
|
-
config.timing.
|
|
399
|
+
config.timing.responseStart = timing.firstByteTime;
|
|
376
400
|
}
|
|
377
401
|
if (eventEmitter) {
|
|
378
402
|
const progressEvent = {
|
|
@@ -404,13 +428,10 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
404
428
|
};
|
|
405
429
|
}
|
|
406
430
|
xhr.onload = () => {
|
|
407
|
-
if (!
|
|
431
|
+
if (!timing.firstByteTime) {
|
|
408
432
|
timing.firstByteTime = performance.now();
|
|
409
|
-
config.timing.
|
|
433
|
+
config.timing.responseStart = timing.firstByteTime;
|
|
410
434
|
}
|
|
411
|
-
config.timing.endTimestamp = Date.now();
|
|
412
|
-
config.timing.durationMs = performance.now() - timing.startTime;
|
|
413
|
-
config.timing.transferMs = timing.firstByteTime ? performance.now() - timing.firstByteTime : config.timing.durationMs;
|
|
414
435
|
const status = xhr.status;
|
|
415
436
|
const statusText = xhr.statusText;
|
|
416
437
|
const responseHeaders = parseXHRHeaders(xhr);
|
|
@@ -427,8 +448,8 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
427
448
|
contentLength: contentLength ? parseInt(contentLength, 10) : undefined,
|
|
428
449
|
cookies: cookies.array,
|
|
429
450
|
timing: {
|
|
430
|
-
firstByte: config.timing.
|
|
431
|
-
total: performance.now() - timing.startTime
|
|
451
|
+
firstByte: config.timing.responseStart - config.timing.startTime,
|
|
452
|
+
total: performance.now() - config.timing.startTime
|
|
432
453
|
}
|
|
433
454
|
};
|
|
434
455
|
eventEmitter.emit("headers", headersEvent);
|
|
@@ -472,8 +493,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
472
493
|
responseData = text;
|
|
473
494
|
}
|
|
474
495
|
}
|
|
475
|
-
config
|
|
476
|
-
config.transfer.responseSize = bodySize;
|
|
496
|
+
updateTiming(config, timing, bodySize);
|
|
477
497
|
if (status >= 400) {
|
|
478
498
|
const error = builErrorFromResponse(`HTTP Error ${status}: ${statusText}`, {
|
|
479
499
|
status,
|
|
@@ -509,14 +529,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
509
529
|
finalUrl: xhr.responseURL || url,
|
|
510
530
|
cookies,
|
|
511
531
|
urls: buildUrlTree(config, xhr.responseURL || url),
|
|
512
|
-
timing:
|
|
513
|
-
total: config.timing.durationMs || 0,
|
|
514
|
-
dns: config.timing.dnsMs,
|
|
515
|
-
tcp: config.timing.tcpMs,
|
|
516
|
-
tls: config.timing.tlsMs,
|
|
517
|
-
firstByte: config.timing.ttfbMs,
|
|
518
|
-
download: config.timing.transferMs
|
|
519
|
-
},
|
|
532
|
+
timing: getTimingDurations(config),
|
|
520
533
|
config: sanitizeConfig(config)
|
|
521
534
|
};
|
|
522
535
|
streamResult.emit("finish", streamFinishEvent);
|
|
@@ -537,14 +550,10 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
537
550
|
fileName: config.fileName || "",
|
|
538
551
|
fileSize: bodySize,
|
|
539
552
|
timing: {
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
tcp: config.timing.tcpMs,
|
|
543
|
-
tls: config.timing.tlsMs,
|
|
544
|
-
firstByte: config.timing.ttfbMs,
|
|
545
|
-
download: config.timing.transferMs || 0
|
|
553
|
+
...getTimingDurations(config),
|
|
554
|
+
download: getTimingDurations(config).download || 0
|
|
546
555
|
},
|
|
547
|
-
averageSpeed: config.
|
|
556
|
+
averageSpeed: getTimingDurations(config).download ? bodySize / getTimingDurations(config).download * 1000 : 0,
|
|
548
557
|
config: sanitizeConfig(config)
|
|
549
558
|
};
|
|
550
559
|
downloadResult.emit("finish", downloadFinishEvent);
|
|
@@ -566,16 +575,12 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
566
575
|
urls: buildUrlTree(config, xhr.responseURL || url),
|
|
567
576
|
uploadSize: config.transfer.requestSize || 0,
|
|
568
577
|
timing: {
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
tls: config.timing.tlsMs,
|
|
573
|
-
upload: config.timing.transferMs || 0,
|
|
574
|
-
waiting: config.timing.ttfbMs || 0,
|
|
575
|
-
download: config.timing.transferMs
|
|
578
|
+
...getTimingDurations(config),
|
|
579
|
+
upload: getTimingDurations(config).firstByte || 0,
|
|
580
|
+
waiting: getTimingDurations(config).download > 0 && getTimingDurations(config).firstByte > 0 ? getTimingDurations(config).download - getTimingDurations(config).firstByte : 0
|
|
576
581
|
},
|
|
577
|
-
averageUploadSpeed: config.
|
|
578
|
-
averageDownloadSpeed: config.
|
|
582
|
+
averageUploadSpeed: getTimingDurations(config).firstByte && config.transfer.requestSize ? config.transfer.requestSize / getTimingDurations(config).firstByte * 1000 : 0,
|
|
583
|
+
averageDownloadSpeed: getTimingDurations(config).download ? bodySize / getTimingDurations(config).download * 1000 : 0,
|
|
579
584
|
config: sanitizeConfig(config)
|
|
580
585
|
};
|
|
581
586
|
uploadResult.emit("finish", uploadFinishEvent);
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_npvfdd = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_npvfdd.LRUCache;;
|
|
3
|
+
const _mod_4em23l = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_4em23l.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_4em23l.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_4em23l.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_ij8xvl = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_ij8xvl.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_ij8xvl.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_ox3maj = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_ox3maj.FileCacher;;
|
|
12
|
+
const _mod_ihxikf = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_ihxikf.UrlStore;;
|
package/dist/crawler.d.ts
CHANGED
|
@@ -2029,24 +2029,26 @@ export interface RezoConfig {
|
|
|
2029
2029
|
* - removeAllCookiesSync(): Clear all cookies
|
|
2030
2030
|
*/
|
|
2031
2031
|
cookieJar: RezoCookieJar;
|
|
2032
|
-
/** @description Comprehensive timing information */
|
|
2032
|
+
/** @description Comprehensive timing information (matches PerformanceResourceTiming API) */
|
|
2033
2033
|
timing: {
|
|
2034
|
-
/** @description Request start timestamp (
|
|
2035
|
-
|
|
2036
|
-
/** @description
|
|
2037
|
-
|
|
2038
|
-
/** @description DNS lookup
|
|
2039
|
-
|
|
2040
|
-
/** @description
|
|
2041
|
-
|
|
2042
|
-
/** @description TLS handshake
|
|
2043
|
-
|
|
2044
|
-
/** @description
|
|
2045
|
-
|
|
2046
|
-
/** @description
|
|
2047
|
-
|
|
2048
|
-
/** @description
|
|
2049
|
-
|
|
2034
|
+
/** @description Request start timestamp (performance.now() value when request began) */
|
|
2035
|
+
startTime: number;
|
|
2036
|
+
/** @description Timestamp when DNS lookup started */
|
|
2037
|
+
domainLookupStart: number;
|
|
2038
|
+
/** @description Timestamp when DNS lookup ended */
|
|
2039
|
+
domainLookupEnd: number;
|
|
2040
|
+
/** @description Timestamp when connection started */
|
|
2041
|
+
connectStart: number;
|
|
2042
|
+
/** @description Timestamp when TLS handshake started (0 for HTTP) */
|
|
2043
|
+
secureConnectionStart: number;
|
|
2044
|
+
/** @description Timestamp when connection completed */
|
|
2045
|
+
connectEnd: number;
|
|
2046
|
+
/** @description Timestamp when request was sent */
|
|
2047
|
+
requestStart: number;
|
|
2048
|
+
/** @description Timestamp when first byte of response received */
|
|
2049
|
+
responseStart: number;
|
|
2050
|
+
/** @description Timestamp when response completed */
|
|
2051
|
+
responseEnd: number;
|
|
2050
2052
|
};
|
|
2051
2053
|
/** @description Network connection information */
|
|
2052
2054
|
network: {
|
|
@@ -2175,7 +2177,6 @@ declare class RezoError<T = any> extends Error {
|
|
|
2175
2177
|
readonly isSocksError: boolean;
|
|
2176
2178
|
readonly isTlsError: boolean;
|
|
2177
2179
|
readonly isRetryable: boolean;
|
|
2178
|
-
readonly details: string;
|
|
2179
2180
|
readonly suggestion: string;
|
|
2180
2181
|
constructor(message: string, config: RezoConfig, code?: string, request?: RezoHttpRequest, response?: RezoResponse<T>);
|
|
2181
2182
|
static isRezoError(error: unknown): error is RezoError;
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_k28o05 = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_k28o05.Crawler;;
|
|
3
|
+
const _mod_qjgrqf = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_qjgrqf.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_qjgrqf.Domain;;
|