rezo 1.0.41 → 1.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +143 -32
- package/dist/adapters/curl.js +143 -32
- package/dist/adapters/entries/curl.d.ts +65 -0
- package/dist/adapters/entries/fetch.d.ts +65 -0
- package/dist/adapters/entries/http.d.ts +65 -0
- package/dist/adapters/entries/http2.d.ts +65 -0
- package/dist/adapters/entries/react-native.d.ts +65 -0
- package/dist/adapters/entries/xhr.d.ts +65 -0
- package/dist/adapters/fetch.cjs +98 -12
- package/dist/adapters/fetch.js +98 -12
- package/dist/adapters/http.cjs +26 -14
- package/dist/adapters/http.js +26 -14
- package/dist/adapters/http2.cjs +756 -227
- package/dist/adapters/http2.js +756 -227
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/xhr.cjs +94 -2
- package/dist/adapters/xhr.js +94 -2
- package/dist/cache/dns-cache.cjs +5 -3
- package/dist/cache/dns-cache.js +5 -3
- package/dist/cache/file-cacher.cjs +7 -1
- package/dist/cache/file-cacher.js +7 -1
- package/dist/cache/index.cjs +15 -13
- package/dist/cache/index.js +1 -0
- package/dist/cache/navigation-history.cjs +298 -0
- package/dist/cache/navigation-history.js +296 -0
- package/dist/cache/url-store.cjs +7 -1
- package/dist/cache/url-store.js +7 -1
- package/dist/core/rezo.cjs +7 -0
- package/dist/core/rezo.js +7 -0
- package/dist/crawler.d.ts +196 -11
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +27 -24
- package/dist/index.d.ts +73 -0
- package/dist/index.js +1 -0
- package/dist/internal/agents/base.cjs +113 -0
- package/dist/internal/agents/base.js +110 -0
- package/dist/internal/agents/http-proxy.cjs +89 -0
- package/dist/internal/agents/http-proxy.js +86 -0
- package/dist/internal/agents/https-proxy.cjs +176 -0
- package/dist/internal/agents/https-proxy.js +173 -0
- package/dist/internal/agents/index.cjs +10 -0
- package/dist/internal/agents/index.js +5 -0
- package/dist/internal/agents/socks-client.cjs +571 -0
- package/dist/internal/agents/socks-client.js +567 -0
- package/dist/internal/agents/socks-proxy.cjs +75 -0
- package/dist/internal/agents/socks-proxy.js +72 -0
- package/dist/platform/browser.d.ts +65 -0
- package/dist/platform/bun.d.ts +65 -0
- package/dist/platform/deno.d.ts +65 -0
- package/dist/platform/node.d.ts +65 -0
- package/dist/platform/react-native.d.ts +65 -0
- package/dist/platform/worker.d.ts +65 -0
- package/dist/plugin/crawler-options.cjs +1 -1
- package/dist/plugin/crawler-options.js +1 -1
- package/dist/plugin/crawler.cjs +192 -1
- package/dist/plugin/crawler.js +192 -1
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +18 -16
- package/dist/proxy/index.js +17 -12
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/buildError.cjs +11 -2
- package/dist/responses/buildError.js +11 -2
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +1 -17
- package/dist/utils/agent-pool.js +1 -17
- package/dist/utils/curl.cjs +317 -0
- package/dist/utils/curl.js +314 -0
- package/package.json +1 -1
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_63iyz4 = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_63iyz4.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_63iyz4.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_63iyz4.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_63iyz4.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_63iyz4.selectAdapter;;
|
package/dist/adapters/xhr.cjs
CHANGED
|
@@ -16,6 +16,85 @@ const Environment = {
|
|
|
16
16
|
hasFormData: typeof FormData !== "undefined",
|
|
17
17
|
hasBlob: typeof Blob !== "undefined"
|
|
18
18
|
};
|
|
19
|
+
const debugLog = {
|
|
20
|
+
requestStart: (config, url, method) => {
|
|
21
|
+
if (config.debug) {
|
|
22
|
+
console.log(`
|
|
23
|
+
[Rezo Debug] ─────────────────────────────────────`);
|
|
24
|
+
console.log(`[Rezo Debug] ${method} ${url}`);
|
|
25
|
+
console.log(`[Rezo Debug] Request ID: ${config.requestId}`);
|
|
26
|
+
console.log(`[Rezo Debug] Adapter: xhr`);
|
|
27
|
+
if (config.originalRequest?.headers) {
|
|
28
|
+
const headers = config.originalRequest.headers instanceof RezoHeaders ? config.originalRequest.headers.toObject() : config.originalRequest.headers;
|
|
29
|
+
console.log(`[Rezo Debug] Request Headers:`, JSON.stringify(headers, null, 2));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (config.trackUrl) {
|
|
33
|
+
console.log(`[Rezo Track] → ${method} ${url}`);
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
retry: (config, attempt, maxRetries, statusCode, delay) => {
|
|
37
|
+
if (config.debug) {
|
|
38
|
+
console.log(`[Rezo Debug] Retry ${attempt}/${maxRetries} after status ${statusCode}${delay > 0 ? ` (waiting ${delay}ms)` : ""}`);
|
|
39
|
+
}
|
|
40
|
+
if (config.trackUrl) {
|
|
41
|
+
console.log(`[Rezo Track] ⟳ Retry ${attempt}/${maxRetries} (status ${statusCode})`);
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
maxRetries: (config, maxRetries) => {
|
|
45
|
+
if (config.debug) {
|
|
46
|
+
console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
|
|
47
|
+
}
|
|
48
|
+
if (config.trackUrl) {
|
|
49
|
+
console.log(`[Rezo Track] ✗ Max retries reached`);
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
response: (config, status, statusText, duration) => {
|
|
53
|
+
if (config.debug) {
|
|
54
|
+
console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
|
|
55
|
+
}
|
|
56
|
+
if (config.trackUrl) {
|
|
57
|
+
console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
responseHeaders: (config, headers) => {
|
|
61
|
+
if (config.debug) {
|
|
62
|
+
console.log(`[Rezo Debug] Response Headers:`, JSON.stringify(headers, null, 2));
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
cookies: (config, cookieCount) => {
|
|
66
|
+
if (config.debug && cookieCount > 0) {
|
|
67
|
+
console.log(`[Rezo Debug] Cookies received: ${cookieCount}`);
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
timing: (config, timing) => {
|
|
71
|
+
if (config.debug) {
|
|
72
|
+
const parts = [];
|
|
73
|
+
if (timing.ttfb)
|
|
74
|
+
parts.push(`TTFB: ${timing.ttfb.toFixed(2)}ms`);
|
|
75
|
+
if (timing.total)
|
|
76
|
+
parts.push(`Total: ${timing.total.toFixed(2)}ms`);
|
|
77
|
+
if (parts.length > 0) {
|
|
78
|
+
console.log(`[Rezo Debug] Timing: ${parts.join(" | ")}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
complete: (config, url) => {
|
|
83
|
+
if (config.debug) {
|
|
84
|
+
console.log(`[Rezo Debug] Request complete: ${url}`);
|
|
85
|
+
console.log(`[Rezo Debug] ─────────────────────────────────────
|
|
86
|
+
`);
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
error: (config, error) => {
|
|
90
|
+
if (config.debug) {
|
|
91
|
+
console.log(`[Rezo Debug] Error: ${error instanceof Error ? error.message : error}`);
|
|
92
|
+
}
|
|
93
|
+
if (config.trackUrl) {
|
|
94
|
+
console.log(`[Rezo Track] ✗ Error: ${error instanceof Error ? error.message : error}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
19
98
|
function updateTiming(config, timing, bodySize) {
|
|
20
99
|
const now = performance.now();
|
|
21
100
|
config.timing.domainLookupStart = config.timing.startTime;
|
|
@@ -330,11 +409,14 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
|
|
|
330
409
|
throw response;
|
|
331
410
|
}
|
|
332
411
|
if (maxRetries <= retries) {
|
|
412
|
+
debugLog.maxRetries(config, maxRetries);
|
|
333
413
|
throw response;
|
|
334
414
|
}
|
|
335
415
|
retries++;
|
|
336
|
-
|
|
337
|
-
|
|
416
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
417
|
+
debugLog.retry(config, retries, maxRetries, response.status || 0, currentDelay);
|
|
418
|
+
if (currentDelay > 0) {
|
|
419
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
338
420
|
}
|
|
339
421
|
}
|
|
340
422
|
config.retryAttempts++;
|
|
@@ -359,6 +441,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
359
441
|
config.isSecure = isSecure;
|
|
360
442
|
config.finalUrl = url;
|
|
361
443
|
config.network.protocol = isSecure ? "https" : "http";
|
|
444
|
+
debugLog.requestStart(config, url, fetchOptions.method?.toUpperCase() || "GET");
|
|
362
445
|
const xhr = new XMLHttpRequest;
|
|
363
446
|
xhr.open(fetchOptions.method.toUpperCase(), url, true);
|
|
364
447
|
const headers = toXHRHeaders(fetchOptions.headers);
|
|
@@ -513,6 +596,15 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
513
596
|
resolve(error);
|
|
514
597
|
return;
|
|
515
598
|
}
|
|
599
|
+
const duration = performance.now() - timing.startTime;
|
|
600
|
+
debugLog.response(config, status, statusText, duration);
|
|
601
|
+
debugLog.responseHeaders(config, responseHeaders.toObject());
|
|
602
|
+
debugLog.cookies(config, cookies.array.length);
|
|
603
|
+
debugLog.timing(config, {
|
|
604
|
+
ttfb: config.timing.responseStart - config.timing.startTime,
|
|
605
|
+
total: duration
|
|
606
|
+
});
|
|
607
|
+
debugLog.complete(config, xhr.responseURL || url);
|
|
516
608
|
const finalResponse = {
|
|
517
609
|
data: responseData,
|
|
518
610
|
status,
|
package/dist/adapters/xhr.js
CHANGED
|
@@ -16,6 +16,85 @@ const Environment = {
|
|
|
16
16
|
hasFormData: typeof FormData !== "undefined",
|
|
17
17
|
hasBlob: typeof Blob !== "undefined"
|
|
18
18
|
};
|
|
19
|
+
const debugLog = {
|
|
20
|
+
requestStart: (config, url, method) => {
|
|
21
|
+
if (config.debug) {
|
|
22
|
+
console.log(`
|
|
23
|
+
[Rezo Debug] ─────────────────────────────────────`);
|
|
24
|
+
console.log(`[Rezo Debug] ${method} ${url}`);
|
|
25
|
+
console.log(`[Rezo Debug] Request ID: ${config.requestId}`);
|
|
26
|
+
console.log(`[Rezo Debug] Adapter: xhr`);
|
|
27
|
+
if (config.originalRequest?.headers) {
|
|
28
|
+
const headers = config.originalRequest.headers instanceof RezoHeaders ? config.originalRequest.headers.toObject() : config.originalRequest.headers;
|
|
29
|
+
console.log(`[Rezo Debug] Request Headers:`, JSON.stringify(headers, null, 2));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
if (config.trackUrl) {
|
|
33
|
+
console.log(`[Rezo Track] → ${method} ${url}`);
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
retry: (config, attempt, maxRetries, statusCode, delay) => {
|
|
37
|
+
if (config.debug) {
|
|
38
|
+
console.log(`[Rezo Debug] Retry ${attempt}/${maxRetries} after status ${statusCode}${delay > 0 ? ` (waiting ${delay}ms)` : ""}`);
|
|
39
|
+
}
|
|
40
|
+
if (config.trackUrl) {
|
|
41
|
+
console.log(`[Rezo Track] ⟳ Retry ${attempt}/${maxRetries} (status ${statusCode})`);
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
maxRetries: (config, maxRetries) => {
|
|
45
|
+
if (config.debug) {
|
|
46
|
+
console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
|
|
47
|
+
}
|
|
48
|
+
if (config.trackUrl) {
|
|
49
|
+
console.log(`[Rezo Track] ✗ Max retries reached`);
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
response: (config, status, statusText, duration) => {
|
|
53
|
+
if (config.debug) {
|
|
54
|
+
console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
|
|
55
|
+
}
|
|
56
|
+
if (config.trackUrl) {
|
|
57
|
+
console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
responseHeaders: (config, headers) => {
|
|
61
|
+
if (config.debug) {
|
|
62
|
+
console.log(`[Rezo Debug] Response Headers:`, JSON.stringify(headers, null, 2));
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
cookies: (config, cookieCount) => {
|
|
66
|
+
if (config.debug && cookieCount > 0) {
|
|
67
|
+
console.log(`[Rezo Debug] Cookies received: ${cookieCount}`);
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
timing: (config, timing) => {
|
|
71
|
+
if (config.debug) {
|
|
72
|
+
const parts = [];
|
|
73
|
+
if (timing.ttfb)
|
|
74
|
+
parts.push(`TTFB: ${timing.ttfb.toFixed(2)}ms`);
|
|
75
|
+
if (timing.total)
|
|
76
|
+
parts.push(`Total: ${timing.total.toFixed(2)}ms`);
|
|
77
|
+
if (parts.length > 0) {
|
|
78
|
+
console.log(`[Rezo Debug] Timing: ${parts.join(" | ")}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
},
|
|
82
|
+
complete: (config, url) => {
|
|
83
|
+
if (config.debug) {
|
|
84
|
+
console.log(`[Rezo Debug] Request complete: ${url}`);
|
|
85
|
+
console.log(`[Rezo Debug] ─────────────────────────────────────
|
|
86
|
+
`);
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
error: (config, error) => {
|
|
90
|
+
if (config.debug) {
|
|
91
|
+
console.log(`[Rezo Debug] Error: ${error instanceof Error ? error.message : error}`);
|
|
92
|
+
}
|
|
93
|
+
if (config.trackUrl) {
|
|
94
|
+
console.log(`[Rezo Track] ✗ Error: ${error instanceof Error ? error.message : error}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
19
98
|
function updateTiming(config, timing, bodySize) {
|
|
20
99
|
const now = performance.now();
|
|
21
100
|
config.timing.domainLookupStart = config.timing.startTime;
|
|
@@ -330,11 +409,14 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
|
|
|
330
409
|
throw response;
|
|
331
410
|
}
|
|
332
411
|
if (maxRetries <= retries) {
|
|
412
|
+
debugLog.maxRetries(config, maxRetries);
|
|
333
413
|
throw response;
|
|
334
414
|
}
|
|
335
415
|
retries++;
|
|
336
|
-
|
|
337
|
-
|
|
416
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
417
|
+
debugLog.retry(config, retries, maxRetries, response.status || 0, currentDelay);
|
|
418
|
+
if (currentDelay > 0) {
|
|
419
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
338
420
|
}
|
|
339
421
|
}
|
|
340
422
|
config.retryAttempts++;
|
|
@@ -359,6 +441,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
359
441
|
config.isSecure = isSecure;
|
|
360
442
|
config.finalUrl = url;
|
|
361
443
|
config.network.protocol = isSecure ? "https" : "http";
|
|
444
|
+
debugLog.requestStart(config, url, fetchOptions.method?.toUpperCase() || "GET");
|
|
362
445
|
const xhr = new XMLHttpRequest;
|
|
363
446
|
xhr.open(fetchOptions.method.toUpperCase(), url, true);
|
|
364
447
|
const headers = toXHRHeaders(fetchOptions.headers);
|
|
@@ -513,6 +596,15 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
|
|
|
513
596
|
resolve(error);
|
|
514
597
|
return;
|
|
515
598
|
}
|
|
599
|
+
const duration = performance.now() - timing.startTime;
|
|
600
|
+
debugLog.response(config, status, statusText, duration);
|
|
601
|
+
debugLog.responseHeaders(config, responseHeaders.toObject());
|
|
602
|
+
debugLog.cookies(config, cookies.array.length);
|
|
603
|
+
debugLog.timing(config, {
|
|
604
|
+
ttfb: config.timing.responseStart - config.timing.startTime,
|
|
605
|
+
total: duration
|
|
606
|
+
});
|
|
607
|
+
debugLog.complete(config, xhr.responseURL || url);
|
|
516
608
|
const finalResponse = {
|
|
517
609
|
data: responseData,
|
|
518
610
|
status,
|
package/dist/cache/dns-cache.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const { LRUCache } = require('./lru-cache.cjs');
|
|
2
|
-
const dns = require("dns");
|
|
2
|
+
const dns = require("node:dns");
|
|
3
3
|
const DEFAULT_DNS_TTL = 60000;
|
|
4
4
|
const DEFAULT_DNS_MAX_ENTRIES = 1000;
|
|
5
5
|
|
|
@@ -24,10 +24,12 @@ class DNSCache {
|
|
|
24
24
|
const cached = this.cache.get(key);
|
|
25
25
|
if (cached && cached.addresses.length > 0) {
|
|
26
26
|
const address = cached.addresses[Math.floor(Math.random() * cached.addresses.length)];
|
|
27
|
-
|
|
27
|
+
if (address && typeof address === "string") {
|
|
28
|
+
return { address, family: cached.family };
|
|
29
|
+
}
|
|
28
30
|
}
|
|
29
31
|
const result = await this.resolveDNS(hostname, family);
|
|
30
|
-
if (result) {
|
|
32
|
+
if (result && result.address && typeof result.address === "string") {
|
|
31
33
|
this.cache.set(key, {
|
|
32
34
|
addresses: [result.address],
|
|
33
35
|
family: result.family,
|
package/dist/cache/dns-cache.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LRUCache } from './lru-cache.js';
|
|
2
|
-
import dns from "dns";
|
|
2
|
+
import dns from "node:dns";
|
|
3
3
|
const DEFAULT_DNS_TTL = 60000;
|
|
4
4
|
const DEFAULT_DNS_MAX_ENTRIES = 1000;
|
|
5
5
|
|
|
@@ -24,10 +24,12 @@ export class DNSCache {
|
|
|
24
24
|
const cached = this.cache.get(key);
|
|
25
25
|
if (cached && cached.addresses.length > 0) {
|
|
26
26
|
const address = cached.addresses[Math.floor(Math.random() * cached.addresses.length)];
|
|
27
|
-
|
|
27
|
+
if (address && typeof address === "string") {
|
|
28
|
+
return { address, family: cached.family };
|
|
29
|
+
}
|
|
28
30
|
}
|
|
29
31
|
const result = await this.resolveDNS(hostname, family);
|
|
30
|
-
if (result) {
|
|
32
|
+
if (result && result.address && typeof result.address === "string") {
|
|
31
33
|
this.cache.set(key, {
|
|
32
34
|
addresses: [result.address],
|
|
33
35
|
family: result.family,
|
|
@@ -45,7 +45,13 @@ async function createDatabase(dbPath) {
|
|
|
45
45
|
const { DatabaseSync } = await import("node:sqlite");
|
|
46
46
|
const db = new DatabaseSync(dbPath);
|
|
47
47
|
return {
|
|
48
|
-
run: (sql, ...params) =>
|
|
48
|
+
run: (sql, ...params) => {
|
|
49
|
+
if (params.length === 0) {
|
|
50
|
+
db.exec(sql);
|
|
51
|
+
} else {
|
|
52
|
+
db.prepare(sql).run(...params);
|
|
53
|
+
}
|
|
54
|
+
},
|
|
49
55
|
get: (sql, ...params) => {
|
|
50
56
|
const stmt = db.prepare(sql);
|
|
51
57
|
return stmt.get(...params);
|
|
@@ -45,7 +45,13 @@ async function createDatabase(dbPath) {
|
|
|
45
45
|
const { DatabaseSync } = await import("node:sqlite");
|
|
46
46
|
const db = new DatabaseSync(dbPath);
|
|
47
47
|
return {
|
|
48
|
-
run: (sql, ...params) =>
|
|
48
|
+
run: (sql, ...params) => {
|
|
49
|
+
if (params.length === 0) {
|
|
50
|
+
db.exec(sql);
|
|
51
|
+
} else {
|
|
52
|
+
db.prepare(sql).run(...params);
|
|
53
|
+
}
|
|
54
|
+
},
|
|
49
55
|
get: (sql, ...params) => {
|
|
50
56
|
const stmt = db.prepare(sql);
|
|
51
57
|
return stmt.get(...params);
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_6dj0p0 = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_6dj0p0.LRUCache;;
|
|
3
|
+
const _mod_9z4pm1 = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_9z4pm1.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_9z4pm1.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_9z4pm1.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_5ylf2b = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_5ylf2b.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_5ylf2b.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_fk65c2 = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_fk65c2.FileCacher;;
|
|
12
|
+
const _mod_7tzsb5 = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_7tzsb5.UrlStore;;
|
|
14
|
+
const _mod_04c8wb = require('./navigation-history.cjs');
|
|
15
|
+
exports.NavigationHistory = _mod_04c8wb.NavigationHistory;;
|
package/dist/cache/index.js
CHANGED
|
@@ -3,3 +3,4 @@ export { DNSCache, getGlobalDNSCache, resetGlobalDNSCache } from './dns-cache.js
|
|
|
3
3
|
export { ResponseCache, normalizeResponseCacheConfig } from './response-cache.js';
|
|
4
4
|
export { FileCacher } from './file-cacher.js';
|
|
5
5
|
export { UrlStore } from './url-store.js';
|
|
6
|
+
export { NavigationHistory } from './navigation-history.js';
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
const fs = require("node:fs");
|
|
2
|
+
const path = require("node:path");
|
|
3
|
+
const { createHash } = require("node:crypto");
|
|
4
|
+
function detectRuntime() {
|
|
5
|
+
if (typeof globalThis.Bun !== "undefined")
|
|
6
|
+
return "bun";
|
|
7
|
+
if (typeof globalThis.Deno !== "undefined")
|
|
8
|
+
return "deno";
|
|
9
|
+
return "node";
|
|
10
|
+
}
|
|
11
|
+
async function createDatabase(dbPath) {
|
|
12
|
+
const runtime = detectRuntime();
|
|
13
|
+
if (runtime === "bun") {
|
|
14
|
+
const { Database } = await import("bun:sqlite");
|
|
15
|
+
const db = new Database(dbPath);
|
|
16
|
+
return {
|
|
17
|
+
run: (sql, ...params) => db.run(sql, ...params),
|
|
18
|
+
get: (sql, ...params) => db.query(sql).get(...params),
|
|
19
|
+
all: (sql, ...params) => db.query(sql).all(...params),
|
|
20
|
+
close: () => db.close()
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
if (runtime === "deno") {
|
|
24
|
+
try {
|
|
25
|
+
const { Database } = await import("node:sqlite");
|
|
26
|
+
const db = new Database(dbPath);
|
|
27
|
+
return {
|
|
28
|
+
run: (sql, ...params) => db.exec(sql, params),
|
|
29
|
+
get: (sql, ...params) => {
|
|
30
|
+
const stmt = db.prepare(sql);
|
|
31
|
+
return stmt.get(...params);
|
|
32
|
+
},
|
|
33
|
+
all: (sql, ...params) => {
|
|
34
|
+
const stmt = db.prepare(sql);
|
|
35
|
+
return stmt.all(...params);
|
|
36
|
+
},
|
|
37
|
+
close: () => db.close()
|
|
38
|
+
};
|
|
39
|
+
} catch {
|
|
40
|
+
throw new Error("Deno SQLite support requires Node.js compatibility mode");
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
44
|
+
const db = new DatabaseSync(dbPath);
|
|
45
|
+
return {
|
|
46
|
+
run: (sql, ...params) => {
|
|
47
|
+
const stmt = db.prepare(sql);
|
|
48
|
+
stmt.run(...params);
|
|
49
|
+
},
|
|
50
|
+
get: (sql, ...params) => {
|
|
51
|
+
const stmt = db.prepare(sql);
|
|
52
|
+
return stmt.get(...params);
|
|
53
|
+
},
|
|
54
|
+
all: (sql, ...params) => {
|
|
55
|
+
const stmt = db.prepare(sql);
|
|
56
|
+
return stmt.all(...params);
|
|
57
|
+
},
|
|
58
|
+
close: () => db.close()
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
class NavigationHistory {
|
|
63
|
+
db = null;
|
|
64
|
+
options;
|
|
65
|
+
storeDir;
|
|
66
|
+
dbPath;
|
|
67
|
+
closed = false;
|
|
68
|
+
initPromise = null;
|
|
69
|
+
constructor(options = {}) {
|
|
70
|
+
this.options = {
|
|
71
|
+
storeDir: options.storeDir || "./navigation-history",
|
|
72
|
+
dbFileName: options.dbFileName || "navigation.db",
|
|
73
|
+
hashUrls: options.hashUrls ?? false
|
|
74
|
+
};
|
|
75
|
+
this.storeDir = path.resolve(this.options.storeDir);
|
|
76
|
+
this.dbPath = path.join(this.storeDir, this.options.dbFileName);
|
|
77
|
+
if (!fs.existsSync(this.storeDir)) {
|
|
78
|
+
fs.mkdirSync(this.storeDir, { recursive: true });
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
static async create(options = {}) {
|
|
82
|
+
const store = new NavigationHistory(options);
|
|
83
|
+
await store.initialize();
|
|
84
|
+
return store;
|
|
85
|
+
}
|
|
86
|
+
async initialize() {
|
|
87
|
+
if (this.initPromise)
|
|
88
|
+
return this.initPromise;
|
|
89
|
+
this.initPromise = (async () => {
|
|
90
|
+
this.db = await createDatabase(this.dbPath);
|
|
91
|
+
this.db.run(`
|
|
92
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
93
|
+
sessionId TEXT PRIMARY KEY,
|
|
94
|
+
baseUrl TEXT NOT NULL,
|
|
95
|
+
startedAt INTEGER NOT NULL,
|
|
96
|
+
lastActivityAt INTEGER NOT NULL,
|
|
97
|
+
status TEXT DEFAULT 'running',
|
|
98
|
+
urlsVisited INTEGER DEFAULT 0,
|
|
99
|
+
urlsQueued INTEGER DEFAULT 0,
|
|
100
|
+
urlsFailed INTEGER DEFAULT 0,
|
|
101
|
+
metadata TEXT
|
|
102
|
+
)
|
|
103
|
+
`);
|
|
104
|
+
this.db.run(`
|
|
105
|
+
CREATE TABLE IF NOT EXISTS queue (
|
|
106
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
107
|
+
sessionId TEXT NOT NULL,
|
|
108
|
+
urlKey TEXT NOT NULL,
|
|
109
|
+
originalUrl TEXT NOT NULL,
|
|
110
|
+
method TEXT DEFAULT 'GET',
|
|
111
|
+
priority INTEGER DEFAULT 0,
|
|
112
|
+
body TEXT,
|
|
113
|
+
headers TEXT,
|
|
114
|
+
metadata TEXT,
|
|
115
|
+
addedAt INTEGER NOT NULL,
|
|
116
|
+
UNIQUE(sessionId, urlKey)
|
|
117
|
+
)
|
|
118
|
+
`);
|
|
119
|
+
this.db.run(`
|
|
120
|
+
CREATE TABLE IF NOT EXISTS visited (
|
|
121
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
122
|
+
sessionId TEXT NOT NULL,
|
|
123
|
+
urlKey TEXT NOT NULL,
|
|
124
|
+
originalUrl TEXT NOT NULL,
|
|
125
|
+
status INTEGER,
|
|
126
|
+
visitedAt INTEGER NOT NULL,
|
|
127
|
+
finalUrl TEXT,
|
|
128
|
+
contentType TEXT,
|
|
129
|
+
errorMessage TEXT,
|
|
130
|
+
UNIQUE(sessionId, urlKey)
|
|
131
|
+
)
|
|
132
|
+
`);
|
|
133
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_queue_session ON queue(sessionId)");
|
|
134
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_queue_priority ON queue(sessionId, priority DESC)");
|
|
135
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_visited_session ON visited(sessionId)");
|
|
136
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_sessions_status ON sessions(status)");
|
|
137
|
+
})();
|
|
138
|
+
return this.initPromise;
|
|
139
|
+
}
|
|
140
|
+
getUrlKey(url) {
|
|
141
|
+
if (this.options.hashUrls) {
|
|
142
|
+
return createHash("sha256").update(url).digest("hex");
|
|
143
|
+
}
|
|
144
|
+
return url;
|
|
145
|
+
}
|
|
146
|
+
async createSession(sessionId, baseUrl, metadata) {
|
|
147
|
+
if (this.closed || !this.db)
|
|
148
|
+
throw new Error("NavigationHistory is closed");
|
|
149
|
+
const now = Date.now();
|
|
150
|
+
const session = {
|
|
151
|
+
sessionId,
|
|
152
|
+
baseUrl,
|
|
153
|
+
startedAt: now,
|
|
154
|
+
lastActivityAt: now,
|
|
155
|
+
status: "running",
|
|
156
|
+
urlsVisited: 0,
|
|
157
|
+
urlsQueued: 0,
|
|
158
|
+
urlsFailed: 0,
|
|
159
|
+
metadata: metadata ? JSON.stringify(metadata) : undefined
|
|
160
|
+
};
|
|
161
|
+
this.db.run(`INSERT OR REPLACE INTO sessions (sessionId, baseUrl, startedAt, lastActivityAt, status, urlsVisited, urlsQueued, urlsFailed, metadata)
|
|
162
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, baseUrl, now, now, "running", 0, 0, 0, session.metadata ?? null);
|
|
163
|
+
return session;
|
|
164
|
+
}
|
|
165
|
+
async getSession(sessionId) {
|
|
166
|
+
if (this.closed || !this.db)
|
|
167
|
+
throw new Error("NavigationHistory is closed");
|
|
168
|
+
return this.db.get("SELECT * FROM sessions WHERE sessionId = ?", sessionId);
|
|
169
|
+
}
|
|
170
|
+
async updateSessionStatus(sessionId, status) {
|
|
171
|
+
if (this.closed || !this.db)
|
|
172
|
+
throw new Error("NavigationHistory is closed");
|
|
173
|
+
this.db.run("UPDATE sessions SET status = ?, lastActivityAt = ? WHERE sessionId = ?", status, Date.now(), sessionId);
|
|
174
|
+
}
|
|
175
|
+
async updateSessionStats(sessionId, stats) {
|
|
176
|
+
if (this.closed || !this.db)
|
|
177
|
+
throw new Error("NavigationHistory is closed");
|
|
178
|
+
const updates = ["lastActivityAt = ?"];
|
|
179
|
+
const params = [Date.now()];
|
|
180
|
+
if (stats.urlsVisited !== undefined) {
|
|
181
|
+
updates.push("urlsVisited = ?");
|
|
182
|
+
params.push(stats.urlsVisited);
|
|
183
|
+
}
|
|
184
|
+
if (stats.urlsQueued !== undefined) {
|
|
185
|
+
updates.push("urlsQueued = ?");
|
|
186
|
+
params.push(stats.urlsQueued);
|
|
187
|
+
}
|
|
188
|
+
if (stats.urlsFailed !== undefined) {
|
|
189
|
+
updates.push("urlsFailed = ?");
|
|
190
|
+
params.push(stats.urlsFailed);
|
|
191
|
+
}
|
|
192
|
+
params.push(sessionId);
|
|
193
|
+
this.db.run(`UPDATE sessions SET ${updates.join(", ")} WHERE sessionId = ?`, ...params);
|
|
194
|
+
}
|
|
195
|
+
async addToQueue(sessionId, url, options = {}) {
|
|
196
|
+
if (this.closed || !this.db)
|
|
197
|
+
throw new Error("NavigationHistory is closed");
|
|
198
|
+
const urlKey = this.getUrlKey(url);
|
|
199
|
+
const existing = this.db.get("SELECT id FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
200
|
+
if (existing)
|
|
201
|
+
return false;
|
|
202
|
+
const isVisited = this.db.get("SELECT id FROM visited WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
203
|
+
if (isVisited)
|
|
204
|
+
return false;
|
|
205
|
+
this.db.run(`INSERT INTO queue (sessionId, urlKey, originalUrl, method, priority, body, headers, metadata, addedAt)
|
|
206
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, urlKey, url, options.method || "GET", options.priority || 0, options.body ? JSON.stringify(options.body) : null, options.headers ? JSON.stringify(options.headers) : null, options.metadata ? JSON.stringify(options.metadata) : null, Date.now());
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
async getNextFromQueue(sessionId) {
|
|
210
|
+
if (this.closed || !this.db)
|
|
211
|
+
throw new Error("NavigationHistory is closed");
|
|
212
|
+
const item = this.db.get("SELECT originalUrl as url, method, priority, body, headers, metadata, addedAt FROM queue WHERE sessionId = ? ORDER BY priority DESC, addedAt ASC LIMIT 1", sessionId);
|
|
213
|
+
return item;
|
|
214
|
+
}
|
|
215
|
+
async removeFromQueue(sessionId, url) {
|
|
216
|
+
if (this.closed || !this.db)
|
|
217
|
+
throw new Error("NavigationHistory is closed");
|
|
218
|
+
const urlKey = this.getUrlKey(url);
|
|
219
|
+
this.db.run("DELETE FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
220
|
+
return true;
|
|
221
|
+
}
|
|
222
|
+
async getQueueSize(sessionId) {
|
|
223
|
+
if (this.closed || !this.db)
|
|
224
|
+
throw new Error("NavigationHistory is closed");
|
|
225
|
+
const result = this.db.get("SELECT COUNT(*) as count FROM queue WHERE sessionId = ?", sessionId);
|
|
226
|
+
return result?.count || 0;
|
|
227
|
+
}
|
|
228
|
+
async markVisited(sessionId, url, result = {}) {
|
|
229
|
+
if (this.closed || !this.db)
|
|
230
|
+
throw new Error("NavigationHistory is closed");
|
|
231
|
+
const urlKey = this.getUrlKey(url);
|
|
232
|
+
this.db.run("DELETE FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
233
|
+
this.db.run(`INSERT OR REPLACE INTO visited (sessionId, urlKey, originalUrl, status, visitedAt, finalUrl, contentType, errorMessage)
|
|
234
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, urlKey, url, result.status || 0, Date.now(), result.finalUrl ?? null, result.contentType ?? null, result.errorMessage ?? null);
|
|
235
|
+
}
|
|
236
|
+
async isVisited(sessionId, url) {
|
|
237
|
+
if (this.closed || !this.db)
|
|
238
|
+
throw new Error("NavigationHistory is closed");
|
|
239
|
+
const urlKey = this.getUrlKey(url);
|
|
240
|
+
const result = this.db.get("SELECT id FROM visited WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
241
|
+
return !!result;
|
|
242
|
+
}
|
|
243
|
+
async getVisitedCount(sessionId) {
|
|
244
|
+
if (this.closed || !this.db)
|
|
245
|
+
throw new Error("NavigationHistory is closed");
|
|
246
|
+
const result = this.db.get("SELECT COUNT(*) as count FROM visited WHERE sessionId = ?", sessionId);
|
|
247
|
+
return result?.count || 0;
|
|
248
|
+
}
|
|
249
|
+
async getFailedUrls(sessionId) {
|
|
250
|
+
if (this.closed || !this.db)
|
|
251
|
+
throw new Error("NavigationHistory is closed");
|
|
252
|
+
return this.db.all("SELECT url, status, visitedAt, finalUrl, contentType, errorMessage FROM visited WHERE sessionId = ? AND (status >= 400 OR errorMessage IS NOT NULL)", sessionId);
|
|
253
|
+
}
|
|
254
|
+
async getAllQueuedUrls(sessionId) {
|
|
255
|
+
if (this.closed || !this.db)
|
|
256
|
+
throw new Error("NavigationHistory is closed");
|
|
257
|
+
return this.db.all("SELECT originalUrl as url, method, priority, body, headers, metadata, addedAt FROM queue WHERE sessionId = ? ORDER BY priority DESC, addedAt ASC", sessionId);
|
|
258
|
+
}
|
|
259
|
+
async clearQueue(sessionId) {
|
|
260
|
+
if (this.closed || !this.db)
|
|
261
|
+
throw new Error("NavigationHistory is closed");
|
|
262
|
+
this.db.run("DELETE FROM queue WHERE sessionId = ?", sessionId);
|
|
263
|
+
}
|
|
264
|
+
async clearVisited(sessionId) {
|
|
265
|
+
if (this.closed || !this.db)
|
|
266
|
+
throw new Error("NavigationHistory is closed");
|
|
267
|
+
this.db.run("DELETE FROM visited WHERE sessionId = ?", sessionId);
|
|
268
|
+
}
|
|
269
|
+
async deleteSession(sessionId) {
|
|
270
|
+
if (this.closed || !this.db)
|
|
271
|
+
throw new Error("NavigationHistory is closed");
|
|
272
|
+
this.db.run("DELETE FROM queue WHERE sessionId = ?", sessionId);
|
|
273
|
+
this.db.run("DELETE FROM visited WHERE sessionId = ?", sessionId);
|
|
274
|
+
this.db.run("DELETE FROM sessions WHERE sessionId = ?", sessionId);
|
|
275
|
+
}
|
|
276
|
+
async getResumableSessions() {
|
|
277
|
+
if (this.closed || !this.db)
|
|
278
|
+
throw new Error("NavigationHistory is closed");
|
|
279
|
+
return this.db.all("SELECT * FROM sessions WHERE status IN ('running', 'paused') ORDER BY lastActivityAt DESC");
|
|
280
|
+
}
|
|
281
|
+
async close() {
|
|
282
|
+
if (this.closed)
|
|
283
|
+
return;
|
|
284
|
+
this.closed = true;
|
|
285
|
+
if (this.db) {
|
|
286
|
+
this.db.close();
|
|
287
|
+
this.db = null;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
get isClosed() {
|
|
291
|
+
return this.closed;
|
|
292
|
+
}
|
|
293
|
+
get databasePath() {
|
|
294
|
+
return this.dbPath;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
exports.NavigationHistory = NavigationHistory;
|