rezo 1.0.41 → 1.0.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist/adapters/curl.cjs +143 -32
  2. package/dist/adapters/curl.js +143 -32
  3. package/dist/adapters/entries/curl.d.ts +65 -0
  4. package/dist/adapters/entries/fetch.d.ts +65 -0
  5. package/dist/adapters/entries/http.d.ts +65 -0
  6. package/dist/adapters/entries/http2.d.ts +65 -0
  7. package/dist/adapters/entries/react-native.d.ts +65 -0
  8. package/dist/adapters/entries/xhr.d.ts +65 -0
  9. package/dist/adapters/fetch.cjs +98 -12
  10. package/dist/adapters/fetch.js +98 -12
  11. package/dist/adapters/http.cjs +26 -14
  12. package/dist/adapters/http.js +26 -14
  13. package/dist/adapters/http2.cjs +756 -227
  14. package/dist/adapters/http2.js +756 -227
  15. package/dist/adapters/index.cjs +6 -6
  16. package/dist/adapters/xhr.cjs +94 -2
  17. package/dist/adapters/xhr.js +94 -2
  18. package/dist/cache/dns-cache.cjs +5 -3
  19. package/dist/cache/dns-cache.js +5 -3
  20. package/dist/cache/file-cacher.cjs +7 -1
  21. package/dist/cache/file-cacher.js +7 -1
  22. package/dist/cache/index.cjs +15 -13
  23. package/dist/cache/index.js +1 -0
  24. package/dist/cache/navigation-history.cjs +298 -0
  25. package/dist/cache/navigation-history.js +296 -0
  26. package/dist/cache/url-store.cjs +7 -1
  27. package/dist/cache/url-store.js +7 -1
  28. package/dist/core/rezo.cjs +7 -0
  29. package/dist/core/rezo.js +7 -0
  30. package/dist/crawler.d.ts +196 -11
  31. package/dist/entries/crawler.cjs +5 -5
  32. package/dist/index.cjs +27 -24
  33. package/dist/index.d.ts +73 -0
  34. package/dist/index.js +1 -0
  35. package/dist/internal/agents/base.cjs +113 -0
  36. package/dist/internal/agents/base.js +110 -0
  37. package/dist/internal/agents/http-proxy.cjs +89 -0
  38. package/dist/internal/agents/http-proxy.js +86 -0
  39. package/dist/internal/agents/https-proxy.cjs +176 -0
  40. package/dist/internal/agents/https-proxy.js +173 -0
  41. package/dist/internal/agents/index.cjs +10 -0
  42. package/dist/internal/agents/index.js +5 -0
  43. package/dist/internal/agents/socks-client.cjs +571 -0
  44. package/dist/internal/agents/socks-client.js +567 -0
  45. package/dist/internal/agents/socks-proxy.cjs +75 -0
  46. package/dist/internal/agents/socks-proxy.js +72 -0
  47. package/dist/platform/browser.d.ts +65 -0
  48. package/dist/platform/bun.d.ts +65 -0
  49. package/dist/platform/deno.d.ts +65 -0
  50. package/dist/platform/node.d.ts +65 -0
  51. package/dist/platform/react-native.d.ts +65 -0
  52. package/dist/platform/worker.d.ts +65 -0
  53. package/dist/plugin/crawler-options.cjs +1 -1
  54. package/dist/plugin/crawler-options.js +1 -1
  55. package/dist/plugin/crawler.cjs +192 -1
  56. package/dist/plugin/crawler.js +192 -1
  57. package/dist/plugin/index.cjs +36 -36
  58. package/dist/proxy/index.cjs +18 -16
  59. package/dist/proxy/index.js +17 -12
  60. package/dist/queue/index.cjs +8 -8
  61. package/dist/responses/buildError.cjs +11 -2
  62. package/dist/responses/buildError.js +11 -2
  63. package/dist/responses/universal/index.cjs +11 -11
  64. package/dist/utils/agent-pool.cjs +1 -17
  65. package/dist/utils/agent-pool.js +1 -17
  66. package/dist/utils/curl.cjs +317 -0
  67. package/dist/utils/curl.js +314 -0
  68. package/package.json +1 -1
@@ -1,6 +1,6 @@
1
- const _mod_ds8hb1 = require('./picker.cjs');
2
- exports.detectRuntime = _mod_ds8hb1.detectRuntime;
3
- exports.getAdapterCapabilities = _mod_ds8hb1.getAdapterCapabilities;
4
- exports.buildAdapterContext = _mod_ds8hb1.buildAdapterContext;
5
- exports.getAvailableAdapters = _mod_ds8hb1.getAvailableAdapters;
6
- exports.selectAdapter = _mod_ds8hb1.selectAdapter;;
1
+ const _mod_63iyz4 = require('./picker.cjs');
2
+ exports.detectRuntime = _mod_63iyz4.detectRuntime;
3
+ exports.getAdapterCapabilities = _mod_63iyz4.getAdapterCapabilities;
4
+ exports.buildAdapterContext = _mod_63iyz4.buildAdapterContext;
5
+ exports.getAvailableAdapters = _mod_63iyz4.getAvailableAdapters;
6
+ exports.selectAdapter = _mod_63iyz4.selectAdapter;;
@@ -16,6 +16,85 @@ const Environment = {
16
16
  hasFormData: typeof FormData !== "undefined",
17
17
  hasBlob: typeof Blob !== "undefined"
18
18
  };
19
+ const debugLog = {
20
+ requestStart: (config, url, method) => {
21
+ if (config.debug) {
22
+ console.log(`
23
+ [Rezo Debug] ─────────────────────────────────────`);
24
+ console.log(`[Rezo Debug] ${method} ${url}`);
25
+ console.log(`[Rezo Debug] Request ID: ${config.requestId}`);
26
+ console.log(`[Rezo Debug] Adapter: xhr`);
27
+ if (config.originalRequest?.headers) {
28
+ const headers = config.originalRequest.headers instanceof RezoHeaders ? config.originalRequest.headers.toObject() : config.originalRequest.headers;
29
+ console.log(`[Rezo Debug] Request Headers:`, JSON.stringify(headers, null, 2));
30
+ }
31
+ }
32
+ if (config.trackUrl) {
33
+ console.log(`[Rezo Track] → ${method} ${url}`);
34
+ }
35
+ },
36
+ retry: (config, attempt, maxRetries, statusCode, delay) => {
37
+ if (config.debug) {
38
+ console.log(`[Rezo Debug] Retry ${attempt}/${maxRetries} after status ${statusCode}${delay > 0 ? ` (waiting ${delay}ms)` : ""}`);
39
+ }
40
+ if (config.trackUrl) {
41
+ console.log(`[Rezo Track] ⟳ Retry ${attempt}/${maxRetries} (status ${statusCode})`);
42
+ }
43
+ },
44
+ maxRetries: (config, maxRetries) => {
45
+ if (config.debug) {
46
+ console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
47
+ }
48
+ if (config.trackUrl) {
49
+ console.log(`[Rezo Track] ✗ Max retries reached`);
50
+ }
51
+ },
52
+ response: (config, status, statusText, duration) => {
53
+ if (config.debug) {
54
+ console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
55
+ }
56
+ if (config.trackUrl) {
57
+ console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
58
+ }
59
+ },
60
+ responseHeaders: (config, headers) => {
61
+ if (config.debug) {
62
+ console.log(`[Rezo Debug] Response Headers:`, JSON.stringify(headers, null, 2));
63
+ }
64
+ },
65
+ cookies: (config, cookieCount) => {
66
+ if (config.debug && cookieCount > 0) {
67
+ console.log(`[Rezo Debug] Cookies received: ${cookieCount}`);
68
+ }
69
+ },
70
+ timing: (config, timing) => {
71
+ if (config.debug) {
72
+ const parts = [];
73
+ if (timing.ttfb)
74
+ parts.push(`TTFB: ${timing.ttfb.toFixed(2)}ms`);
75
+ if (timing.total)
76
+ parts.push(`Total: ${timing.total.toFixed(2)}ms`);
77
+ if (parts.length > 0) {
78
+ console.log(`[Rezo Debug] Timing: ${parts.join(" | ")}`);
79
+ }
80
+ }
81
+ },
82
+ complete: (config, url) => {
83
+ if (config.debug) {
84
+ console.log(`[Rezo Debug] Request complete: ${url}`);
85
+ console.log(`[Rezo Debug] ─────────────────────────────────────
86
+ `);
87
+ }
88
+ },
89
+ error: (config, error) => {
90
+ if (config.debug) {
91
+ console.log(`[Rezo Debug] Error: ${error instanceof Error ? error.message : error}`);
92
+ }
93
+ if (config.trackUrl) {
94
+ console.log(`[Rezo Track] ✗ Error: ${error instanceof Error ? error.message : error}`);
95
+ }
96
+ }
97
+ };
19
98
  function updateTiming(config, timing, bodySize) {
20
99
  const now = performance.now();
21
100
  config.timing.domainLookupStart = config.timing.startTime;
@@ -330,11 +409,14 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
330
409
  throw response;
331
410
  }
332
411
  if (maxRetries <= retries) {
412
+ debugLog.maxRetries(config, maxRetries);
333
413
  throw response;
334
414
  }
335
415
  retries++;
336
- if (retryDelay > 0) {
337
- await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
416
+ const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
417
+ debugLog.retry(config, retries, maxRetries, response.status || 0, currentDelay);
418
+ if (currentDelay > 0) {
419
+ await new Promise((resolve) => setTimeout(resolve, currentDelay));
338
420
  }
339
421
  }
340
422
  config.retryAttempts++;
@@ -359,6 +441,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
359
441
  config.isSecure = isSecure;
360
442
  config.finalUrl = url;
361
443
  config.network.protocol = isSecure ? "https" : "http";
444
+ debugLog.requestStart(config, url, fetchOptions.method?.toUpperCase() || "GET");
362
445
  const xhr = new XMLHttpRequest;
363
446
  xhr.open(fetchOptions.method.toUpperCase(), url, true);
364
447
  const headers = toXHRHeaders(fetchOptions.headers);
@@ -513,6 +596,15 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
513
596
  resolve(error);
514
597
  return;
515
598
  }
599
+ const duration = performance.now() - timing.startTime;
600
+ debugLog.response(config, status, statusText, duration);
601
+ debugLog.responseHeaders(config, responseHeaders.toObject());
602
+ debugLog.cookies(config, cookies.array.length);
603
+ debugLog.timing(config, {
604
+ ttfb: config.timing.responseStart - config.timing.startTime,
605
+ total: duration
606
+ });
607
+ debugLog.complete(config, xhr.responseURL || url);
516
608
  const finalResponse = {
517
609
  data: responseData,
518
610
  status,
@@ -16,6 +16,85 @@ const Environment = {
16
16
  hasFormData: typeof FormData !== "undefined",
17
17
  hasBlob: typeof Blob !== "undefined"
18
18
  };
19
+ const debugLog = {
20
+ requestStart: (config, url, method) => {
21
+ if (config.debug) {
22
+ console.log(`
23
+ [Rezo Debug] ─────────────────────────────────────`);
24
+ console.log(`[Rezo Debug] ${method} ${url}`);
25
+ console.log(`[Rezo Debug] Request ID: ${config.requestId}`);
26
+ console.log(`[Rezo Debug] Adapter: xhr`);
27
+ if (config.originalRequest?.headers) {
28
+ const headers = config.originalRequest.headers instanceof RezoHeaders ? config.originalRequest.headers.toObject() : config.originalRequest.headers;
29
+ console.log(`[Rezo Debug] Request Headers:`, JSON.stringify(headers, null, 2));
30
+ }
31
+ }
32
+ if (config.trackUrl) {
33
+ console.log(`[Rezo Track] → ${method} ${url}`);
34
+ }
35
+ },
36
+ retry: (config, attempt, maxRetries, statusCode, delay) => {
37
+ if (config.debug) {
38
+ console.log(`[Rezo Debug] Retry ${attempt}/${maxRetries} after status ${statusCode}${delay > 0 ? ` (waiting ${delay}ms)` : ""}`);
39
+ }
40
+ if (config.trackUrl) {
41
+ console.log(`[Rezo Track] ⟳ Retry ${attempt}/${maxRetries} (status ${statusCode})`);
42
+ }
43
+ },
44
+ maxRetries: (config, maxRetries) => {
45
+ if (config.debug) {
46
+ console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
47
+ }
48
+ if (config.trackUrl) {
49
+ console.log(`[Rezo Track] ✗ Max retries reached`);
50
+ }
51
+ },
52
+ response: (config, status, statusText, duration) => {
53
+ if (config.debug) {
54
+ console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
55
+ }
56
+ if (config.trackUrl) {
57
+ console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
58
+ }
59
+ },
60
+ responseHeaders: (config, headers) => {
61
+ if (config.debug) {
62
+ console.log(`[Rezo Debug] Response Headers:`, JSON.stringify(headers, null, 2));
63
+ }
64
+ },
65
+ cookies: (config, cookieCount) => {
66
+ if (config.debug && cookieCount > 0) {
67
+ console.log(`[Rezo Debug] Cookies received: ${cookieCount}`);
68
+ }
69
+ },
70
+ timing: (config, timing) => {
71
+ if (config.debug) {
72
+ const parts = [];
73
+ if (timing.ttfb)
74
+ parts.push(`TTFB: ${timing.ttfb.toFixed(2)}ms`);
75
+ if (timing.total)
76
+ parts.push(`Total: ${timing.total.toFixed(2)}ms`);
77
+ if (parts.length > 0) {
78
+ console.log(`[Rezo Debug] Timing: ${parts.join(" | ")}`);
79
+ }
80
+ }
81
+ },
82
+ complete: (config, url) => {
83
+ if (config.debug) {
84
+ console.log(`[Rezo Debug] Request complete: ${url}`);
85
+ console.log(`[Rezo Debug] ─────────────────────────────────────
86
+ `);
87
+ }
88
+ },
89
+ error: (config, error) => {
90
+ if (config.debug) {
91
+ console.log(`[Rezo Debug] Error: ${error instanceof Error ? error.message : error}`);
92
+ }
93
+ if (config.trackUrl) {
94
+ console.log(`[Rezo Track] ✗ Error: ${error instanceof Error ? error.message : error}`);
95
+ }
96
+ }
97
+ };
19
98
  function updateTiming(config, timing, bodySize) {
20
99
  const now = performance.now();
21
100
  config.timing.domainLookupStart = config.timing.startTime;
@@ -330,11 +409,14 @@ async function executeXHRRequest(fetchOptions, config, options, perform, streamR
330
409
  throw response;
331
410
  }
332
411
  if (maxRetries <= retries) {
412
+ debugLog.maxRetries(config, maxRetries);
333
413
  throw response;
334
414
  }
335
415
  retries++;
336
- if (retryDelay > 0) {
337
- await new Promise((resolve) => setTimeout(resolve, incrementDelay ? retryDelay * retries : retryDelay));
416
+ const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
417
+ debugLog.retry(config, retries, maxRetries, response.status || 0, currentDelay);
418
+ if (currentDelay > 0) {
419
+ await new Promise((resolve) => setTimeout(resolve, currentDelay));
338
420
  }
339
421
  }
340
422
  config.retryAttempts++;
@@ -359,6 +441,7 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
359
441
  config.isSecure = isSecure;
360
442
  config.finalUrl = url;
361
443
  config.network.protocol = isSecure ? "https" : "http";
444
+ debugLog.requestStart(config, url, fetchOptions.method?.toUpperCase() || "GET");
362
445
  const xhr = new XMLHttpRequest;
363
446
  xhr.open(fetchOptions.method.toUpperCase(), url, true);
364
447
  const headers = toXHRHeaders(fetchOptions.headers);
@@ -513,6 +596,15 @@ function executeSingleXHRRequest(config, fetchOptions, timing, streamResult, dow
513
596
  resolve(error);
514
597
  return;
515
598
  }
599
+ const duration = performance.now() - timing.startTime;
600
+ debugLog.response(config, status, statusText, duration);
601
+ debugLog.responseHeaders(config, responseHeaders.toObject());
602
+ debugLog.cookies(config, cookies.array.length);
603
+ debugLog.timing(config, {
604
+ ttfb: config.timing.responseStart - config.timing.startTime,
605
+ total: duration
606
+ });
607
+ debugLog.complete(config, xhr.responseURL || url);
516
608
  const finalResponse = {
517
609
  data: responseData,
518
610
  status,
@@ -1,5 +1,5 @@
1
1
  const { LRUCache } = require('./lru-cache.cjs');
2
- const dns = require("dns");
2
+ const dns = require("node:dns");
3
3
  const DEFAULT_DNS_TTL = 60000;
4
4
  const DEFAULT_DNS_MAX_ENTRIES = 1000;
5
5
 
@@ -24,10 +24,12 @@ class DNSCache {
24
24
  const cached = this.cache.get(key);
25
25
  if (cached && cached.addresses.length > 0) {
26
26
  const address = cached.addresses[Math.floor(Math.random() * cached.addresses.length)];
27
- return { address, family: cached.family };
27
+ if (address && typeof address === "string") {
28
+ return { address, family: cached.family };
29
+ }
28
30
  }
29
31
  const result = await this.resolveDNS(hostname, family);
30
- if (result) {
32
+ if (result && result.address && typeof result.address === "string") {
31
33
  this.cache.set(key, {
32
34
  addresses: [result.address],
33
35
  family: result.family,
@@ -1,5 +1,5 @@
1
1
  import { LRUCache } from './lru-cache.js';
2
- import dns from "dns";
2
+ import dns from "node:dns";
3
3
  const DEFAULT_DNS_TTL = 60000;
4
4
  const DEFAULT_DNS_MAX_ENTRIES = 1000;
5
5
 
@@ -24,10 +24,12 @@ export class DNSCache {
24
24
  const cached = this.cache.get(key);
25
25
  if (cached && cached.addresses.length > 0) {
26
26
  const address = cached.addresses[Math.floor(Math.random() * cached.addresses.length)];
27
- return { address, family: cached.family };
27
+ if (address && typeof address === "string") {
28
+ return { address, family: cached.family };
29
+ }
28
30
  }
29
31
  const result = await this.resolveDNS(hostname, family);
30
- if (result) {
32
+ if (result && result.address && typeof result.address === "string") {
31
33
  this.cache.set(key, {
32
34
  addresses: [result.address],
33
35
  family: result.family,
@@ -45,7 +45,13 @@ async function createDatabase(dbPath) {
45
45
  const { DatabaseSync } = await import("node:sqlite");
46
46
  const db = new DatabaseSync(dbPath);
47
47
  return {
48
- run: (sql, ...params) => db.exec(sql),
48
+ run: (sql, ...params) => {
49
+ if (params.length === 0) {
50
+ db.exec(sql);
51
+ } else {
52
+ db.prepare(sql).run(...params);
53
+ }
54
+ },
49
55
  get: (sql, ...params) => {
50
56
  const stmt = db.prepare(sql);
51
57
  return stmt.get(...params);
@@ -45,7 +45,13 @@ async function createDatabase(dbPath) {
45
45
  const { DatabaseSync } = await import("node:sqlite");
46
46
  const db = new DatabaseSync(dbPath);
47
47
  return {
48
- run: (sql, ...params) => db.exec(sql),
48
+ run: (sql, ...params) => {
49
+ if (params.length === 0) {
50
+ db.exec(sql);
51
+ } else {
52
+ db.prepare(sql).run(...params);
53
+ }
54
+ },
49
55
  get: (sql, ...params) => {
50
56
  const stmt = db.prepare(sql);
51
57
  return stmt.get(...params);
@@ -1,13 +1,15 @@
1
- const _mod_pf5i2i = require('./lru-cache.cjs');
2
- exports.LRUCache = _mod_pf5i2i.LRUCache;;
3
- const _mod_iftoun = require('./dns-cache.cjs');
4
- exports.DNSCache = _mod_iftoun.DNSCache;
5
- exports.getGlobalDNSCache = _mod_iftoun.getGlobalDNSCache;
6
- exports.resetGlobalDNSCache = _mod_iftoun.resetGlobalDNSCache;;
7
- const _mod_lg2ryg = require('./response-cache.cjs');
8
- exports.ResponseCache = _mod_lg2ryg.ResponseCache;
9
- exports.normalizeResponseCacheConfig = _mod_lg2ryg.normalizeResponseCacheConfig;;
10
- const _mod_btk7am = require('./file-cacher.cjs');
11
- exports.FileCacher = _mod_btk7am.FileCacher;;
12
- const _mod_56wgw7 = require('./url-store.cjs');
13
- exports.UrlStore = _mod_56wgw7.UrlStore;;
1
+ const _mod_6dj0p0 = require('./lru-cache.cjs');
2
+ exports.LRUCache = _mod_6dj0p0.LRUCache;;
3
+ const _mod_9z4pm1 = require('./dns-cache.cjs');
4
+ exports.DNSCache = _mod_9z4pm1.DNSCache;
5
+ exports.getGlobalDNSCache = _mod_9z4pm1.getGlobalDNSCache;
6
+ exports.resetGlobalDNSCache = _mod_9z4pm1.resetGlobalDNSCache;;
7
+ const _mod_5ylf2b = require('./response-cache.cjs');
8
+ exports.ResponseCache = _mod_5ylf2b.ResponseCache;
9
+ exports.normalizeResponseCacheConfig = _mod_5ylf2b.normalizeResponseCacheConfig;;
10
+ const _mod_fk65c2 = require('./file-cacher.cjs');
11
+ exports.FileCacher = _mod_fk65c2.FileCacher;;
12
+ const _mod_7tzsb5 = require('./url-store.cjs');
13
+ exports.UrlStore = _mod_7tzsb5.UrlStore;;
14
+ const _mod_04c8wb = require('./navigation-history.cjs');
15
+ exports.NavigationHistory = _mod_04c8wb.NavigationHistory;;
@@ -3,3 +3,4 @@ export { DNSCache, getGlobalDNSCache, resetGlobalDNSCache } from './dns-cache.js
3
3
  export { ResponseCache, normalizeResponseCacheConfig } from './response-cache.js';
4
4
  export { FileCacher } from './file-cacher.js';
5
5
  export { UrlStore } from './url-store.js';
6
+ export { NavigationHistory } from './navigation-history.js';
@@ -0,0 +1,298 @@
1
+ const fs = require("node:fs");
2
+ const path = require("node:path");
3
+ const { createHash } = require("node:crypto");
4
+ function detectRuntime() {
5
+ if (typeof globalThis.Bun !== "undefined")
6
+ return "bun";
7
+ if (typeof globalThis.Deno !== "undefined")
8
+ return "deno";
9
+ return "node";
10
+ }
11
+ async function createDatabase(dbPath) {
12
+ const runtime = detectRuntime();
13
+ if (runtime === "bun") {
14
+ const { Database } = await import("bun:sqlite");
15
+ const db = new Database(dbPath);
16
+ return {
17
+ run: (sql, ...params) => db.run(sql, ...params),
18
+ get: (sql, ...params) => db.query(sql).get(...params),
19
+ all: (sql, ...params) => db.query(sql).all(...params),
20
+ close: () => db.close()
21
+ };
22
+ }
23
+ if (runtime === "deno") {
24
+ try {
25
+ const { Database } = await import("node:sqlite");
26
+ const db = new Database(dbPath);
27
+ return {
28
+ run: (sql, ...params) => db.exec(sql, params),
29
+ get: (sql, ...params) => {
30
+ const stmt = db.prepare(sql);
31
+ return stmt.get(...params);
32
+ },
33
+ all: (sql, ...params) => {
34
+ const stmt = db.prepare(sql);
35
+ return stmt.all(...params);
36
+ },
37
+ close: () => db.close()
38
+ };
39
+ } catch {
40
+ throw new Error("Deno SQLite support requires Node.js compatibility mode");
41
+ }
42
+ }
43
+ const { DatabaseSync } = await import("node:sqlite");
44
+ const db = new DatabaseSync(dbPath);
45
+ return {
46
+ run: (sql, ...params) => {
47
+ const stmt = db.prepare(sql);
48
+ stmt.run(...params);
49
+ },
50
+ get: (sql, ...params) => {
51
+ const stmt = db.prepare(sql);
52
+ return stmt.get(...params);
53
+ },
54
+ all: (sql, ...params) => {
55
+ const stmt = db.prepare(sql);
56
+ return stmt.all(...params);
57
+ },
58
+ close: () => db.close()
59
+ };
60
+ }
61
+
62
+ class NavigationHistory {
63
+ db = null;
64
+ options;
65
+ storeDir;
66
+ dbPath;
67
+ closed = false;
68
+ initPromise = null;
69
+ constructor(options = {}) {
70
+ this.options = {
71
+ storeDir: options.storeDir || "./navigation-history",
72
+ dbFileName: options.dbFileName || "navigation.db",
73
+ hashUrls: options.hashUrls ?? false
74
+ };
75
+ this.storeDir = path.resolve(this.options.storeDir);
76
+ this.dbPath = path.join(this.storeDir, this.options.dbFileName);
77
+ if (!fs.existsSync(this.storeDir)) {
78
+ fs.mkdirSync(this.storeDir, { recursive: true });
79
+ }
80
+ }
81
+ static async create(options = {}) {
82
+ const store = new NavigationHistory(options);
83
+ await store.initialize();
84
+ return store;
85
+ }
86
+ async initialize() {
87
+ if (this.initPromise)
88
+ return this.initPromise;
89
+ this.initPromise = (async () => {
90
+ this.db = await createDatabase(this.dbPath);
91
+ this.db.run(`
92
+ CREATE TABLE IF NOT EXISTS sessions (
93
+ sessionId TEXT PRIMARY KEY,
94
+ baseUrl TEXT NOT NULL,
95
+ startedAt INTEGER NOT NULL,
96
+ lastActivityAt INTEGER NOT NULL,
97
+ status TEXT DEFAULT 'running',
98
+ urlsVisited INTEGER DEFAULT 0,
99
+ urlsQueued INTEGER DEFAULT 0,
100
+ urlsFailed INTEGER DEFAULT 0,
101
+ metadata TEXT
102
+ )
103
+ `);
104
+ this.db.run(`
105
+ CREATE TABLE IF NOT EXISTS queue (
106
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
107
+ sessionId TEXT NOT NULL,
108
+ urlKey TEXT NOT NULL,
109
+ originalUrl TEXT NOT NULL,
110
+ method TEXT DEFAULT 'GET',
111
+ priority INTEGER DEFAULT 0,
112
+ body TEXT,
113
+ headers TEXT,
114
+ metadata TEXT,
115
+ addedAt INTEGER NOT NULL,
116
+ UNIQUE(sessionId, urlKey)
117
+ )
118
+ `);
119
+ this.db.run(`
120
+ CREATE TABLE IF NOT EXISTS visited (
121
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
122
+ sessionId TEXT NOT NULL,
123
+ urlKey TEXT NOT NULL,
124
+ originalUrl TEXT NOT NULL,
125
+ status INTEGER,
126
+ visitedAt INTEGER NOT NULL,
127
+ finalUrl TEXT,
128
+ contentType TEXT,
129
+ errorMessage TEXT,
130
+ UNIQUE(sessionId, urlKey)
131
+ )
132
+ `);
133
+ this.db.run("CREATE INDEX IF NOT EXISTS idx_queue_session ON queue(sessionId)");
134
+ this.db.run("CREATE INDEX IF NOT EXISTS idx_queue_priority ON queue(sessionId, priority DESC)");
135
+ this.db.run("CREATE INDEX IF NOT EXISTS idx_visited_session ON visited(sessionId)");
136
+ this.db.run("CREATE INDEX IF NOT EXISTS idx_sessions_status ON sessions(status)");
137
+ })();
138
+ return this.initPromise;
139
+ }
140
+ getUrlKey(url) {
141
+ if (this.options.hashUrls) {
142
+ return createHash("sha256").update(url).digest("hex");
143
+ }
144
+ return url;
145
+ }
146
+ async createSession(sessionId, baseUrl, metadata) {
147
+ if (this.closed || !this.db)
148
+ throw new Error("NavigationHistory is closed");
149
+ const now = Date.now();
150
+ const session = {
151
+ sessionId,
152
+ baseUrl,
153
+ startedAt: now,
154
+ lastActivityAt: now,
155
+ status: "running",
156
+ urlsVisited: 0,
157
+ urlsQueued: 0,
158
+ urlsFailed: 0,
159
+ metadata: metadata ? JSON.stringify(metadata) : undefined
160
+ };
161
+ this.db.run(`INSERT OR REPLACE INTO sessions (sessionId, baseUrl, startedAt, lastActivityAt, status, urlsVisited, urlsQueued, urlsFailed, metadata)
162
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, baseUrl, now, now, "running", 0, 0, 0, session.metadata ?? null);
163
+ return session;
164
+ }
165
+ async getSession(sessionId) {
166
+ if (this.closed || !this.db)
167
+ throw new Error("NavigationHistory is closed");
168
+ return this.db.get("SELECT * FROM sessions WHERE sessionId = ?", sessionId);
169
+ }
170
+ async updateSessionStatus(sessionId, status) {
171
+ if (this.closed || !this.db)
172
+ throw new Error("NavigationHistory is closed");
173
+ this.db.run("UPDATE sessions SET status = ?, lastActivityAt = ? WHERE sessionId = ?", status, Date.now(), sessionId);
174
+ }
175
+ async updateSessionStats(sessionId, stats) {
176
+ if (this.closed || !this.db)
177
+ throw new Error("NavigationHistory is closed");
178
+ const updates = ["lastActivityAt = ?"];
179
+ const params = [Date.now()];
180
+ if (stats.urlsVisited !== undefined) {
181
+ updates.push("urlsVisited = ?");
182
+ params.push(stats.urlsVisited);
183
+ }
184
+ if (stats.urlsQueued !== undefined) {
185
+ updates.push("urlsQueued = ?");
186
+ params.push(stats.urlsQueued);
187
+ }
188
+ if (stats.urlsFailed !== undefined) {
189
+ updates.push("urlsFailed = ?");
190
+ params.push(stats.urlsFailed);
191
+ }
192
+ params.push(sessionId);
193
+ this.db.run(`UPDATE sessions SET ${updates.join(", ")} WHERE sessionId = ?`, ...params);
194
+ }
195
+ async addToQueue(sessionId, url, options = {}) {
196
+ if (this.closed || !this.db)
197
+ throw new Error("NavigationHistory is closed");
198
+ const urlKey = this.getUrlKey(url);
199
+ const existing = this.db.get("SELECT id FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
200
+ if (existing)
201
+ return false;
202
+ const isVisited = this.db.get("SELECT id FROM visited WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
203
+ if (isVisited)
204
+ return false;
205
+ this.db.run(`INSERT INTO queue (sessionId, urlKey, originalUrl, method, priority, body, headers, metadata, addedAt)
206
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, urlKey, url, options.method || "GET", options.priority || 0, options.body ? JSON.stringify(options.body) : null, options.headers ? JSON.stringify(options.headers) : null, options.metadata ? JSON.stringify(options.metadata) : null, Date.now());
207
+ return true;
208
+ }
209
+ async getNextFromQueue(sessionId) {
210
+ if (this.closed || !this.db)
211
+ throw new Error("NavigationHistory is closed");
212
+ const item = this.db.get("SELECT originalUrl as url, method, priority, body, headers, metadata, addedAt FROM queue WHERE sessionId = ? ORDER BY priority DESC, addedAt ASC LIMIT 1", sessionId);
213
+ return item;
214
+ }
215
+ async removeFromQueue(sessionId, url) {
216
+ if (this.closed || !this.db)
217
+ throw new Error("NavigationHistory is closed");
218
+ const urlKey = this.getUrlKey(url);
219
+ this.db.run("DELETE FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
220
+ return true;
221
+ }
222
+ async getQueueSize(sessionId) {
223
+ if (this.closed || !this.db)
224
+ throw new Error("NavigationHistory is closed");
225
+ const result = this.db.get("SELECT COUNT(*) as count FROM queue WHERE sessionId = ?", sessionId);
226
+ return result?.count || 0;
227
+ }
228
+ async markVisited(sessionId, url, result = {}) {
229
+ if (this.closed || !this.db)
230
+ throw new Error("NavigationHistory is closed");
231
+ const urlKey = this.getUrlKey(url);
232
+ this.db.run("DELETE FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
233
+ this.db.run(`INSERT OR REPLACE INTO visited (sessionId, urlKey, originalUrl, status, visitedAt, finalUrl, contentType, errorMessage)
234
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, urlKey, url, result.status || 0, Date.now(), result.finalUrl ?? null, result.contentType ?? null, result.errorMessage ?? null);
235
+ }
236
+ async isVisited(sessionId, url) {
237
+ if (this.closed || !this.db)
238
+ throw new Error("NavigationHistory is closed");
239
+ const urlKey = this.getUrlKey(url);
240
+ const result = this.db.get("SELECT id FROM visited WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
241
+ return !!result;
242
+ }
243
+ async getVisitedCount(sessionId) {
244
+ if (this.closed || !this.db)
245
+ throw new Error("NavigationHistory is closed");
246
+ const result = this.db.get("SELECT COUNT(*) as count FROM visited WHERE sessionId = ?", sessionId);
247
+ return result?.count || 0;
248
+ }
249
+ async getFailedUrls(sessionId) {
250
+ if (this.closed || !this.db)
251
+ throw new Error("NavigationHistory is closed");
252
+ return this.db.all("SELECT url, status, visitedAt, finalUrl, contentType, errorMessage FROM visited WHERE sessionId = ? AND (status >= 400 OR errorMessage IS NOT NULL)", sessionId);
253
+ }
254
+ async getAllQueuedUrls(sessionId) {
255
+ if (this.closed || !this.db)
256
+ throw new Error("NavigationHistory is closed");
257
+ return this.db.all("SELECT originalUrl as url, method, priority, body, headers, metadata, addedAt FROM queue WHERE sessionId = ? ORDER BY priority DESC, addedAt ASC", sessionId);
258
+ }
259
+ async clearQueue(sessionId) {
260
+ if (this.closed || !this.db)
261
+ throw new Error("NavigationHistory is closed");
262
+ this.db.run("DELETE FROM queue WHERE sessionId = ?", sessionId);
263
+ }
264
+ async clearVisited(sessionId) {
265
+ if (this.closed || !this.db)
266
+ throw new Error("NavigationHistory is closed");
267
+ this.db.run("DELETE FROM visited WHERE sessionId = ?", sessionId);
268
+ }
269
+ async deleteSession(sessionId) {
270
+ if (this.closed || !this.db)
271
+ throw new Error("NavigationHistory is closed");
272
+ this.db.run("DELETE FROM queue WHERE sessionId = ?", sessionId);
273
+ this.db.run("DELETE FROM visited WHERE sessionId = ?", sessionId);
274
+ this.db.run("DELETE FROM sessions WHERE sessionId = ?", sessionId);
275
+ }
276
+ async getResumableSessions() {
277
+ if (this.closed || !this.db)
278
+ throw new Error("NavigationHistory is closed");
279
+ return this.db.all("SELECT * FROM sessions WHERE status IN ('running', 'paused') ORDER BY lastActivityAt DESC");
280
+ }
281
+ async close() {
282
+ if (this.closed)
283
+ return;
284
+ this.closed = true;
285
+ if (this.db) {
286
+ this.db.close();
287
+ this.db = null;
288
+ }
289
+ }
290
+ get isClosed() {
291
+ return this.closed;
292
+ }
293
+ get databasePath() {
294
+ return this.dbPath;
295
+ }
296
+ }
297
+
298
+ exports.NavigationHistory = NavigationHistory;