rezo 1.0.43 → 1.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +115 -0
- package/dist/adapters/entries/fetch.d.ts +115 -0
- package/dist/adapters/entries/http.d.ts +115 -0
- package/dist/adapters/entries/http2.d.ts +115 -0
- package/dist/adapters/entries/react-native.d.ts +115 -0
- package/dist/adapters/entries/xhr.d.ts +115 -0
- package/dist/adapters/fetch.cjs +18 -0
- package/dist/adapters/fetch.js +18 -0
- package/dist/adapters/http.cjs +18 -0
- package/dist/adapters/http.js +18 -0
- package/dist/adapters/http2.cjs +18 -0
- package/dist/adapters/http2.js +18 -0
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/xhr.cjs +19 -0
- package/dist/adapters/xhr.js +19 -0
- package/dist/cache/index.cjs +9 -15
- package/dist/cache/index.js +0 -3
- package/dist/core/hooks.cjs +4 -2
- package/dist/core/hooks.js +4 -2
- package/dist/crawler/addon/decodo/index.cjs +1 -0
- package/dist/crawler/addon/decodo/index.js +1 -0
- package/dist/crawler/crawler-options.cjs +1 -0
- package/dist/crawler/crawler-options.js +1 -0
- package/dist/{plugin → crawler}/crawler.cjs +392 -32
- package/dist/{plugin → crawler}/crawler.js +392 -32
- package/dist/crawler/index.cjs +40 -0
- package/dist/{plugin → crawler}/index.js +4 -2
- package/dist/crawler/plugin/file-cacher.cjs +19 -0
- package/dist/crawler/plugin/file-cacher.js +19 -0
- package/dist/crawler/plugin/index.cjs +1 -0
- package/dist/crawler/plugin/index.js +1 -0
- package/dist/crawler/plugin/navigation-history.cjs +43 -0
- package/dist/crawler/plugin/navigation-history.js +43 -0
- package/dist/crawler/plugin/robots-txt.cjs +2 -0
- package/dist/crawler/plugin/robots-txt.js +2 -0
- package/dist/crawler/plugin/url-store.cjs +18 -0
- package/dist/crawler/plugin/url-store.js +18 -0
- package/dist/crawler.d.ts +430 -172
- package/dist/entries/crawler.cjs +5 -5
- package/dist/entries/crawler.js +2 -2
- package/dist/index.cjs +27 -27
- package/dist/index.d.ts +115 -0
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/platform/browser.d.ts +115 -0
- package/dist/platform/bun.d.ts +115 -0
- package/dist/platform/deno.d.ts +115 -0
- package/dist/platform/node.d.ts +115 -0
- package/dist/platform/react-native.d.ts +115 -0
- package/dist/platform/worker.d.ts +115 -0
- package/dist/proxy/index.cjs +5 -5
- package/dist/proxy/index.js +1 -1
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/rate-limit-wait.cjs +217 -0
- package/dist/utils/rate-limit-wait.js +208 -0
- package/package.json +2 -6
- package/dist/cache/file-cacher.cjs +0 -270
- package/dist/cache/file-cacher.js +0 -267
- package/dist/cache/navigation-history.cjs +0 -298
- package/dist/cache/navigation-history.js +0 -296
- package/dist/cache/url-store.cjs +0 -294
- package/dist/cache/url-store.js +0 -291
- package/dist/plugin/addon/decodo/index.cjs +0 -1
- package/dist/plugin/addon/decodo/index.js +0 -1
- package/dist/plugin/crawler-options.cjs +0 -1
- package/dist/plugin/crawler-options.js +0 -1
- package/dist/plugin/index.cjs +0 -36
- /package/dist/{plugin → crawler}/addon/decodo/options.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/options.js +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/types.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/types.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/index.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/index.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/options.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/options.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/types.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/types.js +0 -0
- /package/dist/{plugin → crawler}/scraper.cjs +0 -0
- /package/dist/{plugin → crawler}/scraper.js +0 -0
|
@@ -1,267 +0,0 @@
|
|
|
1
|
-
import fs from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { createHash } from "node:crypto";
|
|
4
|
-
function detectRuntime() {
|
|
5
|
-
if (typeof globalThis.Bun !== "undefined") {
|
|
6
|
-
return "bun";
|
|
7
|
-
}
|
|
8
|
-
if (typeof globalThis.Deno !== "undefined") {
|
|
9
|
-
return "deno";
|
|
10
|
-
}
|
|
11
|
-
return "node";
|
|
12
|
-
}
|
|
13
|
-
async function createDatabase(dbPath) {
|
|
14
|
-
const runtime = detectRuntime();
|
|
15
|
-
if (runtime === "bun") {
|
|
16
|
-
const { Database } = await import("bun:sqlite");
|
|
17
|
-
const db = new Database(dbPath);
|
|
18
|
-
return {
|
|
19
|
-
run: (sql, ...params) => db.run(sql, ...params),
|
|
20
|
-
get: (sql, ...params) => db.query(sql).get(...params),
|
|
21
|
-
all: (sql, ...params) => db.query(sql).all(...params),
|
|
22
|
-
close: () => db.close()
|
|
23
|
-
};
|
|
24
|
-
}
|
|
25
|
-
if (runtime === "deno") {
|
|
26
|
-
try {
|
|
27
|
-
const { Database } = await import("node:sqlite");
|
|
28
|
-
const db = new Database(dbPath);
|
|
29
|
-
return {
|
|
30
|
-
run: (sql, ...params) => db.exec(sql, params),
|
|
31
|
-
get: (sql, ...params) => {
|
|
32
|
-
const stmt = db.prepare(sql);
|
|
33
|
-
return stmt.get(...params);
|
|
34
|
-
},
|
|
35
|
-
all: (sql, ...params) => {
|
|
36
|
-
const stmt = db.prepare(sql);
|
|
37
|
-
return stmt.all(...params);
|
|
38
|
-
},
|
|
39
|
-
close: () => db.close()
|
|
40
|
-
};
|
|
41
|
-
} catch {
|
|
42
|
-
throw new Error("Deno SQLite support requires Node.js compatibility mode");
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
const { DatabaseSync } = await import("node:sqlite");
|
|
46
|
-
const db = new DatabaseSync(dbPath);
|
|
47
|
-
return {
|
|
48
|
-
run: (sql, ...params) => {
|
|
49
|
-
if (params.length === 0) {
|
|
50
|
-
db.exec(sql);
|
|
51
|
-
} else {
|
|
52
|
-
db.prepare(sql).run(...params);
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
get: (sql, ...params) => {
|
|
56
|
-
const stmt = db.prepare(sql);
|
|
57
|
-
return stmt.get(...params);
|
|
58
|
-
},
|
|
59
|
-
all: (sql, ...params) => {
|
|
60
|
-
const stmt = db.prepare(sql);
|
|
61
|
-
return stmt.all(...params);
|
|
62
|
-
},
|
|
63
|
-
close: () => db.close()
|
|
64
|
-
};
|
|
65
|
-
}
|
|
66
|
-
async function compressData(data) {
|
|
67
|
-
try {
|
|
68
|
-
const zlib = await import("node:zlib");
|
|
69
|
-
if (typeof zlib.zstdCompressSync === "function") {
|
|
70
|
-
return zlib.zstdCompressSync(data);
|
|
71
|
-
}
|
|
72
|
-
return data;
|
|
73
|
-
} catch {
|
|
74
|
-
return data;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
async function decompressData(data) {
|
|
78
|
-
try {
|
|
79
|
-
const zlib = await import("node:zlib");
|
|
80
|
-
if (typeof zlib.zstdDecompressSync === "function") {
|
|
81
|
-
return zlib.zstdDecompressSync(data);
|
|
82
|
-
}
|
|
83
|
-
return data;
|
|
84
|
-
} catch {
|
|
85
|
-
return data;
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
export class FileCacher {
|
|
90
|
-
databases = new Map;
|
|
91
|
-
options;
|
|
92
|
-
cacheDir;
|
|
93
|
-
closed = false;
|
|
94
|
-
constructor(options = {}) {
|
|
95
|
-
this.options = {
|
|
96
|
-
cacheDir: options.cacheDir || "./cache",
|
|
97
|
-
ttl: options.ttl || 604800000,
|
|
98
|
-
compression: options.compression ?? false,
|
|
99
|
-
softDelete: options.softDelete ?? false,
|
|
100
|
-
encryptNamespace: options.encryptNamespace ?? false,
|
|
101
|
-
maxEntries: options.maxEntries ?? 0
|
|
102
|
-
};
|
|
103
|
-
this.cacheDir = path.resolve(this.options.cacheDir);
|
|
104
|
-
if (!fs.existsSync(this.cacheDir)) {
|
|
105
|
-
fs.mkdirSync(this.cacheDir, { recursive: true });
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
static async create(options = {}) {
|
|
109
|
-
const cacher = new FileCacher(options);
|
|
110
|
-
return cacher;
|
|
111
|
-
}
|
|
112
|
-
async getDatabase(namespace) {
|
|
113
|
-
const nsKey = this.options.encryptNamespace ? createHash("md5").update(namespace).digest("hex") : namespace.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
114
|
-
if (this.databases.has(nsKey)) {
|
|
115
|
-
return this.databases.get(nsKey);
|
|
116
|
-
}
|
|
117
|
-
const dbPath = path.join(this.cacheDir, `${nsKey}.db`);
|
|
118
|
-
const db = await createDatabase(dbPath);
|
|
119
|
-
db.run(`
|
|
120
|
-
CREATE TABLE IF NOT EXISTS cache (
|
|
121
|
-
key TEXT PRIMARY KEY,
|
|
122
|
-
value TEXT NOT NULL,
|
|
123
|
-
expiresAt INTEGER NOT NULL,
|
|
124
|
-
createdAt INTEGER NOT NULL,
|
|
125
|
-
compressed INTEGER DEFAULT 0,
|
|
126
|
-
deleted INTEGER DEFAULT 0
|
|
127
|
-
)
|
|
128
|
-
`);
|
|
129
|
-
db.run("CREATE INDEX IF NOT EXISTS idx_expires ON cache(expiresAt)");
|
|
130
|
-
db.run("CREATE INDEX IF NOT EXISTS idx_deleted ON cache(deleted)");
|
|
131
|
-
this.databases.set(nsKey, db);
|
|
132
|
-
return db;
|
|
133
|
-
}
|
|
134
|
-
async set(key, value, ttl, namespace = "default") {
|
|
135
|
-
if (this.closed)
|
|
136
|
-
throw new Error("FileCacher is closed");
|
|
137
|
-
const db = await this.getDatabase(namespace);
|
|
138
|
-
const now = Date.now();
|
|
139
|
-
const expiresAt = now + (ttl ?? this.options.ttl);
|
|
140
|
-
let serialized = JSON.stringify(value);
|
|
141
|
-
let compressed = 0;
|
|
142
|
-
if (this.options.compression) {
|
|
143
|
-
try {
|
|
144
|
-
const compressedData = await compressData(Buffer.from(serialized, "utf-8"));
|
|
145
|
-
serialized = compressedData.toString("base64");
|
|
146
|
-
compressed = 1;
|
|
147
|
-
} catch {}
|
|
148
|
-
}
|
|
149
|
-
db.run(`
|
|
150
|
-
INSERT OR REPLACE INTO cache (key, value, expiresAt, createdAt, compressed, deleted)
|
|
151
|
-
VALUES (?, ?, ?, ?, ?, 0)
|
|
152
|
-
`, key, serialized, expiresAt, now, compressed);
|
|
153
|
-
if (this.options.maxEntries > 0) {
|
|
154
|
-
const count = db.get("SELECT COUNT(*) as cnt FROM cache WHERE deleted = 0");
|
|
155
|
-
if (count && count.cnt > this.options.maxEntries) {
|
|
156
|
-
const excess = count.cnt - this.options.maxEntries;
|
|
157
|
-
db.run(`
|
|
158
|
-
DELETE FROM cache WHERE key IN (
|
|
159
|
-
SELECT key FROM cache WHERE deleted = 0 ORDER BY createdAt ASC LIMIT ?
|
|
160
|
-
)
|
|
161
|
-
`, excess);
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
async get(key, namespace = "default") {
|
|
166
|
-
if (this.closed)
|
|
167
|
-
throw new Error("FileCacher is closed");
|
|
168
|
-
const db = await this.getDatabase(namespace);
|
|
169
|
-
const entry = db.get("SELECT * FROM cache WHERE key = ? AND deleted = 0", key);
|
|
170
|
-
if (!entry)
|
|
171
|
-
return null;
|
|
172
|
-
if (entry.expiresAt < Date.now()) {
|
|
173
|
-
if (this.options.softDelete) {
|
|
174
|
-
db.run("UPDATE cache SET deleted = 1 WHERE key = ?", key);
|
|
175
|
-
} else {
|
|
176
|
-
db.run("DELETE FROM cache WHERE key = ?", key);
|
|
177
|
-
}
|
|
178
|
-
return null;
|
|
179
|
-
}
|
|
180
|
-
let value = entry.value;
|
|
181
|
-
if (entry.compressed) {
|
|
182
|
-
try {
|
|
183
|
-
const decompressed = await decompressData(Buffer.from(value, "base64"));
|
|
184
|
-
value = decompressed.toString("utf-8");
|
|
185
|
-
} catch {
|
|
186
|
-
return null;
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
try {
|
|
190
|
-
return JSON.parse(value);
|
|
191
|
-
} catch {
|
|
192
|
-
return null;
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
async has(key, namespace = "default") {
|
|
196
|
-
if (this.closed)
|
|
197
|
-
return false;
|
|
198
|
-
const db = await this.getDatabase(namespace);
|
|
199
|
-
const entry = db.get("SELECT key, expiresAt FROM cache WHERE key = ? AND deleted = 0", key);
|
|
200
|
-
if (!entry)
|
|
201
|
-
return false;
|
|
202
|
-
return entry.expiresAt >= Date.now();
|
|
203
|
-
}
|
|
204
|
-
async delete(key, namespace = "default") {
|
|
205
|
-
if (this.closed)
|
|
206
|
-
return false;
|
|
207
|
-
const db = await this.getDatabase(namespace);
|
|
208
|
-
if (this.options.softDelete) {
|
|
209
|
-
db.run("UPDATE cache SET deleted = 1 WHERE key = ?", key);
|
|
210
|
-
} else {
|
|
211
|
-
db.run("DELETE FROM cache WHERE key = ?", key);
|
|
212
|
-
}
|
|
213
|
-
return true;
|
|
214
|
-
}
|
|
215
|
-
async clear(namespace = "default") {
|
|
216
|
-
if (this.closed)
|
|
217
|
-
return;
|
|
218
|
-
const db = await this.getDatabase(namespace);
|
|
219
|
-
if (this.options.softDelete) {
|
|
220
|
-
db.run("UPDATE cache SET deleted = 1");
|
|
221
|
-
} else {
|
|
222
|
-
db.run("DELETE FROM cache");
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
async cleanup(namespace = "default") {
|
|
226
|
-
if (this.closed)
|
|
227
|
-
return 0;
|
|
228
|
-
const db = await this.getDatabase(namespace);
|
|
229
|
-
const now = Date.now();
|
|
230
|
-
const countBefore = db.get("SELECT COUNT(*) as cnt FROM cache");
|
|
231
|
-
db.run("DELETE FROM cache WHERE expiresAt < ? OR deleted = 1", now);
|
|
232
|
-
const countAfter = db.get("SELECT COUNT(*) as cnt FROM cache");
|
|
233
|
-
return (countBefore?.cnt || 0) - (countAfter?.cnt || 0);
|
|
234
|
-
}
|
|
235
|
-
async stats(namespace = "default") {
|
|
236
|
-
if (this.closed)
|
|
237
|
-
return { count: 0, expired: 0, deleted: 0 };
|
|
238
|
-
const db = await this.getDatabase(namespace);
|
|
239
|
-
const now = Date.now();
|
|
240
|
-
const total = db.get("SELECT COUNT(*) as cnt FROM cache WHERE deleted = 0");
|
|
241
|
-
const expired = db.get("SELECT COUNT(*) as cnt FROM cache WHERE expiresAt < ? AND deleted = 0", now);
|
|
242
|
-
const deleted = db.get("SELECT COUNT(*) as cnt FROM cache WHERE deleted = 1");
|
|
243
|
-
return {
|
|
244
|
-
count: total?.cnt || 0,
|
|
245
|
-
expired: expired?.cnt || 0,
|
|
246
|
-
deleted: deleted?.cnt || 0
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
|
-
async close() {
|
|
250
|
-
if (this.closed)
|
|
251
|
-
return;
|
|
252
|
-
this.closed = true;
|
|
253
|
-
for (const db of this.databases.values()) {
|
|
254
|
-
try {
|
|
255
|
-
db.close();
|
|
256
|
-
} catch {}
|
|
257
|
-
}
|
|
258
|
-
this.databases.clear();
|
|
259
|
-
}
|
|
260
|
-
get isClosed() {
|
|
261
|
-
return this.closed;
|
|
262
|
-
}
|
|
263
|
-
get directory() {
|
|
264
|
-
return this.cacheDir;
|
|
265
|
-
}
|
|
266
|
-
}
|
|
267
|
-
export default FileCacher;
|
|
@@ -1,298 +0,0 @@
|
|
|
1
|
-
const fs = require("node:fs");
|
|
2
|
-
const path = require("node:path");
|
|
3
|
-
const { createHash } = require("node:crypto");
|
|
4
|
-
function detectRuntime() {
|
|
5
|
-
if (typeof globalThis.Bun !== "undefined")
|
|
6
|
-
return "bun";
|
|
7
|
-
if (typeof globalThis.Deno !== "undefined")
|
|
8
|
-
return "deno";
|
|
9
|
-
return "node";
|
|
10
|
-
}
|
|
11
|
-
async function createDatabase(dbPath) {
|
|
12
|
-
const runtime = detectRuntime();
|
|
13
|
-
if (runtime === "bun") {
|
|
14
|
-
const { Database } = await import("bun:sqlite");
|
|
15
|
-
const db = new Database(dbPath);
|
|
16
|
-
return {
|
|
17
|
-
run: (sql, ...params) => db.run(sql, ...params),
|
|
18
|
-
get: (sql, ...params) => db.query(sql).get(...params),
|
|
19
|
-
all: (sql, ...params) => db.query(sql).all(...params),
|
|
20
|
-
close: () => db.close()
|
|
21
|
-
};
|
|
22
|
-
}
|
|
23
|
-
if (runtime === "deno") {
|
|
24
|
-
try {
|
|
25
|
-
const { Database } = await import("node:sqlite");
|
|
26
|
-
const db = new Database(dbPath);
|
|
27
|
-
return {
|
|
28
|
-
run: (sql, ...params) => db.exec(sql, params),
|
|
29
|
-
get: (sql, ...params) => {
|
|
30
|
-
const stmt = db.prepare(sql);
|
|
31
|
-
return stmt.get(...params);
|
|
32
|
-
},
|
|
33
|
-
all: (sql, ...params) => {
|
|
34
|
-
const stmt = db.prepare(sql);
|
|
35
|
-
return stmt.all(...params);
|
|
36
|
-
},
|
|
37
|
-
close: () => db.close()
|
|
38
|
-
};
|
|
39
|
-
} catch {
|
|
40
|
-
throw new Error("Deno SQLite support requires Node.js compatibility mode");
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
const { DatabaseSync } = await import("node:sqlite");
|
|
44
|
-
const db = new DatabaseSync(dbPath);
|
|
45
|
-
return {
|
|
46
|
-
run: (sql, ...params) => {
|
|
47
|
-
const stmt = db.prepare(sql);
|
|
48
|
-
stmt.run(...params);
|
|
49
|
-
},
|
|
50
|
-
get: (sql, ...params) => {
|
|
51
|
-
const stmt = db.prepare(sql);
|
|
52
|
-
return stmt.get(...params);
|
|
53
|
-
},
|
|
54
|
-
all: (sql, ...params) => {
|
|
55
|
-
const stmt = db.prepare(sql);
|
|
56
|
-
return stmt.all(...params);
|
|
57
|
-
},
|
|
58
|
-
close: () => db.close()
|
|
59
|
-
};
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
class NavigationHistory {
|
|
63
|
-
db = null;
|
|
64
|
-
options;
|
|
65
|
-
storeDir;
|
|
66
|
-
dbPath;
|
|
67
|
-
closed = false;
|
|
68
|
-
initPromise = null;
|
|
69
|
-
constructor(options = {}) {
|
|
70
|
-
this.options = {
|
|
71
|
-
storeDir: options.storeDir || "./navigation-history",
|
|
72
|
-
dbFileName: options.dbFileName || "navigation.db",
|
|
73
|
-
hashUrls: options.hashUrls ?? false
|
|
74
|
-
};
|
|
75
|
-
this.storeDir = path.resolve(this.options.storeDir);
|
|
76
|
-
this.dbPath = path.join(this.storeDir, this.options.dbFileName);
|
|
77
|
-
if (!fs.existsSync(this.storeDir)) {
|
|
78
|
-
fs.mkdirSync(this.storeDir, { recursive: true });
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
static async create(options = {}) {
|
|
82
|
-
const store = new NavigationHistory(options);
|
|
83
|
-
await store.initialize();
|
|
84
|
-
return store;
|
|
85
|
-
}
|
|
86
|
-
async initialize() {
|
|
87
|
-
if (this.initPromise)
|
|
88
|
-
return this.initPromise;
|
|
89
|
-
this.initPromise = (async () => {
|
|
90
|
-
this.db = await createDatabase(this.dbPath);
|
|
91
|
-
this.db.run(`
|
|
92
|
-
CREATE TABLE IF NOT EXISTS sessions (
|
|
93
|
-
sessionId TEXT PRIMARY KEY,
|
|
94
|
-
baseUrl TEXT NOT NULL,
|
|
95
|
-
startedAt INTEGER NOT NULL,
|
|
96
|
-
lastActivityAt INTEGER NOT NULL,
|
|
97
|
-
status TEXT DEFAULT 'running',
|
|
98
|
-
urlsVisited INTEGER DEFAULT 0,
|
|
99
|
-
urlsQueued INTEGER DEFAULT 0,
|
|
100
|
-
urlsFailed INTEGER DEFAULT 0,
|
|
101
|
-
metadata TEXT
|
|
102
|
-
)
|
|
103
|
-
`);
|
|
104
|
-
this.db.run(`
|
|
105
|
-
CREATE TABLE IF NOT EXISTS queue (
|
|
106
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
107
|
-
sessionId TEXT NOT NULL,
|
|
108
|
-
urlKey TEXT NOT NULL,
|
|
109
|
-
originalUrl TEXT NOT NULL,
|
|
110
|
-
method TEXT DEFAULT 'GET',
|
|
111
|
-
priority INTEGER DEFAULT 0,
|
|
112
|
-
body TEXT,
|
|
113
|
-
headers TEXT,
|
|
114
|
-
metadata TEXT,
|
|
115
|
-
addedAt INTEGER NOT NULL,
|
|
116
|
-
UNIQUE(sessionId, urlKey)
|
|
117
|
-
)
|
|
118
|
-
`);
|
|
119
|
-
this.db.run(`
|
|
120
|
-
CREATE TABLE IF NOT EXISTS visited (
|
|
121
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
122
|
-
sessionId TEXT NOT NULL,
|
|
123
|
-
urlKey TEXT NOT NULL,
|
|
124
|
-
originalUrl TEXT NOT NULL,
|
|
125
|
-
status INTEGER,
|
|
126
|
-
visitedAt INTEGER NOT NULL,
|
|
127
|
-
finalUrl TEXT,
|
|
128
|
-
contentType TEXT,
|
|
129
|
-
errorMessage TEXT,
|
|
130
|
-
UNIQUE(sessionId, urlKey)
|
|
131
|
-
)
|
|
132
|
-
`);
|
|
133
|
-
this.db.run("CREATE INDEX IF NOT EXISTS idx_queue_session ON queue(sessionId)");
|
|
134
|
-
this.db.run("CREATE INDEX IF NOT EXISTS idx_queue_priority ON queue(sessionId, priority DESC)");
|
|
135
|
-
this.db.run("CREATE INDEX IF NOT EXISTS idx_visited_session ON visited(sessionId)");
|
|
136
|
-
this.db.run("CREATE INDEX IF NOT EXISTS idx_sessions_status ON sessions(status)");
|
|
137
|
-
})();
|
|
138
|
-
return this.initPromise;
|
|
139
|
-
}
|
|
140
|
-
getUrlKey(url) {
|
|
141
|
-
if (this.options.hashUrls) {
|
|
142
|
-
return createHash("sha256").update(url).digest("hex");
|
|
143
|
-
}
|
|
144
|
-
return url;
|
|
145
|
-
}
|
|
146
|
-
async createSession(sessionId, baseUrl, metadata) {
|
|
147
|
-
if (this.closed || !this.db)
|
|
148
|
-
throw new Error("NavigationHistory is closed");
|
|
149
|
-
const now = Date.now();
|
|
150
|
-
const session = {
|
|
151
|
-
sessionId,
|
|
152
|
-
baseUrl,
|
|
153
|
-
startedAt: now,
|
|
154
|
-
lastActivityAt: now,
|
|
155
|
-
status: "running",
|
|
156
|
-
urlsVisited: 0,
|
|
157
|
-
urlsQueued: 0,
|
|
158
|
-
urlsFailed: 0,
|
|
159
|
-
metadata: metadata ? JSON.stringify(metadata) : undefined
|
|
160
|
-
};
|
|
161
|
-
this.db.run(`INSERT OR REPLACE INTO sessions (sessionId, baseUrl, startedAt, lastActivityAt, status, urlsVisited, urlsQueued, urlsFailed, metadata)
|
|
162
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, baseUrl, now, now, "running", 0, 0, 0, session.metadata ?? null);
|
|
163
|
-
return session;
|
|
164
|
-
}
|
|
165
|
-
async getSession(sessionId) {
|
|
166
|
-
if (this.closed || !this.db)
|
|
167
|
-
throw new Error("NavigationHistory is closed");
|
|
168
|
-
return this.db.get("SELECT * FROM sessions WHERE sessionId = ?", sessionId);
|
|
169
|
-
}
|
|
170
|
-
async updateSessionStatus(sessionId, status) {
|
|
171
|
-
if (this.closed || !this.db)
|
|
172
|
-
throw new Error("NavigationHistory is closed");
|
|
173
|
-
this.db.run("UPDATE sessions SET status = ?, lastActivityAt = ? WHERE sessionId = ?", status, Date.now(), sessionId);
|
|
174
|
-
}
|
|
175
|
-
async updateSessionStats(sessionId, stats) {
|
|
176
|
-
if (this.closed || !this.db)
|
|
177
|
-
throw new Error("NavigationHistory is closed");
|
|
178
|
-
const updates = ["lastActivityAt = ?"];
|
|
179
|
-
const params = [Date.now()];
|
|
180
|
-
if (stats.urlsVisited !== undefined) {
|
|
181
|
-
updates.push("urlsVisited = ?");
|
|
182
|
-
params.push(stats.urlsVisited);
|
|
183
|
-
}
|
|
184
|
-
if (stats.urlsQueued !== undefined) {
|
|
185
|
-
updates.push("urlsQueued = ?");
|
|
186
|
-
params.push(stats.urlsQueued);
|
|
187
|
-
}
|
|
188
|
-
if (stats.urlsFailed !== undefined) {
|
|
189
|
-
updates.push("urlsFailed = ?");
|
|
190
|
-
params.push(stats.urlsFailed);
|
|
191
|
-
}
|
|
192
|
-
params.push(sessionId);
|
|
193
|
-
this.db.run(`UPDATE sessions SET ${updates.join(", ")} WHERE sessionId = ?`, ...params);
|
|
194
|
-
}
|
|
195
|
-
async addToQueue(sessionId, url, options = {}) {
|
|
196
|
-
if (this.closed || !this.db)
|
|
197
|
-
throw new Error("NavigationHistory is closed");
|
|
198
|
-
const urlKey = this.getUrlKey(url);
|
|
199
|
-
const existing = this.db.get("SELECT id FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
200
|
-
if (existing)
|
|
201
|
-
return false;
|
|
202
|
-
const isVisited = this.db.get("SELECT id FROM visited WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
203
|
-
if (isVisited)
|
|
204
|
-
return false;
|
|
205
|
-
this.db.run(`INSERT INTO queue (sessionId, urlKey, originalUrl, method, priority, body, headers, metadata, addedAt)
|
|
206
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, urlKey, url, options.method || "GET", options.priority || 0, options.body ? JSON.stringify(options.body) : null, options.headers ? JSON.stringify(options.headers) : null, options.metadata ? JSON.stringify(options.metadata) : null, Date.now());
|
|
207
|
-
return true;
|
|
208
|
-
}
|
|
209
|
-
async getNextFromQueue(sessionId) {
|
|
210
|
-
if (this.closed || !this.db)
|
|
211
|
-
throw new Error("NavigationHistory is closed");
|
|
212
|
-
const item = this.db.get("SELECT originalUrl as url, method, priority, body, headers, metadata, addedAt FROM queue WHERE sessionId = ? ORDER BY priority DESC, addedAt ASC LIMIT 1", sessionId);
|
|
213
|
-
return item;
|
|
214
|
-
}
|
|
215
|
-
async removeFromQueue(sessionId, url) {
|
|
216
|
-
if (this.closed || !this.db)
|
|
217
|
-
throw new Error("NavigationHistory is closed");
|
|
218
|
-
const urlKey = this.getUrlKey(url);
|
|
219
|
-
this.db.run("DELETE FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
220
|
-
return true;
|
|
221
|
-
}
|
|
222
|
-
async getQueueSize(sessionId) {
|
|
223
|
-
if (this.closed || !this.db)
|
|
224
|
-
throw new Error("NavigationHistory is closed");
|
|
225
|
-
const result = this.db.get("SELECT COUNT(*) as count FROM queue WHERE sessionId = ?", sessionId);
|
|
226
|
-
return result?.count || 0;
|
|
227
|
-
}
|
|
228
|
-
async markVisited(sessionId, url, result = {}) {
|
|
229
|
-
if (this.closed || !this.db)
|
|
230
|
-
throw new Error("NavigationHistory is closed");
|
|
231
|
-
const urlKey = this.getUrlKey(url);
|
|
232
|
-
this.db.run("DELETE FROM queue WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
233
|
-
this.db.run(`INSERT OR REPLACE INTO visited (sessionId, urlKey, originalUrl, status, visitedAt, finalUrl, contentType, errorMessage)
|
|
234
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, sessionId, urlKey, url, result.status || 0, Date.now(), result.finalUrl ?? null, result.contentType ?? null, result.errorMessage ?? null);
|
|
235
|
-
}
|
|
236
|
-
async isVisited(sessionId, url) {
|
|
237
|
-
if (this.closed || !this.db)
|
|
238
|
-
throw new Error("NavigationHistory is closed");
|
|
239
|
-
const urlKey = this.getUrlKey(url);
|
|
240
|
-
const result = this.db.get("SELECT id FROM visited WHERE sessionId = ? AND urlKey = ?", sessionId, urlKey);
|
|
241
|
-
return !!result;
|
|
242
|
-
}
|
|
243
|
-
async getVisitedCount(sessionId) {
|
|
244
|
-
if (this.closed || !this.db)
|
|
245
|
-
throw new Error("NavigationHistory is closed");
|
|
246
|
-
const result = this.db.get("SELECT COUNT(*) as count FROM visited WHERE sessionId = ?", sessionId);
|
|
247
|
-
return result?.count || 0;
|
|
248
|
-
}
|
|
249
|
-
async getFailedUrls(sessionId) {
|
|
250
|
-
if (this.closed || !this.db)
|
|
251
|
-
throw new Error("NavigationHistory is closed");
|
|
252
|
-
return this.db.all("SELECT url, status, visitedAt, finalUrl, contentType, errorMessage FROM visited WHERE sessionId = ? AND (status >= 400 OR errorMessage IS NOT NULL)", sessionId);
|
|
253
|
-
}
|
|
254
|
-
async getAllQueuedUrls(sessionId) {
|
|
255
|
-
if (this.closed || !this.db)
|
|
256
|
-
throw new Error("NavigationHistory is closed");
|
|
257
|
-
return this.db.all("SELECT originalUrl as url, method, priority, body, headers, metadata, addedAt FROM queue WHERE sessionId = ? ORDER BY priority DESC, addedAt ASC", sessionId);
|
|
258
|
-
}
|
|
259
|
-
async clearQueue(sessionId) {
|
|
260
|
-
if (this.closed || !this.db)
|
|
261
|
-
throw new Error("NavigationHistory is closed");
|
|
262
|
-
this.db.run("DELETE FROM queue WHERE sessionId = ?", sessionId);
|
|
263
|
-
}
|
|
264
|
-
async clearVisited(sessionId) {
|
|
265
|
-
if (this.closed || !this.db)
|
|
266
|
-
throw new Error("NavigationHistory is closed");
|
|
267
|
-
this.db.run("DELETE FROM visited WHERE sessionId = ?", sessionId);
|
|
268
|
-
}
|
|
269
|
-
async deleteSession(sessionId) {
|
|
270
|
-
if (this.closed || !this.db)
|
|
271
|
-
throw new Error("NavigationHistory is closed");
|
|
272
|
-
this.db.run("DELETE FROM queue WHERE sessionId = ?", sessionId);
|
|
273
|
-
this.db.run("DELETE FROM visited WHERE sessionId = ?", sessionId);
|
|
274
|
-
this.db.run("DELETE FROM sessions WHERE sessionId = ?", sessionId);
|
|
275
|
-
}
|
|
276
|
-
async getResumableSessions() {
|
|
277
|
-
if (this.closed || !this.db)
|
|
278
|
-
throw new Error("NavigationHistory is closed");
|
|
279
|
-
return this.db.all("SELECT * FROM sessions WHERE status IN ('running', 'paused') ORDER BY lastActivityAt DESC");
|
|
280
|
-
}
|
|
281
|
-
async close() {
|
|
282
|
-
if (this.closed)
|
|
283
|
-
return;
|
|
284
|
-
this.closed = true;
|
|
285
|
-
if (this.db) {
|
|
286
|
-
this.db.close();
|
|
287
|
-
this.db = null;
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
get isClosed() {
|
|
291
|
-
return this.closed;
|
|
292
|
-
}
|
|
293
|
-
get databasePath() {
|
|
294
|
-
return this.dbPath;
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
exports.NavigationHistory = NavigationHistory;
|