rezo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +1507 -0
- package/assets/icon.svg +37 -0
- package/assets/logo-dark.svg +47 -0
- package/assets/logo.svg +58 -0
- package/dist/adapters/curl.cjs +1034 -0
- package/dist/adapters/curl.js +1031 -0
- package/dist/adapters/entries/curl.cjs +4 -0
- package/dist/adapters/entries/curl.d.ts +2136 -0
- package/dist/adapters/entries/curl.js +2 -0
- package/dist/adapters/entries/fetch.cjs +2 -0
- package/dist/adapters/entries/fetch.d.ts +2127 -0
- package/dist/adapters/entries/fetch.js +1 -0
- package/dist/adapters/entries/http.cjs +2 -0
- package/dist/adapters/entries/http.d.ts +2126 -0
- package/dist/adapters/entries/http.js +1 -0
- package/dist/adapters/entries/http2.cjs +4 -0
- package/dist/adapters/entries/http2.d.ts +2136 -0
- package/dist/adapters/entries/http2.js +2 -0
- package/dist/adapters/entries/react-native.cjs +2 -0
- package/dist/adapters/entries/react-native.d.ts +2126 -0
- package/dist/adapters/entries/react-native.js +1 -0
- package/dist/adapters/entries/xhr.cjs +2 -0
- package/dist/adapters/entries/xhr.d.ts +2127 -0
- package/dist/adapters/entries/xhr.js +1 -0
- package/dist/adapters/fetch.cjs +740 -0
- package/dist/adapters/fetch.js +739 -0
- package/dist/adapters/http.cjs +1153 -0
- package/dist/adapters/http.js +1151 -0
- package/dist/adapters/http2.cjs +957 -0
- package/dist/adapters/http2.js +956 -0
- package/dist/adapters/index.cjs +6 -0
- package/dist/adapters/index.js +7 -0
- package/dist/adapters/picker.cjs +342 -0
- package/dist/adapters/picker.js +331 -0
- package/dist/adapters/react-native.cjs +545 -0
- package/dist/adapters/react-native.js +544 -0
- package/dist/adapters/xhr.cjs +622 -0
- package/dist/adapters/xhr.js +621 -0
- package/dist/cache/dns-cache.cjs +118 -0
- package/dist/cache/dns-cache.js +113 -0
- package/dist/cache/file-cacher.cjs +264 -0
- package/dist/cache/file-cacher.js +261 -0
- package/dist/cache/index.cjs +13 -0
- package/dist/cache/index.js +5 -0
- package/dist/cache/lru-cache.cjs +96 -0
- package/dist/cache/lru-cache.js +93 -0
- package/dist/cache/response-cache.cjs +314 -0
- package/dist/cache/response-cache.js +310 -0
- package/dist/cache/url-store.cjs +288 -0
- package/dist/cache/url-store.js +285 -0
- package/dist/core/hooks.cjs +133 -0
- package/dist/core/hooks.js +120 -0
- package/dist/core/rezo.cjs +464 -0
- package/dist/core/rezo.js +458 -0
- package/dist/crawler.d.ts +6255 -0
- package/dist/dom/index.cjs +1 -0
- package/dist/dom/index.d.ts +23 -0
- package/dist/dom/index.js +1 -0
- package/dist/entries/crawler.cjs +5 -0
- package/dist/entries/crawler.js +2 -0
- package/dist/errors/rezo-error.cjs +722 -0
- package/dist/errors/rezo-error.js +716 -0
- package/dist/index.cjs +34 -0
- package/dist/index.d.ts +3335 -0
- package/dist/index.js +26 -0
- package/dist/platform/browser.cjs +9 -0
- package/dist/platform/browser.d.ts +3203 -0
- package/dist/platform/browser.js +7 -0
- package/dist/platform/bun.cjs +9 -0
- package/dist/platform/bun.d.ts +3203 -0
- package/dist/platform/bun.js +7 -0
- package/dist/platform/deno.cjs +9 -0
- package/dist/platform/deno.d.ts +3203 -0
- package/dist/platform/deno.js +7 -0
- package/dist/platform/node.cjs +9 -0
- package/dist/platform/node.d.ts +3203 -0
- package/dist/platform/node.js +7 -0
- package/dist/platform/react-native.cjs +9 -0
- package/dist/platform/react-native.d.ts +3203 -0
- package/dist/platform/react-native.js +7 -0
- package/dist/platform/worker.cjs +9 -0
- package/dist/platform/worker.d.ts +3203 -0
- package/dist/platform/worker.js +7 -0
- package/dist/plugin/addon/decodo/index.cjs +1 -0
- package/dist/plugin/addon/decodo/index.js +1 -0
- package/dist/plugin/addon/decodo/options.cjs +1 -0
- package/dist/plugin/addon/decodo/options.js +1 -0
- package/dist/plugin/addon/oxylabs/index.cjs +1 -0
- package/dist/plugin/addon/oxylabs/index.js +1 -0
- package/dist/plugin/addon/oxylabs/options.cjs +1 -0
- package/dist/plugin/addon/oxylabs/options.js +1 -0
- package/dist/plugin/crawler-options.cjs +1 -0
- package/dist/plugin/crawler-options.js +1 -0
- package/dist/plugin/crawler.cjs +519 -0
- package/dist/plugin/crawler.js +517 -0
- package/dist/plugin/index.cjs +36 -0
- package/dist/plugin/index.js +32 -0
- package/dist/proxy/index.cjs +142 -0
- package/dist/proxy/index.js +139 -0
- package/dist/responses/buildError.cjs +452 -0
- package/dist/responses/buildError.js +441 -0
- package/dist/responses/buildResponse.cjs +365 -0
- package/dist/responses/buildResponse.js +361 -0
- package/dist/responses/download.cjs +54 -0
- package/dist/responses/download.js +52 -0
- package/dist/responses/stream.cjs +60 -0
- package/dist/responses/stream.js +58 -0
- package/dist/responses/upload.cjs +54 -0
- package/dist/responses/upload.js +52 -0
- package/dist/types/cookies.cjs +394 -0
- package/dist/types/cookies.js +391 -0
- package/dist/types/download.cjs +10 -0
- package/dist/types/download.js +10 -0
- package/dist/types/rezo-request.cjs +131 -0
- package/dist/types/rezo-request.js +131 -0
- package/dist/utils/agent-merger.cjs +111 -0
- package/dist/utils/agent-merger.js +108 -0
- package/dist/utils/compression.cjs +84 -0
- package/dist/utils/compression.js +82 -0
- package/dist/utils/cookies.cjs +514 -0
- package/dist/utils/cookies.js +511 -0
- package/dist/utils/data-operations.cjs +75 -0
- package/dist/utils/data-operations.js +73 -0
- package/dist/utils/form-data.cjs +164 -0
- package/dist/utils/form-data.js +161 -0
- package/dist/utils/headers.cjs +162 -0
- package/dist/utils/headers.js +161 -0
- package/dist/utils/http-config.cjs +723 -0
- package/dist/utils/http-config.js +718 -0
- package/dist/utils/index.cjs +8 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/tools.cjs +18 -0
- package/dist/utils/tools.js +15 -0
- package/package.json +172 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
const fs = require("node:fs");
|
|
2
|
+
const path = require("node:path");
|
|
3
|
+
const { createHash } = require("node:crypto");
|
|
4
|
+
function detectRuntime() {
|
|
5
|
+
if (typeof globalThis.Bun !== "undefined") {
|
|
6
|
+
return "bun";
|
|
7
|
+
}
|
|
8
|
+
if (typeof globalThis.Deno !== "undefined") {
|
|
9
|
+
return "deno";
|
|
10
|
+
}
|
|
11
|
+
return "node";
|
|
12
|
+
}
|
|
13
|
+
async function createDatabase(dbPath) {
|
|
14
|
+
const runtime = detectRuntime();
|
|
15
|
+
if (runtime === "bun") {
|
|
16
|
+
const { Database } = await import("bun:sqlite");
|
|
17
|
+
const db = new Database(dbPath);
|
|
18
|
+
return {
|
|
19
|
+
run: (sql, ...params) => db.run(sql, ...params),
|
|
20
|
+
get: (sql, ...params) => db.query(sql).get(...params),
|
|
21
|
+
all: (sql, ...params) => db.query(sql).all(...params),
|
|
22
|
+
close: () => db.close()
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
if (runtime === "deno") {
|
|
26
|
+
try {
|
|
27
|
+
const { Database } = await import("node:sqlite");
|
|
28
|
+
const db = new Database(dbPath);
|
|
29
|
+
return {
|
|
30
|
+
run: (sql, ...params) => db.exec(sql, params),
|
|
31
|
+
get: (sql, ...params) => {
|
|
32
|
+
const stmt = db.prepare(sql);
|
|
33
|
+
return stmt.get(...params);
|
|
34
|
+
},
|
|
35
|
+
all: (sql, ...params) => {
|
|
36
|
+
const stmt = db.prepare(sql);
|
|
37
|
+
return stmt.all(...params);
|
|
38
|
+
},
|
|
39
|
+
close: () => db.close()
|
|
40
|
+
};
|
|
41
|
+
} catch {
|
|
42
|
+
throw new Error("Deno SQLite support requires Node.js compatibility mode");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
46
|
+
const db = new DatabaseSync(dbPath);
|
|
47
|
+
return {
|
|
48
|
+
run: (sql, ...params) => db.exec(sql),
|
|
49
|
+
get: (sql, ...params) => {
|
|
50
|
+
const stmt = db.prepare(sql);
|
|
51
|
+
return stmt.get(...params);
|
|
52
|
+
},
|
|
53
|
+
all: (sql, ...params) => {
|
|
54
|
+
const stmt = db.prepare(sql);
|
|
55
|
+
return stmt.all(...params);
|
|
56
|
+
},
|
|
57
|
+
close: () => db.close()
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
class UrlStore {
|
|
62
|
+
db = null;
|
|
63
|
+
options;
|
|
64
|
+
storeDir;
|
|
65
|
+
dbPath;
|
|
66
|
+
closed = false;
|
|
67
|
+
initPromise = null;
|
|
68
|
+
constructor(options = {}) {
|
|
69
|
+
this.options = {
|
|
70
|
+
storeDir: options.storeDir || "./url-store",
|
|
71
|
+
dbFileName: options.dbFileName || "urls.db",
|
|
72
|
+
ttl: options.ttl || 604800000,
|
|
73
|
+
maxUrls: options.maxUrls ?? 0,
|
|
74
|
+
hashUrls: options.hashUrls ?? false
|
|
75
|
+
};
|
|
76
|
+
this.storeDir = path.resolve(this.options.storeDir);
|
|
77
|
+
this.dbPath = path.join(this.storeDir, this.options.dbFileName);
|
|
78
|
+
if (!fs.existsSync(this.storeDir)) {
|
|
79
|
+
fs.mkdirSync(this.storeDir, { recursive: true });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
static async create(options = {}) {
|
|
83
|
+
const store = new UrlStore(options);
|
|
84
|
+
await store.initialize();
|
|
85
|
+
return store;
|
|
86
|
+
}
|
|
87
|
+
async initialize() {
|
|
88
|
+
if (this.initPromise)
|
|
89
|
+
return this.initPromise;
|
|
90
|
+
this.initPromise = (async () => {
|
|
91
|
+
this.db = await createDatabase(this.dbPath);
|
|
92
|
+
this.db.run(`
|
|
93
|
+
CREATE TABLE IF NOT EXISTS urls (
|
|
94
|
+
url TEXT PRIMARY KEY,
|
|
95
|
+
visitedAt INTEGER NOT NULL,
|
|
96
|
+
expiresAt INTEGER NOT NULL,
|
|
97
|
+
namespace TEXT DEFAULT 'default',
|
|
98
|
+
metadata TEXT
|
|
99
|
+
)
|
|
100
|
+
`);
|
|
101
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_expires ON urls(expiresAt)");
|
|
102
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_namespace ON urls(namespace)");
|
|
103
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_visited ON urls(visitedAt)");
|
|
104
|
+
})();
|
|
105
|
+
return this.initPromise;
|
|
106
|
+
}
|
|
107
|
+
getUrlKey(url) {
|
|
108
|
+
if (this.options.hashUrls) {
|
|
109
|
+
return createHash("sha256").update(url).digest("hex");
|
|
110
|
+
}
|
|
111
|
+
return url;
|
|
112
|
+
}
|
|
113
|
+
async set(url, namespace = "default", metadata, ttl) {
|
|
114
|
+
if (this.closed)
|
|
115
|
+
throw new Error("UrlStore is closed");
|
|
116
|
+
await this.initialize();
|
|
117
|
+
const key = this.getUrlKey(url);
|
|
118
|
+
const now = Date.now();
|
|
119
|
+
const expiresAt = now + (ttl ?? this.options.ttl);
|
|
120
|
+
const metaStr = metadata ? JSON.stringify(metadata) : null;
|
|
121
|
+
this.db.run(`
|
|
122
|
+
INSERT OR REPLACE INTO urls (url, visitedAt, expiresAt, namespace, metadata)
|
|
123
|
+
VALUES (?, ?, ?, ?, ?)
|
|
124
|
+
`, key, now, expiresAt, namespace, metaStr);
|
|
125
|
+
if (this.options.maxUrls > 0) {
|
|
126
|
+
const count = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
127
|
+
if (count && count.cnt > this.options.maxUrls) {
|
|
128
|
+
const excess = count.cnt - this.options.maxUrls;
|
|
129
|
+
this.db.run(`
|
|
130
|
+
DELETE FROM urls WHERE url IN (
|
|
131
|
+
SELECT url FROM urls ORDER BY visitedAt ASC LIMIT ?
|
|
132
|
+
)
|
|
133
|
+
`, excess);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
async has(url, namespace) {
|
|
138
|
+
if (this.closed)
|
|
139
|
+
return false;
|
|
140
|
+
await this.initialize();
|
|
141
|
+
const key = this.getUrlKey(url);
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
let entry;
|
|
144
|
+
if (namespace) {
|
|
145
|
+
entry = this.db.get("SELECT url, expiresAt FROM urls WHERE url = ? AND namespace = ?", key, namespace);
|
|
146
|
+
} else {
|
|
147
|
+
entry = this.db.get("SELECT url, expiresAt FROM urls WHERE url = ?", key);
|
|
148
|
+
}
|
|
149
|
+
if (!entry)
|
|
150
|
+
return false;
|
|
151
|
+
return entry.expiresAt >= now;
|
|
152
|
+
}
|
|
153
|
+
async hasMany(urls, namespace) {
|
|
154
|
+
if (this.closed)
|
|
155
|
+
return new Set;
|
|
156
|
+
await this.initialize();
|
|
157
|
+
const result = new Set;
|
|
158
|
+
const now = Date.now();
|
|
159
|
+
const batchSize = 100;
|
|
160
|
+
for (let i = 0;i < urls.length; i += batchSize) {
|
|
161
|
+
const batch = urls.slice(i, i + batchSize);
|
|
162
|
+
const keys = batch.map((u) => this.getUrlKey(u));
|
|
163
|
+
const placeholders = keys.map(() => "?").join(",");
|
|
164
|
+
let entries;
|
|
165
|
+
if (namespace) {
|
|
166
|
+
entries = this.db.all(`SELECT url, expiresAt FROM urls WHERE url IN (${placeholders}) AND namespace = ?`, ...keys, namespace);
|
|
167
|
+
} else {
|
|
168
|
+
entries = this.db.all(`SELECT url, expiresAt FROM urls WHERE url IN (${placeholders})`, ...keys);
|
|
169
|
+
}
|
|
170
|
+
for (const entry of entries) {
|
|
171
|
+
if (entry.expiresAt >= now) {
|
|
172
|
+
const idx = keys.indexOf(entry.url);
|
|
173
|
+
if (idx !== -1) {
|
|
174
|
+
result.add(batch[idx]);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return result;
|
|
180
|
+
}
|
|
181
|
+
async getMetadata(url, namespace) {
|
|
182
|
+
if (this.closed)
|
|
183
|
+
return null;
|
|
184
|
+
await this.initialize();
|
|
185
|
+
const key = this.getUrlKey(url);
|
|
186
|
+
let entry;
|
|
187
|
+
if (namespace) {
|
|
188
|
+
entry = this.db.get("SELECT metadata FROM urls WHERE url = ? AND namespace = ?", key, namespace);
|
|
189
|
+
} else {
|
|
190
|
+
entry = this.db.get("SELECT metadata FROM urls WHERE url = ?", key);
|
|
191
|
+
}
|
|
192
|
+
if (!entry?.metadata)
|
|
193
|
+
return null;
|
|
194
|
+
try {
|
|
195
|
+
return JSON.parse(entry.metadata);
|
|
196
|
+
} catch {
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
async delete(url, namespace) {
|
|
201
|
+
if (this.closed)
|
|
202
|
+
return false;
|
|
203
|
+
await this.initialize();
|
|
204
|
+
const key = this.getUrlKey(url);
|
|
205
|
+
if (namespace) {
|
|
206
|
+
this.db.run("DELETE FROM urls WHERE url = ? AND namespace = ?", key, namespace);
|
|
207
|
+
} else {
|
|
208
|
+
this.db.run("DELETE FROM urls WHERE url = ?", key);
|
|
209
|
+
}
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
212
|
+
async clear(namespace) {
|
|
213
|
+
if (this.closed)
|
|
214
|
+
return;
|
|
215
|
+
await this.initialize();
|
|
216
|
+
if (namespace) {
|
|
217
|
+
this.db.run("DELETE FROM urls WHERE namespace = ?", namespace);
|
|
218
|
+
} else {
|
|
219
|
+
this.db.run("DELETE FROM urls");
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
async cleanup() {
|
|
223
|
+
if (this.closed)
|
|
224
|
+
return 0;
|
|
225
|
+
await this.initialize();
|
|
226
|
+
const now = Date.now();
|
|
227
|
+
const countBefore = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
228
|
+
this.db.run("DELETE FROM urls WHERE expiresAt < ?", now);
|
|
229
|
+
const countAfter = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
230
|
+
return (countBefore?.cnt || 0) - (countAfter?.cnt || 0);
|
|
231
|
+
}
|
|
232
|
+
async getAll(namespace = "default", includeExpired = false) {
|
|
233
|
+
if (this.closed)
|
|
234
|
+
return [];
|
|
235
|
+
await this.initialize();
|
|
236
|
+
const now = Date.now();
|
|
237
|
+
let entries;
|
|
238
|
+
if (includeExpired) {
|
|
239
|
+
entries = this.db.all("SELECT url FROM urls WHERE namespace = ?", namespace);
|
|
240
|
+
} else {
|
|
241
|
+
entries = this.db.all("SELECT url FROM urls WHERE namespace = ? AND expiresAt >= ?", namespace, now);
|
|
242
|
+
}
|
|
243
|
+
return entries.map((e) => e.url);
|
|
244
|
+
}
|
|
245
|
+
async stats(namespace) {
|
|
246
|
+
if (this.closed)
|
|
247
|
+
return { total: 0, expired: 0, namespaces: 0 };
|
|
248
|
+
await this.initialize();
|
|
249
|
+
const now = Date.now();
|
|
250
|
+
let total;
|
|
251
|
+
let expired;
|
|
252
|
+
if (namespace) {
|
|
253
|
+
total = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE namespace = ?", namespace);
|
|
254
|
+
expired = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE namespace = ? AND expiresAt < ?", namespace, now);
|
|
255
|
+
} else {
|
|
256
|
+
total = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
257
|
+
expired = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE expiresAt < ?", now);
|
|
258
|
+
}
|
|
259
|
+
const namespaceCount = this.db.get("SELECT COUNT(DISTINCT namespace) as cnt FROM urls");
|
|
260
|
+
return {
|
|
261
|
+
total: total?.cnt || 0,
|
|
262
|
+
expired: expired?.cnt || 0,
|
|
263
|
+
namespaces: namespaceCount?.cnt || 0
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
async close() {
|
|
267
|
+
if (this.closed)
|
|
268
|
+
return;
|
|
269
|
+
this.closed = true;
|
|
270
|
+
await this.initPromise;
|
|
271
|
+
if (this.db) {
|
|
272
|
+
try {
|
|
273
|
+
this.db.close();
|
|
274
|
+
} catch {}
|
|
275
|
+
this.db = null;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
get isClosed() {
|
|
279
|
+
return this.closed;
|
|
280
|
+
}
|
|
281
|
+
get path() {
|
|
282
|
+
return this.dbPath;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
exports.UrlStore = UrlStore;
|
|
287
|
+
exports.default = UrlStore;
|
|
288
|
+
module.exports = Object.assign(UrlStore, exports);
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
function detectRuntime() {
|
|
5
|
+
if (typeof globalThis.Bun !== "undefined") {
|
|
6
|
+
return "bun";
|
|
7
|
+
}
|
|
8
|
+
if (typeof globalThis.Deno !== "undefined") {
|
|
9
|
+
return "deno";
|
|
10
|
+
}
|
|
11
|
+
return "node";
|
|
12
|
+
}
|
|
13
|
+
async function createDatabase(dbPath) {
|
|
14
|
+
const runtime = detectRuntime();
|
|
15
|
+
if (runtime === "bun") {
|
|
16
|
+
const { Database } = await import("bun:sqlite");
|
|
17
|
+
const db = new Database(dbPath);
|
|
18
|
+
return {
|
|
19
|
+
run: (sql, ...params) => db.run(sql, ...params),
|
|
20
|
+
get: (sql, ...params) => db.query(sql).get(...params),
|
|
21
|
+
all: (sql, ...params) => db.query(sql).all(...params),
|
|
22
|
+
close: () => db.close()
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
if (runtime === "deno") {
|
|
26
|
+
try {
|
|
27
|
+
const { Database } = await import("node:sqlite");
|
|
28
|
+
const db = new Database(dbPath);
|
|
29
|
+
return {
|
|
30
|
+
run: (sql, ...params) => db.exec(sql, params),
|
|
31
|
+
get: (sql, ...params) => {
|
|
32
|
+
const stmt = db.prepare(sql);
|
|
33
|
+
return stmt.get(...params);
|
|
34
|
+
},
|
|
35
|
+
all: (sql, ...params) => {
|
|
36
|
+
const stmt = db.prepare(sql);
|
|
37
|
+
return stmt.all(...params);
|
|
38
|
+
},
|
|
39
|
+
close: () => db.close()
|
|
40
|
+
};
|
|
41
|
+
} catch {
|
|
42
|
+
throw new Error("Deno SQLite support requires Node.js compatibility mode");
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
46
|
+
const db = new DatabaseSync(dbPath);
|
|
47
|
+
return {
|
|
48
|
+
run: (sql, ...params) => db.exec(sql),
|
|
49
|
+
get: (sql, ...params) => {
|
|
50
|
+
const stmt = db.prepare(sql);
|
|
51
|
+
return stmt.get(...params);
|
|
52
|
+
},
|
|
53
|
+
all: (sql, ...params) => {
|
|
54
|
+
const stmt = db.prepare(sql);
|
|
55
|
+
return stmt.all(...params);
|
|
56
|
+
},
|
|
57
|
+
close: () => db.close()
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export class UrlStore {
|
|
62
|
+
db = null;
|
|
63
|
+
options;
|
|
64
|
+
storeDir;
|
|
65
|
+
dbPath;
|
|
66
|
+
closed = false;
|
|
67
|
+
initPromise = null;
|
|
68
|
+
constructor(options = {}) {
|
|
69
|
+
this.options = {
|
|
70
|
+
storeDir: options.storeDir || "./url-store",
|
|
71
|
+
dbFileName: options.dbFileName || "urls.db",
|
|
72
|
+
ttl: options.ttl || 604800000,
|
|
73
|
+
maxUrls: options.maxUrls ?? 0,
|
|
74
|
+
hashUrls: options.hashUrls ?? false
|
|
75
|
+
};
|
|
76
|
+
this.storeDir = path.resolve(this.options.storeDir);
|
|
77
|
+
this.dbPath = path.join(this.storeDir, this.options.dbFileName);
|
|
78
|
+
if (!fs.existsSync(this.storeDir)) {
|
|
79
|
+
fs.mkdirSync(this.storeDir, { recursive: true });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
static async create(options = {}) {
|
|
83
|
+
const store = new UrlStore(options);
|
|
84
|
+
await store.initialize();
|
|
85
|
+
return store;
|
|
86
|
+
}
|
|
87
|
+
async initialize() {
|
|
88
|
+
if (this.initPromise)
|
|
89
|
+
return this.initPromise;
|
|
90
|
+
this.initPromise = (async () => {
|
|
91
|
+
this.db = await createDatabase(this.dbPath);
|
|
92
|
+
this.db.run(`
|
|
93
|
+
CREATE TABLE IF NOT EXISTS urls (
|
|
94
|
+
url TEXT PRIMARY KEY,
|
|
95
|
+
visitedAt INTEGER NOT NULL,
|
|
96
|
+
expiresAt INTEGER NOT NULL,
|
|
97
|
+
namespace TEXT DEFAULT 'default',
|
|
98
|
+
metadata TEXT
|
|
99
|
+
)
|
|
100
|
+
`);
|
|
101
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_expires ON urls(expiresAt)");
|
|
102
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_namespace ON urls(namespace)");
|
|
103
|
+
this.db.run("CREATE INDEX IF NOT EXISTS idx_visited ON urls(visitedAt)");
|
|
104
|
+
})();
|
|
105
|
+
return this.initPromise;
|
|
106
|
+
}
|
|
107
|
+
getUrlKey(url) {
|
|
108
|
+
if (this.options.hashUrls) {
|
|
109
|
+
return createHash("sha256").update(url).digest("hex");
|
|
110
|
+
}
|
|
111
|
+
return url;
|
|
112
|
+
}
|
|
113
|
+
async set(url, namespace = "default", metadata, ttl) {
|
|
114
|
+
if (this.closed)
|
|
115
|
+
throw new Error("UrlStore is closed");
|
|
116
|
+
await this.initialize();
|
|
117
|
+
const key = this.getUrlKey(url);
|
|
118
|
+
const now = Date.now();
|
|
119
|
+
const expiresAt = now + (ttl ?? this.options.ttl);
|
|
120
|
+
const metaStr = metadata ? JSON.stringify(metadata) : null;
|
|
121
|
+
this.db.run(`
|
|
122
|
+
INSERT OR REPLACE INTO urls (url, visitedAt, expiresAt, namespace, metadata)
|
|
123
|
+
VALUES (?, ?, ?, ?, ?)
|
|
124
|
+
`, key, now, expiresAt, namespace, metaStr);
|
|
125
|
+
if (this.options.maxUrls > 0) {
|
|
126
|
+
const count = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
127
|
+
if (count && count.cnt > this.options.maxUrls) {
|
|
128
|
+
const excess = count.cnt - this.options.maxUrls;
|
|
129
|
+
this.db.run(`
|
|
130
|
+
DELETE FROM urls WHERE url IN (
|
|
131
|
+
SELECT url FROM urls ORDER BY visitedAt ASC LIMIT ?
|
|
132
|
+
)
|
|
133
|
+
`, excess);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
async has(url, namespace) {
|
|
138
|
+
if (this.closed)
|
|
139
|
+
return false;
|
|
140
|
+
await this.initialize();
|
|
141
|
+
const key = this.getUrlKey(url);
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
let entry;
|
|
144
|
+
if (namespace) {
|
|
145
|
+
entry = this.db.get("SELECT url, expiresAt FROM urls WHERE url = ? AND namespace = ?", key, namespace);
|
|
146
|
+
} else {
|
|
147
|
+
entry = this.db.get("SELECT url, expiresAt FROM urls WHERE url = ?", key);
|
|
148
|
+
}
|
|
149
|
+
if (!entry)
|
|
150
|
+
return false;
|
|
151
|
+
return entry.expiresAt >= now;
|
|
152
|
+
}
|
|
153
|
+
async hasMany(urls, namespace) {
|
|
154
|
+
if (this.closed)
|
|
155
|
+
return new Set;
|
|
156
|
+
await this.initialize();
|
|
157
|
+
const result = new Set;
|
|
158
|
+
const now = Date.now();
|
|
159
|
+
const batchSize = 100;
|
|
160
|
+
for (let i = 0;i < urls.length; i += batchSize) {
|
|
161
|
+
const batch = urls.slice(i, i + batchSize);
|
|
162
|
+
const keys = batch.map((u) => this.getUrlKey(u));
|
|
163
|
+
const placeholders = keys.map(() => "?").join(",");
|
|
164
|
+
let entries;
|
|
165
|
+
if (namespace) {
|
|
166
|
+
entries = this.db.all(`SELECT url, expiresAt FROM urls WHERE url IN (${placeholders}) AND namespace = ?`, ...keys, namespace);
|
|
167
|
+
} else {
|
|
168
|
+
entries = this.db.all(`SELECT url, expiresAt FROM urls WHERE url IN (${placeholders})`, ...keys);
|
|
169
|
+
}
|
|
170
|
+
for (const entry of entries) {
|
|
171
|
+
if (entry.expiresAt >= now) {
|
|
172
|
+
const idx = keys.indexOf(entry.url);
|
|
173
|
+
if (idx !== -1) {
|
|
174
|
+
result.add(batch[idx]);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
return result;
|
|
180
|
+
}
|
|
181
|
+
async getMetadata(url, namespace) {
|
|
182
|
+
if (this.closed)
|
|
183
|
+
return null;
|
|
184
|
+
await this.initialize();
|
|
185
|
+
const key = this.getUrlKey(url);
|
|
186
|
+
let entry;
|
|
187
|
+
if (namespace) {
|
|
188
|
+
entry = this.db.get("SELECT metadata FROM urls WHERE url = ? AND namespace = ?", key, namespace);
|
|
189
|
+
} else {
|
|
190
|
+
entry = this.db.get("SELECT metadata FROM urls WHERE url = ?", key);
|
|
191
|
+
}
|
|
192
|
+
if (!entry?.metadata)
|
|
193
|
+
return null;
|
|
194
|
+
try {
|
|
195
|
+
return JSON.parse(entry.metadata);
|
|
196
|
+
} catch {
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
async delete(url, namespace) {
|
|
201
|
+
if (this.closed)
|
|
202
|
+
return false;
|
|
203
|
+
await this.initialize();
|
|
204
|
+
const key = this.getUrlKey(url);
|
|
205
|
+
if (namespace) {
|
|
206
|
+
this.db.run("DELETE FROM urls WHERE url = ? AND namespace = ?", key, namespace);
|
|
207
|
+
} else {
|
|
208
|
+
this.db.run("DELETE FROM urls WHERE url = ?", key);
|
|
209
|
+
}
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
212
|
+
async clear(namespace) {
|
|
213
|
+
if (this.closed)
|
|
214
|
+
return;
|
|
215
|
+
await this.initialize();
|
|
216
|
+
if (namespace) {
|
|
217
|
+
this.db.run("DELETE FROM urls WHERE namespace = ?", namespace);
|
|
218
|
+
} else {
|
|
219
|
+
this.db.run("DELETE FROM urls");
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
async cleanup() {
|
|
223
|
+
if (this.closed)
|
|
224
|
+
return 0;
|
|
225
|
+
await this.initialize();
|
|
226
|
+
const now = Date.now();
|
|
227
|
+
const countBefore = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
228
|
+
this.db.run("DELETE FROM urls WHERE expiresAt < ?", now);
|
|
229
|
+
const countAfter = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
230
|
+
return (countBefore?.cnt || 0) - (countAfter?.cnt || 0);
|
|
231
|
+
}
|
|
232
|
+
async getAll(namespace = "default", includeExpired = false) {
|
|
233
|
+
if (this.closed)
|
|
234
|
+
return [];
|
|
235
|
+
await this.initialize();
|
|
236
|
+
const now = Date.now();
|
|
237
|
+
let entries;
|
|
238
|
+
if (includeExpired) {
|
|
239
|
+
entries = this.db.all("SELECT url FROM urls WHERE namespace = ?", namespace);
|
|
240
|
+
} else {
|
|
241
|
+
entries = this.db.all("SELECT url FROM urls WHERE namespace = ? AND expiresAt >= ?", namespace, now);
|
|
242
|
+
}
|
|
243
|
+
return entries.map((e) => e.url);
|
|
244
|
+
}
|
|
245
|
+
async stats(namespace) {
|
|
246
|
+
if (this.closed)
|
|
247
|
+
return { total: 0, expired: 0, namespaces: 0 };
|
|
248
|
+
await this.initialize();
|
|
249
|
+
const now = Date.now();
|
|
250
|
+
let total;
|
|
251
|
+
let expired;
|
|
252
|
+
if (namespace) {
|
|
253
|
+
total = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE namespace = ?", namespace);
|
|
254
|
+
expired = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE namespace = ? AND expiresAt < ?", namespace, now);
|
|
255
|
+
} else {
|
|
256
|
+
total = this.db.get("SELECT COUNT(*) as cnt FROM urls");
|
|
257
|
+
expired = this.db.get("SELECT COUNT(*) as cnt FROM urls WHERE expiresAt < ?", now);
|
|
258
|
+
}
|
|
259
|
+
const namespaceCount = this.db.get("SELECT COUNT(DISTINCT namespace) as cnt FROM urls");
|
|
260
|
+
return {
|
|
261
|
+
total: total?.cnt || 0,
|
|
262
|
+
expired: expired?.cnt || 0,
|
|
263
|
+
namespaces: namespaceCount?.cnt || 0
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
async close() {
|
|
267
|
+
if (this.closed)
|
|
268
|
+
return;
|
|
269
|
+
this.closed = true;
|
|
270
|
+
await this.initPromise;
|
|
271
|
+
if (this.db) {
|
|
272
|
+
try {
|
|
273
|
+
this.db.close();
|
|
274
|
+
} catch {}
|
|
275
|
+
this.db = null;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
get isClosed() {
|
|
279
|
+
return this.closed;
|
|
280
|
+
}
|
|
281
|
+
get path() {
|
|
282
|
+
return this.dbPath;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
export default UrlStore;
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
function createDefaultHooks() {
|
|
2
|
+
return {
|
|
3
|
+
init: [],
|
|
4
|
+
beforeRequest: [],
|
|
5
|
+
beforeRedirect: [],
|
|
6
|
+
beforeRetry: [],
|
|
7
|
+
afterResponse: [],
|
|
8
|
+
beforeError: [],
|
|
9
|
+
beforeCache: [],
|
|
10
|
+
afterHeaders: [],
|
|
11
|
+
afterParse: [],
|
|
12
|
+
beforeCookie: [],
|
|
13
|
+
afterCookie: [],
|
|
14
|
+
onSocket: [],
|
|
15
|
+
onDns: [],
|
|
16
|
+
onTls: [],
|
|
17
|
+
onTimeout: [],
|
|
18
|
+
onAbort: []
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function mergeHooks(base, overrides) {
|
|
22
|
+
if (!overrides)
|
|
23
|
+
return { ...base };
|
|
24
|
+
return {
|
|
25
|
+
init: [...base.init, ...overrides.init || []],
|
|
26
|
+
beforeRequest: [...base.beforeRequest, ...overrides.beforeRequest || []],
|
|
27
|
+
beforeRedirect: [...base.beforeRedirect, ...overrides.beforeRedirect || []],
|
|
28
|
+
beforeRetry: [...base.beforeRetry, ...overrides.beforeRetry || []],
|
|
29
|
+
afterResponse: [...base.afterResponse, ...overrides.afterResponse || []],
|
|
30
|
+
beforeError: [...base.beforeError, ...overrides.beforeError || []],
|
|
31
|
+
beforeCache: [...base.beforeCache, ...overrides.beforeCache || []],
|
|
32
|
+
afterHeaders: [...base.afterHeaders, ...overrides.afterHeaders || []],
|
|
33
|
+
afterParse: [...base.afterParse, ...overrides.afterParse || []],
|
|
34
|
+
beforeCookie: [...base.beforeCookie, ...overrides.beforeCookie || []],
|
|
35
|
+
afterCookie: [...base.afterCookie, ...overrides.afterCookie || []],
|
|
36
|
+
onSocket: [...base.onSocket, ...overrides.onSocket || []],
|
|
37
|
+
onDns: [...base.onDns, ...overrides.onDns || []],
|
|
38
|
+
onTls: [...base.onTls, ...overrides.onTls || []],
|
|
39
|
+
onTimeout: [...base.onTimeout, ...overrides.onTimeout || []],
|
|
40
|
+
onAbort: [...base.onAbort, ...overrides.onAbort || []]
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function serializeHooks(hooks) {
|
|
44
|
+
const result = {};
|
|
45
|
+
let hasAnyHooks = false;
|
|
46
|
+
for (const [key, value] of Object.entries(hooks)) {
|
|
47
|
+
if (Array.isArray(value) && value.length > 0) {
|
|
48
|
+
result[key] = value;
|
|
49
|
+
hasAnyHooks = true;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return hasAnyHooks ? result : null;
|
|
53
|
+
}
|
|
54
|
+
async function runVoidHooks(hooks, arg, ...args) {
|
|
55
|
+
for (const hook of hooks) {
|
|
56
|
+
await hook(arg, ...args);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function runVoidHooksSync(hooks, arg, ...args) {
|
|
60
|
+
for (const hook of hooks) {
|
|
61
|
+
hook(arg, ...args);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async function runTransformHooks(hooks, initial, ...args) {
|
|
65
|
+
let result = initial;
|
|
66
|
+
for (const hook of hooks) {
|
|
67
|
+
result = await hook(result, ...args);
|
|
68
|
+
}
|
|
69
|
+
return result;
|
|
70
|
+
}
|
|
71
|
+
function runBooleanHooks(hooks, arg, ...args) {
|
|
72
|
+
for (const hook of hooks) {
|
|
73
|
+
const result = hook(arg, ...args);
|
|
74
|
+
if (result === false) {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
async function runBooleanHooksAsync(hooks, arg, ...args) {
|
|
81
|
+
for (const hook of hooks) {
|
|
82
|
+
const result = await hook(arg, ...args);
|
|
83
|
+
if (result === false) {
|
|
84
|
+
return false;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
async function runEarlyReturnHooks(hooks, arg, ...args) {
|
|
90
|
+
for (const hook of hooks) {
|
|
91
|
+
const result = await hook(arg, ...args);
|
|
92
|
+
if (result !== undefined && result !== null) {
|
|
93
|
+
return result;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
function runEventHooks(hooks, arg, ...args) {
|
|
99
|
+
for (const hook of hooks) {
|
|
100
|
+
try {
|
|
101
|
+
hook(arg, ...args);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
console.error("[Rezo] Event hook error:", error);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
function createBeforeRequestContext(retryCount, isRedirect, redirectCount, startTime) {
|
|
108
|
+
return {
|
|
109
|
+
retryCount,
|
|
110
|
+
isRedirect,
|
|
111
|
+
redirectCount,
|
|
112
|
+
startTime
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
function createAfterResponseContext(retryCount, retryFn) {
|
|
116
|
+
return {
|
|
117
|
+
retryCount,
|
|
118
|
+
retryWithMergedOptions: retryFn
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
exports.createDefaultHooks = createDefaultHooks;
|
|
123
|
+
exports.mergeHooks = mergeHooks;
|
|
124
|
+
exports.serializeHooks = serializeHooks;
|
|
125
|
+
exports.runVoidHooks = runVoidHooks;
|
|
126
|
+
exports.runVoidHooksSync = runVoidHooksSync;
|
|
127
|
+
exports.runTransformHooks = runTransformHooks;
|
|
128
|
+
exports.runBooleanHooks = runBooleanHooks;
|
|
129
|
+
exports.runBooleanHooksAsync = runBooleanHooksAsync;
|
|
130
|
+
exports.runEarlyReturnHooks = runEarlyReturnHooks;
|
|
131
|
+
exports.runEventHooks = runEventHooks;
|
|
132
|
+
exports.createBeforeRequestContext = createBeforeRequestContext;
|
|
133
|
+
exports.createAfterResponseContext = createAfterResponseContext;
|