rezo 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +1507 -0
- package/assets/icon.svg +37 -0
- package/assets/logo-dark.svg +47 -0
- package/assets/logo.svg +58 -0
- package/dist/adapters/curl.cjs +1034 -0
- package/dist/adapters/curl.js +1031 -0
- package/dist/adapters/entries/curl.cjs +4 -0
- package/dist/adapters/entries/curl.d.ts +2136 -0
- package/dist/adapters/entries/curl.js +2 -0
- package/dist/adapters/entries/fetch.cjs +2 -0
- package/dist/adapters/entries/fetch.d.ts +2127 -0
- package/dist/adapters/entries/fetch.js +1 -0
- package/dist/adapters/entries/http.cjs +2 -0
- package/dist/adapters/entries/http.d.ts +2126 -0
- package/dist/adapters/entries/http.js +1 -0
- package/dist/adapters/entries/http2.cjs +4 -0
- package/dist/adapters/entries/http2.d.ts +2136 -0
- package/dist/adapters/entries/http2.js +2 -0
- package/dist/adapters/entries/react-native.cjs +2 -0
- package/dist/adapters/entries/react-native.d.ts +2126 -0
- package/dist/adapters/entries/react-native.js +1 -0
- package/dist/adapters/entries/xhr.cjs +2 -0
- package/dist/adapters/entries/xhr.d.ts +2127 -0
- package/dist/adapters/entries/xhr.js +1 -0
- package/dist/adapters/fetch.cjs +740 -0
- package/dist/adapters/fetch.js +739 -0
- package/dist/adapters/http.cjs +1153 -0
- package/dist/adapters/http.js +1151 -0
- package/dist/adapters/http2.cjs +957 -0
- package/dist/adapters/http2.js +956 -0
- package/dist/adapters/index.cjs +6 -0
- package/dist/adapters/index.js +7 -0
- package/dist/adapters/picker.cjs +342 -0
- package/dist/adapters/picker.js +331 -0
- package/dist/adapters/react-native.cjs +545 -0
- package/dist/adapters/react-native.js +544 -0
- package/dist/adapters/xhr.cjs +622 -0
- package/dist/adapters/xhr.js +621 -0
- package/dist/cache/dns-cache.cjs +118 -0
- package/dist/cache/dns-cache.js +113 -0
- package/dist/cache/file-cacher.cjs +264 -0
- package/dist/cache/file-cacher.js +261 -0
- package/dist/cache/index.cjs +13 -0
- package/dist/cache/index.js +5 -0
- package/dist/cache/lru-cache.cjs +96 -0
- package/dist/cache/lru-cache.js +93 -0
- package/dist/cache/response-cache.cjs +314 -0
- package/dist/cache/response-cache.js +310 -0
- package/dist/cache/url-store.cjs +288 -0
- package/dist/cache/url-store.js +285 -0
- package/dist/core/hooks.cjs +133 -0
- package/dist/core/hooks.js +120 -0
- package/dist/core/rezo.cjs +464 -0
- package/dist/core/rezo.js +458 -0
- package/dist/crawler.d.ts +6255 -0
- package/dist/dom/index.cjs +1 -0
- package/dist/dom/index.d.ts +23 -0
- package/dist/dom/index.js +1 -0
- package/dist/entries/crawler.cjs +5 -0
- package/dist/entries/crawler.js +2 -0
- package/dist/errors/rezo-error.cjs +722 -0
- package/dist/errors/rezo-error.js +716 -0
- package/dist/index.cjs +34 -0
- package/dist/index.d.ts +3335 -0
- package/dist/index.js +26 -0
- package/dist/platform/browser.cjs +9 -0
- package/dist/platform/browser.d.ts +3203 -0
- package/dist/platform/browser.js +7 -0
- package/dist/platform/bun.cjs +9 -0
- package/dist/platform/bun.d.ts +3203 -0
- package/dist/platform/bun.js +7 -0
- package/dist/platform/deno.cjs +9 -0
- package/dist/platform/deno.d.ts +3203 -0
- package/dist/platform/deno.js +7 -0
- package/dist/platform/node.cjs +9 -0
- package/dist/platform/node.d.ts +3203 -0
- package/dist/platform/node.js +7 -0
- package/dist/platform/react-native.cjs +9 -0
- package/dist/platform/react-native.d.ts +3203 -0
- package/dist/platform/react-native.js +7 -0
- package/dist/platform/worker.cjs +9 -0
- package/dist/platform/worker.d.ts +3203 -0
- package/dist/platform/worker.js +7 -0
- package/dist/plugin/addon/decodo/index.cjs +1 -0
- package/dist/plugin/addon/decodo/index.js +1 -0
- package/dist/plugin/addon/decodo/options.cjs +1 -0
- package/dist/plugin/addon/decodo/options.js +1 -0
- package/dist/plugin/addon/oxylabs/index.cjs +1 -0
- package/dist/plugin/addon/oxylabs/index.js +1 -0
- package/dist/plugin/addon/oxylabs/options.cjs +1 -0
- package/dist/plugin/addon/oxylabs/options.js +1 -0
- package/dist/plugin/crawler-options.cjs +1 -0
- package/dist/plugin/crawler-options.js +1 -0
- package/dist/plugin/crawler.cjs +519 -0
- package/dist/plugin/crawler.js +517 -0
- package/dist/plugin/index.cjs +36 -0
- package/dist/plugin/index.js +32 -0
- package/dist/proxy/index.cjs +142 -0
- package/dist/proxy/index.js +139 -0
- package/dist/responses/buildError.cjs +452 -0
- package/dist/responses/buildError.js +441 -0
- package/dist/responses/buildResponse.cjs +365 -0
- package/dist/responses/buildResponse.js +361 -0
- package/dist/responses/download.cjs +54 -0
- package/dist/responses/download.js +52 -0
- package/dist/responses/stream.cjs +60 -0
- package/dist/responses/stream.js +58 -0
- package/dist/responses/upload.cjs +54 -0
- package/dist/responses/upload.js +52 -0
- package/dist/types/cookies.cjs +394 -0
- package/dist/types/cookies.js +391 -0
- package/dist/types/download.cjs +10 -0
- package/dist/types/download.js +10 -0
- package/dist/types/rezo-request.cjs +131 -0
- package/dist/types/rezo-request.js +131 -0
- package/dist/utils/agent-merger.cjs +111 -0
- package/dist/utils/agent-merger.js +108 -0
- package/dist/utils/compression.cjs +84 -0
- package/dist/utils/compression.js +82 -0
- package/dist/utils/cookies.cjs +514 -0
- package/dist/utils/cookies.js +511 -0
- package/dist/utils/data-operations.cjs +75 -0
- package/dist/utils/data-operations.js +73 -0
- package/dist/utils/form-data.cjs +164 -0
- package/dist/utils/form-data.js +161 -0
- package/dist/utils/headers.cjs +162 -0
- package/dist/utils/headers.js +161 -0
- package/dist/utils/http-config.cjs +723 -0
- package/dist/utils/http-config.js +718 -0
- package/dist/utils/index.cjs +8 -0
- package/dist/utils/index.js +8 -0
- package/dist/utils/tools.cjs +18 -0
- package/dist/utils/tools.js +15 -0
- package/package.json +172 -0
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
const { LRUCache } = require('./lru-cache.cjs');
|
|
2
|
+
const fs = require("fs");
|
|
3
|
+
const path = require("path");
|
|
4
|
+
const fsPromises = fs.promises;
|
|
5
|
+
const DEFAULT_TTL = 3000000;
|
|
6
|
+
const DEFAULT_MAX_ENTRIES = 500;
|
|
7
|
+
const DEFAULT_METHODS = ["GET", "HEAD"];
|
|
8
|
+
|
|
9
|
+
class ResponseCache {
|
|
10
|
+
memoryCache;
|
|
11
|
+
config;
|
|
12
|
+
persistenceEnabled;
|
|
13
|
+
initialized = false;
|
|
14
|
+
constructor(options = true) {
|
|
15
|
+
const config = options === true ? {} : options === false ? { enable: false } : options;
|
|
16
|
+
this.config = {
|
|
17
|
+
enable: config.enable !== false,
|
|
18
|
+
cacheDir: config.cacheDir,
|
|
19
|
+
networkCheck: config.networkCheck ?? false,
|
|
20
|
+
ttl: config.ttl ?? DEFAULT_TTL,
|
|
21
|
+
maxEntries: config.maxEntries ?? DEFAULT_MAX_ENTRIES,
|
|
22
|
+
methods: config.methods ?? DEFAULT_METHODS,
|
|
23
|
+
respectHeaders: config.respectHeaders ?? true
|
|
24
|
+
};
|
|
25
|
+
this.persistenceEnabled = !!this.config.cacheDir;
|
|
26
|
+
this.memoryCache = new LRUCache({
|
|
27
|
+
maxEntries: this.config.maxEntries,
|
|
28
|
+
ttl: this.config.ttl,
|
|
29
|
+
onEvict: this.persistenceEnabled ? (key, value) => this.persistToDisk(key, value) : undefined
|
|
30
|
+
});
|
|
31
|
+
if (this.persistenceEnabled) {
|
|
32
|
+
this.initializePersistence();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
initializePersistence() {
|
|
36
|
+
if (!this.config.cacheDir || this.initialized)
|
|
37
|
+
return;
|
|
38
|
+
this.initializePersistenceAsync().catch(() => {
|
|
39
|
+
this.persistenceEnabled = false;
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
async initializePersistenceAsync() {
|
|
43
|
+
if (!this.config.cacheDir)
|
|
44
|
+
return;
|
|
45
|
+
try {
|
|
46
|
+
await fsPromises.mkdir(this.config.cacheDir, { recursive: true });
|
|
47
|
+
await this.loadFromDiskAsync();
|
|
48
|
+
this.initialized = true;
|
|
49
|
+
} catch {
|
|
50
|
+
this.persistenceEnabled = false;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
getCacheFilePath(key) {
|
|
54
|
+
const safeKey = Buffer.from(key).toString("base64url");
|
|
55
|
+
return path.join(this.config.cacheDir, `${safeKey}.json`);
|
|
56
|
+
}
|
|
57
|
+
persistToDisk(key, entry) {
|
|
58
|
+
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
59
|
+
return;
|
|
60
|
+
const filePath = this.getCacheFilePath(key);
|
|
61
|
+
fsPromises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
|
|
62
|
+
}
|
|
63
|
+
async loadFromDiskAsync() {
|
|
64
|
+
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
65
|
+
return;
|
|
66
|
+
try {
|
|
67
|
+
const files = await fsPromises.readdir(this.config.cacheDir);
|
|
68
|
+
const now = Date.now();
|
|
69
|
+
for (const file of files) {
|
|
70
|
+
if (!file.endsWith(".json"))
|
|
71
|
+
continue;
|
|
72
|
+
try {
|
|
73
|
+
const filePath = path.join(this.config.cacheDir, file);
|
|
74
|
+
const content = await fsPromises.readFile(filePath, "utf-8");
|
|
75
|
+
const entry = JSON.parse(content);
|
|
76
|
+
if (entry.timestamp + entry.ttl > now) {
|
|
77
|
+
const key = Buffer.from(file.replace(".json", ""), "base64url").toString("utf-8");
|
|
78
|
+
const remainingTTL = entry.timestamp + entry.ttl - now;
|
|
79
|
+
this.memoryCache.set(key, entry, remainingTTL);
|
|
80
|
+
} else {
|
|
81
|
+
fsPromises.unlink(filePath).catch(() => {});
|
|
82
|
+
}
|
|
83
|
+
} catch {}
|
|
84
|
+
}
|
|
85
|
+
} catch {}
|
|
86
|
+
}
|
|
87
|
+
generateKey(method, url, headers) {
|
|
88
|
+
let key = `${method.toUpperCase()}:${url}`;
|
|
89
|
+
if (headers) {
|
|
90
|
+
const accept = headers["accept"] || headers["Accept"];
|
|
91
|
+
const acceptEncoding = headers["accept-encoding"] || headers["Accept-Encoding"];
|
|
92
|
+
if (accept)
|
|
93
|
+
key += `:accept=${accept}`;
|
|
94
|
+
if (acceptEncoding)
|
|
95
|
+
key += `:encoding=${acceptEncoding}`;
|
|
96
|
+
}
|
|
97
|
+
return key;
|
|
98
|
+
}
|
|
99
|
+
parseCacheControl(headers) {
|
|
100
|
+
const cacheControl = headers["cache-control"] || headers["Cache-Control"] || "";
|
|
101
|
+
const result = {};
|
|
102
|
+
if (cacheControl.includes("no-store"))
|
|
103
|
+
result.noStore = true;
|
|
104
|
+
if (cacheControl.includes("no-cache"))
|
|
105
|
+
result.noCache = true;
|
|
106
|
+
if (cacheControl.includes("must-revalidate"))
|
|
107
|
+
result.mustRevalidate = true;
|
|
108
|
+
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
|
|
109
|
+
if (maxAgeMatch) {
|
|
110
|
+
result.maxAge = parseInt(maxAgeMatch[1], 10) * 1000;
|
|
111
|
+
}
|
|
112
|
+
const sMaxAgeMatch = cacheControl.match(/s-maxage=(\d+)/);
|
|
113
|
+
if (sMaxAgeMatch) {
|
|
114
|
+
result.maxAge = parseInt(sMaxAgeMatch[1], 10) * 1000;
|
|
115
|
+
}
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
isCacheable(method, status, headers) {
|
|
119
|
+
if (!this.config.enable)
|
|
120
|
+
return false;
|
|
121
|
+
if (!this.config.methods.includes(method.toUpperCase()))
|
|
122
|
+
return false;
|
|
123
|
+
if (status < 200 || status >= 300)
|
|
124
|
+
return false;
|
|
125
|
+
if (this.config.respectHeaders && headers) {
|
|
126
|
+
const cacheControl = this.parseCacheControl(headers);
|
|
127
|
+
if (cacheControl.noStore)
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
get(method, url, headers) {
|
|
133
|
+
if (!this.config.enable)
|
|
134
|
+
return;
|
|
135
|
+
const key = this.generateKey(method, url, headers);
|
|
136
|
+
const cached = this.memoryCache.get(key);
|
|
137
|
+
if (!cached) {
|
|
138
|
+
if (this.persistenceEnabled) {
|
|
139
|
+
return this.loadSingleFromDisk(key);
|
|
140
|
+
}
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
return cached;
|
|
144
|
+
}
|
|
145
|
+
loadSingleFromDisk(key) {
|
|
146
|
+
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
147
|
+
return;
|
|
148
|
+
try {
|
|
149
|
+
const filePath = this.getCacheFilePath(key);
|
|
150
|
+
if (!fs.existsSync(filePath))
|
|
151
|
+
return;
|
|
152
|
+
const content = fs.readFileSync(filePath, "utf-8");
|
|
153
|
+
const entry = JSON.parse(content);
|
|
154
|
+
const now = Date.now();
|
|
155
|
+
if (entry.timestamp + entry.ttl > now) {
|
|
156
|
+
const remainingTTL = entry.timestamp + entry.ttl - now;
|
|
157
|
+
this.memoryCache.set(key, entry, remainingTTL);
|
|
158
|
+
return entry;
|
|
159
|
+
} else {
|
|
160
|
+
fs.unlinkSync(filePath);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
} catch {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
set(method, url, response, requestHeaders) {
|
|
168
|
+
if (!this.config.enable)
|
|
169
|
+
return;
|
|
170
|
+
const responseHeaders = this.normalizeHeaders(response.headers);
|
|
171
|
+
if (!this.isCacheable(method, response.status, responseHeaders))
|
|
172
|
+
return;
|
|
173
|
+
let ttl = this.config.ttl;
|
|
174
|
+
if (this.config.respectHeaders) {
|
|
175
|
+
const cacheControl = this.parseCacheControl(responseHeaders);
|
|
176
|
+
if (cacheControl.maxAge !== undefined) {
|
|
177
|
+
ttl = cacheControl.maxAge;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const key = this.generateKey(method, url, requestHeaders);
|
|
181
|
+
const cached = {
|
|
182
|
+
status: response.status,
|
|
183
|
+
statusText: response.statusText,
|
|
184
|
+
headers: responseHeaders,
|
|
185
|
+
data: response.data,
|
|
186
|
+
url,
|
|
187
|
+
timestamp: Date.now(),
|
|
188
|
+
ttl,
|
|
189
|
+
etag: responseHeaders["etag"],
|
|
190
|
+
lastModified: responseHeaders["last-modified"]
|
|
191
|
+
};
|
|
192
|
+
this.memoryCache.set(key, cached, ttl);
|
|
193
|
+
if (this.persistenceEnabled) {
|
|
194
|
+
this.persistToDisk(key, cached);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
normalizeHeaders(headers) {
|
|
198
|
+
const result = {};
|
|
199
|
+
if (!headers)
|
|
200
|
+
return result;
|
|
201
|
+
if (headers instanceof Headers) {
|
|
202
|
+
headers.forEach((value, key) => {
|
|
203
|
+
result[key.toLowerCase()] = value;
|
|
204
|
+
});
|
|
205
|
+
} else if (typeof headers === "object") {
|
|
206
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
207
|
+
if (typeof value === "string") {
|
|
208
|
+
result[key.toLowerCase()] = value;
|
|
209
|
+
} else if (Array.isArray(value)) {
|
|
210
|
+
result[key.toLowerCase()] = value.join(", ");
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return result;
|
|
215
|
+
}
|
|
216
|
+
getConditionalHeaders(method, url, requestHeaders) {
|
|
217
|
+
const cached = this.get(method, url, requestHeaders);
|
|
218
|
+
if (!cached)
|
|
219
|
+
return;
|
|
220
|
+
const headers = {};
|
|
221
|
+
if (cached.etag) {
|
|
222
|
+
headers["If-None-Match"] = cached.etag;
|
|
223
|
+
}
|
|
224
|
+
if (cached.lastModified) {
|
|
225
|
+
headers["If-Modified-Since"] = cached.lastModified;
|
|
226
|
+
}
|
|
227
|
+
return Object.keys(headers).length > 0 ? headers : undefined;
|
|
228
|
+
}
|
|
229
|
+
updateRevalidated(method, url, newHeaders, requestHeaders) {
|
|
230
|
+
if (!this.config.enable)
|
|
231
|
+
return;
|
|
232
|
+
const key = this.generateKey(method, url, requestHeaders);
|
|
233
|
+
const cached = this.memoryCache.get(key) || this.loadSingleFromDisk(key);
|
|
234
|
+
if (!cached)
|
|
235
|
+
return;
|
|
236
|
+
const normalizedHeaders = this.normalizeHeaders(newHeaders);
|
|
237
|
+
let ttl = cached.ttl;
|
|
238
|
+
if (this.config.respectHeaders) {
|
|
239
|
+
const cacheControl = this.parseCacheControl(normalizedHeaders);
|
|
240
|
+
if (cacheControl.noStore) {
|
|
241
|
+
this.memoryCache.delete(key);
|
|
242
|
+
if (this.persistenceEnabled) {
|
|
243
|
+
const filePath = this.getCacheFilePath(key);
|
|
244
|
+
fsPromises.unlink(filePath).catch(() => {});
|
|
245
|
+
}
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
if (cacheControl.maxAge !== undefined) {
|
|
249
|
+
ttl = cacheControl.maxAge;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
const updated = {
|
|
253
|
+
...cached,
|
|
254
|
+
timestamp: Date.now(),
|
|
255
|
+
ttl,
|
|
256
|
+
headers: { ...cached.headers, ...normalizedHeaders },
|
|
257
|
+
etag: normalizedHeaders["etag"] || cached.etag,
|
|
258
|
+
lastModified: normalizedHeaders["last-modified"] || cached.lastModified
|
|
259
|
+
};
|
|
260
|
+
this.memoryCache.set(key, updated, ttl);
|
|
261
|
+
if (this.persistenceEnabled) {
|
|
262
|
+
this.persistToDisk(key, updated);
|
|
263
|
+
}
|
|
264
|
+
return updated;
|
|
265
|
+
}
|
|
266
|
+
invalidate(url, method) {
|
|
267
|
+
const methods = method ? [method] : this.config.methods;
|
|
268
|
+
for (const m of methods) {
|
|
269
|
+
const key = this.generateKey(m, url);
|
|
270
|
+
this.memoryCache.delete(key);
|
|
271
|
+
if (this.persistenceEnabled) {
|
|
272
|
+
const filePath = this.getCacheFilePath(key);
|
|
273
|
+
fsPromises.unlink(filePath).catch(() => {});
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
clear() {
|
|
278
|
+
this.memoryCache.clear();
|
|
279
|
+
if (this.persistenceEnabled && this.config.cacheDir) {
|
|
280
|
+
const cacheDir = this.config.cacheDir;
|
|
281
|
+
fsPromises.readdir(cacheDir).then((files) => {
|
|
282
|
+
for (const file of files) {
|
|
283
|
+
if (file.endsWith(".json")) {
|
|
284
|
+
fsPromises.unlink(path.join(cacheDir, file)).catch(() => {});
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}).catch(() => {});
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
get size() {
|
|
291
|
+
return this.memoryCache.size;
|
|
292
|
+
}
|
|
293
|
+
get isEnabled() {
|
|
294
|
+
return this.config.enable;
|
|
295
|
+
}
|
|
296
|
+
get isPersistent() {
|
|
297
|
+
return this.persistenceEnabled;
|
|
298
|
+
}
|
|
299
|
+
getConfig() {
|
|
300
|
+
return { ...this.config };
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
function normalizeResponseCacheConfig(option) {
|
|
304
|
+
if (option === undefined || option === false)
|
|
305
|
+
return;
|
|
306
|
+
if (option === true)
|
|
307
|
+
return { enable: true };
|
|
308
|
+
return option;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
exports.ResponseCache = ResponseCache;
|
|
312
|
+
exports.normalizeResponseCacheConfig = normalizeResponseCacheConfig;
|
|
313
|
+
exports.default = ResponseCache;
|
|
314
|
+
module.exports = Object.assign(ResponseCache, exports);
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
import { LRUCache } from './lru-cache.js';
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import path from "path";
|
|
4
|
+
const fsPromises = fs.promises;
|
|
5
|
+
const DEFAULT_TTL = 3000000;
|
|
6
|
+
const DEFAULT_MAX_ENTRIES = 500;
|
|
7
|
+
const DEFAULT_METHODS = ["GET", "HEAD"];
|
|
8
|
+
|
|
9
|
+
export class ResponseCache {
|
|
10
|
+
memoryCache;
|
|
11
|
+
config;
|
|
12
|
+
persistenceEnabled;
|
|
13
|
+
initialized = false;
|
|
14
|
+
constructor(options = true) {
|
|
15
|
+
const config = options === true ? {} : options === false ? { enable: false } : options;
|
|
16
|
+
this.config = {
|
|
17
|
+
enable: config.enable !== false,
|
|
18
|
+
cacheDir: config.cacheDir,
|
|
19
|
+
networkCheck: config.networkCheck ?? false,
|
|
20
|
+
ttl: config.ttl ?? DEFAULT_TTL,
|
|
21
|
+
maxEntries: config.maxEntries ?? DEFAULT_MAX_ENTRIES,
|
|
22
|
+
methods: config.methods ?? DEFAULT_METHODS,
|
|
23
|
+
respectHeaders: config.respectHeaders ?? true
|
|
24
|
+
};
|
|
25
|
+
this.persistenceEnabled = !!this.config.cacheDir;
|
|
26
|
+
this.memoryCache = new LRUCache({
|
|
27
|
+
maxEntries: this.config.maxEntries,
|
|
28
|
+
ttl: this.config.ttl,
|
|
29
|
+
onEvict: this.persistenceEnabled ? (key, value) => this.persistToDisk(key, value) : undefined
|
|
30
|
+
});
|
|
31
|
+
if (this.persistenceEnabled) {
|
|
32
|
+
this.initializePersistence();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
initializePersistence() {
|
|
36
|
+
if (!this.config.cacheDir || this.initialized)
|
|
37
|
+
return;
|
|
38
|
+
this.initializePersistenceAsync().catch(() => {
|
|
39
|
+
this.persistenceEnabled = false;
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
async initializePersistenceAsync() {
|
|
43
|
+
if (!this.config.cacheDir)
|
|
44
|
+
return;
|
|
45
|
+
try {
|
|
46
|
+
await fsPromises.mkdir(this.config.cacheDir, { recursive: true });
|
|
47
|
+
await this.loadFromDiskAsync();
|
|
48
|
+
this.initialized = true;
|
|
49
|
+
} catch {
|
|
50
|
+
this.persistenceEnabled = false;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
getCacheFilePath(key) {
|
|
54
|
+
const safeKey = Buffer.from(key).toString("base64url");
|
|
55
|
+
return path.join(this.config.cacheDir, `${safeKey}.json`);
|
|
56
|
+
}
|
|
57
|
+
persistToDisk(key, entry) {
|
|
58
|
+
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
59
|
+
return;
|
|
60
|
+
const filePath = this.getCacheFilePath(key);
|
|
61
|
+
fsPromises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
|
|
62
|
+
}
|
|
63
|
+
async loadFromDiskAsync() {
|
|
64
|
+
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
65
|
+
return;
|
|
66
|
+
try {
|
|
67
|
+
const files = await fsPromises.readdir(this.config.cacheDir);
|
|
68
|
+
const now = Date.now();
|
|
69
|
+
for (const file of files) {
|
|
70
|
+
if (!file.endsWith(".json"))
|
|
71
|
+
continue;
|
|
72
|
+
try {
|
|
73
|
+
const filePath = path.join(this.config.cacheDir, file);
|
|
74
|
+
const content = await fsPromises.readFile(filePath, "utf-8");
|
|
75
|
+
const entry = JSON.parse(content);
|
|
76
|
+
if (entry.timestamp + entry.ttl > now) {
|
|
77
|
+
const key = Buffer.from(file.replace(".json", ""), "base64url").toString("utf-8");
|
|
78
|
+
const remainingTTL = entry.timestamp + entry.ttl - now;
|
|
79
|
+
this.memoryCache.set(key, entry, remainingTTL);
|
|
80
|
+
} else {
|
|
81
|
+
fsPromises.unlink(filePath).catch(() => {});
|
|
82
|
+
}
|
|
83
|
+
} catch {}
|
|
84
|
+
}
|
|
85
|
+
} catch {}
|
|
86
|
+
}
|
|
87
|
+
generateKey(method, url, headers) {
|
|
88
|
+
let key = `${method.toUpperCase()}:${url}`;
|
|
89
|
+
if (headers) {
|
|
90
|
+
const accept = headers["accept"] || headers["Accept"];
|
|
91
|
+
const acceptEncoding = headers["accept-encoding"] || headers["Accept-Encoding"];
|
|
92
|
+
if (accept)
|
|
93
|
+
key += `:accept=${accept}`;
|
|
94
|
+
if (acceptEncoding)
|
|
95
|
+
key += `:encoding=${acceptEncoding}`;
|
|
96
|
+
}
|
|
97
|
+
return key;
|
|
98
|
+
}
|
|
99
|
+
parseCacheControl(headers) {
|
|
100
|
+
const cacheControl = headers["cache-control"] || headers["Cache-Control"] || "";
|
|
101
|
+
const result = {};
|
|
102
|
+
if (cacheControl.includes("no-store"))
|
|
103
|
+
result.noStore = true;
|
|
104
|
+
if (cacheControl.includes("no-cache"))
|
|
105
|
+
result.noCache = true;
|
|
106
|
+
if (cacheControl.includes("must-revalidate"))
|
|
107
|
+
result.mustRevalidate = true;
|
|
108
|
+
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
|
|
109
|
+
if (maxAgeMatch) {
|
|
110
|
+
result.maxAge = parseInt(maxAgeMatch[1], 10) * 1000;
|
|
111
|
+
}
|
|
112
|
+
const sMaxAgeMatch = cacheControl.match(/s-maxage=(\d+)/);
|
|
113
|
+
if (sMaxAgeMatch) {
|
|
114
|
+
result.maxAge = parseInt(sMaxAgeMatch[1], 10) * 1000;
|
|
115
|
+
}
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
isCacheable(method, status, headers) {
|
|
119
|
+
if (!this.config.enable)
|
|
120
|
+
return false;
|
|
121
|
+
if (!this.config.methods.includes(method.toUpperCase()))
|
|
122
|
+
return false;
|
|
123
|
+
if (status < 200 || status >= 300)
|
|
124
|
+
return false;
|
|
125
|
+
if (this.config.respectHeaders && headers) {
|
|
126
|
+
const cacheControl = this.parseCacheControl(headers);
|
|
127
|
+
if (cacheControl.noStore)
|
|
128
|
+
return false;
|
|
129
|
+
}
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
get(method, url, headers) {
|
|
133
|
+
if (!this.config.enable)
|
|
134
|
+
return;
|
|
135
|
+
const key = this.generateKey(method, url, headers);
|
|
136
|
+
const cached = this.memoryCache.get(key);
|
|
137
|
+
if (!cached) {
|
|
138
|
+
if (this.persistenceEnabled) {
|
|
139
|
+
return this.loadSingleFromDisk(key);
|
|
140
|
+
}
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
return cached;
|
|
144
|
+
}
|
|
145
|
+
loadSingleFromDisk(key) {
|
|
146
|
+
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
147
|
+
return;
|
|
148
|
+
try {
|
|
149
|
+
const filePath = this.getCacheFilePath(key);
|
|
150
|
+
if (!fs.existsSync(filePath))
|
|
151
|
+
return;
|
|
152
|
+
const content = fs.readFileSync(filePath, "utf-8");
|
|
153
|
+
const entry = JSON.parse(content);
|
|
154
|
+
const now = Date.now();
|
|
155
|
+
if (entry.timestamp + entry.ttl > now) {
|
|
156
|
+
const remainingTTL = entry.timestamp + entry.ttl - now;
|
|
157
|
+
this.memoryCache.set(key, entry, remainingTTL);
|
|
158
|
+
return entry;
|
|
159
|
+
} else {
|
|
160
|
+
fs.unlinkSync(filePath);
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
} catch {
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
set(method, url, response, requestHeaders) {
|
|
168
|
+
if (!this.config.enable)
|
|
169
|
+
return;
|
|
170
|
+
const responseHeaders = this.normalizeHeaders(response.headers);
|
|
171
|
+
if (!this.isCacheable(method, response.status, responseHeaders))
|
|
172
|
+
return;
|
|
173
|
+
let ttl = this.config.ttl;
|
|
174
|
+
if (this.config.respectHeaders) {
|
|
175
|
+
const cacheControl = this.parseCacheControl(responseHeaders);
|
|
176
|
+
if (cacheControl.maxAge !== undefined) {
|
|
177
|
+
ttl = cacheControl.maxAge;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const key = this.generateKey(method, url, requestHeaders);
|
|
181
|
+
const cached = {
|
|
182
|
+
status: response.status,
|
|
183
|
+
statusText: response.statusText,
|
|
184
|
+
headers: responseHeaders,
|
|
185
|
+
data: response.data,
|
|
186
|
+
url,
|
|
187
|
+
timestamp: Date.now(),
|
|
188
|
+
ttl,
|
|
189
|
+
etag: responseHeaders["etag"],
|
|
190
|
+
lastModified: responseHeaders["last-modified"]
|
|
191
|
+
};
|
|
192
|
+
this.memoryCache.set(key, cached, ttl);
|
|
193
|
+
if (this.persistenceEnabled) {
|
|
194
|
+
this.persistToDisk(key, cached);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
normalizeHeaders(headers) {
|
|
198
|
+
const result = {};
|
|
199
|
+
if (!headers)
|
|
200
|
+
return result;
|
|
201
|
+
if (headers instanceof Headers) {
|
|
202
|
+
headers.forEach((value, key) => {
|
|
203
|
+
result[key.toLowerCase()] = value;
|
|
204
|
+
});
|
|
205
|
+
} else if (typeof headers === "object") {
|
|
206
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
207
|
+
if (typeof value === "string") {
|
|
208
|
+
result[key.toLowerCase()] = value;
|
|
209
|
+
} else if (Array.isArray(value)) {
|
|
210
|
+
result[key.toLowerCase()] = value.join(", ");
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
return result;
|
|
215
|
+
}
|
|
216
|
+
getConditionalHeaders(method, url, requestHeaders) {
|
|
217
|
+
const cached = this.get(method, url, requestHeaders);
|
|
218
|
+
if (!cached)
|
|
219
|
+
return;
|
|
220
|
+
const headers = {};
|
|
221
|
+
if (cached.etag) {
|
|
222
|
+
headers["If-None-Match"] = cached.etag;
|
|
223
|
+
}
|
|
224
|
+
if (cached.lastModified) {
|
|
225
|
+
headers["If-Modified-Since"] = cached.lastModified;
|
|
226
|
+
}
|
|
227
|
+
return Object.keys(headers).length > 0 ? headers : undefined;
|
|
228
|
+
}
|
|
229
|
+
updateRevalidated(method, url, newHeaders, requestHeaders) {
|
|
230
|
+
if (!this.config.enable)
|
|
231
|
+
return;
|
|
232
|
+
const key = this.generateKey(method, url, requestHeaders);
|
|
233
|
+
const cached = this.memoryCache.get(key) || this.loadSingleFromDisk(key);
|
|
234
|
+
if (!cached)
|
|
235
|
+
return;
|
|
236
|
+
const normalizedHeaders = this.normalizeHeaders(newHeaders);
|
|
237
|
+
let ttl = cached.ttl;
|
|
238
|
+
if (this.config.respectHeaders) {
|
|
239
|
+
const cacheControl = this.parseCacheControl(normalizedHeaders);
|
|
240
|
+
if (cacheControl.noStore) {
|
|
241
|
+
this.memoryCache.delete(key);
|
|
242
|
+
if (this.persistenceEnabled) {
|
|
243
|
+
const filePath = this.getCacheFilePath(key);
|
|
244
|
+
fsPromises.unlink(filePath).catch(() => {});
|
|
245
|
+
}
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
if (cacheControl.maxAge !== undefined) {
|
|
249
|
+
ttl = cacheControl.maxAge;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
const updated = {
|
|
253
|
+
...cached,
|
|
254
|
+
timestamp: Date.now(),
|
|
255
|
+
ttl,
|
|
256
|
+
headers: { ...cached.headers, ...normalizedHeaders },
|
|
257
|
+
etag: normalizedHeaders["etag"] || cached.etag,
|
|
258
|
+
lastModified: normalizedHeaders["last-modified"] || cached.lastModified
|
|
259
|
+
};
|
|
260
|
+
this.memoryCache.set(key, updated, ttl);
|
|
261
|
+
if (this.persistenceEnabled) {
|
|
262
|
+
this.persistToDisk(key, updated);
|
|
263
|
+
}
|
|
264
|
+
return updated;
|
|
265
|
+
}
|
|
266
|
+
invalidate(url, method) {
|
|
267
|
+
const methods = method ? [method] : this.config.methods;
|
|
268
|
+
for (const m of methods) {
|
|
269
|
+
const key = this.generateKey(m, url);
|
|
270
|
+
this.memoryCache.delete(key);
|
|
271
|
+
if (this.persistenceEnabled) {
|
|
272
|
+
const filePath = this.getCacheFilePath(key);
|
|
273
|
+
fsPromises.unlink(filePath).catch(() => {});
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
clear() {
|
|
278
|
+
this.memoryCache.clear();
|
|
279
|
+
if (this.persistenceEnabled && this.config.cacheDir) {
|
|
280
|
+
const cacheDir = this.config.cacheDir;
|
|
281
|
+
fsPromises.readdir(cacheDir).then((files) => {
|
|
282
|
+
for (const file of files) {
|
|
283
|
+
if (file.endsWith(".json")) {
|
|
284
|
+
fsPromises.unlink(path.join(cacheDir, file)).catch(() => {});
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}).catch(() => {});
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
get size() {
|
|
291
|
+
return this.memoryCache.size;
|
|
292
|
+
}
|
|
293
|
+
get isEnabled() {
|
|
294
|
+
return this.config.enable;
|
|
295
|
+
}
|
|
296
|
+
get isPersistent() {
|
|
297
|
+
return this.persistenceEnabled;
|
|
298
|
+
}
|
|
299
|
+
getConfig() {
|
|
300
|
+
return { ...this.config };
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
export function normalizeResponseCacheConfig(option) {
|
|
304
|
+
if (option === undefined || option === false)
|
|
305
|
+
return;
|
|
306
|
+
if (option === true)
|
|
307
|
+
return { enable: true };
|
|
308
|
+
return option;
|
|
309
|
+
}
|
|
310
|
+
export default ResponseCache;
|