rezo 1.0.128 → 1.0.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +1 -1
- package/dist/adapters/entries/fetch.d.ts +1 -1
- package/dist/adapters/entries/http.d.ts +1 -1
- package/dist/adapters/entries/http2.d.ts +1 -1
- package/dist/adapters/entries/react-native.cjs +6 -6
- package/dist/adapters/entries/react-native.d.ts +1 -1
- package/dist/adapters/entries/xhr.d.ts +1 -1
- package/dist/adapters/http.cjs +1 -1
- package/dist/adapters/http.js +1 -1
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/dns-cache.cjs +11 -1
- package/dist/cache/dns-cache.js +11 -1
- package/dist/cache/index.cjs +9 -9
- package/dist/cache/response-cache.cjs +40 -14
- package/dist/cache/response-cache.js +40 -14
- package/dist/cookies/cookie-jar.cjs +9 -12
- package/dist/cookies/cookie-jar.js +5 -8
- package/dist/cookies/index.cjs +10 -10
- package/dist/crawler/addon/oxylabs/index.cjs +1 -1
- package/dist/crawler/index.cjs +42 -42
- package/dist/crawler/plugin/index.cjs +1 -1
- package/dist/entries/crawler.cjs +6 -6
- package/dist/index.cjs +48 -48
- package/dist/index.d.ts +1 -1
- package/dist/internal/agents/index.cjs +14 -14
- package/dist/platform/browser.d.ts +1 -1
- package/dist/platform/bun.d.ts +1 -1
- package/dist/platform/deno.d.ts +1 -1
- package/dist/platform/node.d.ts +1 -1
- package/dist/platform/react-native.cjs +6 -6
- package/dist/platform/react-native.d.ts +1 -1
- package/dist/platform/worker.d.ts +1 -1
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/responses/universal/stream.cjs +11 -9
- package/dist/responses/universal/stream.js +11 -9
- package/dist/stealth/index.cjs +17 -17
- package/dist/stealth/profiles/index.cjs +10 -10
- package/dist/utils/http-config.cjs +21 -13
- package/dist/utils/http-config.js +21 -13
- package/dist/utils/node-runtime.cjs +29 -0
- package/dist/utils/node-runtime.js +26 -0
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/wget/index.cjs +51 -51
- package/package.json +1 -1
|
@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
|
|
|
6027
6027
|
*
|
|
6028
6028
|
* IMPORTANT: Update these values when bumping package version.
|
|
6029
6029
|
*/
|
|
6030
|
-
export declare const VERSION = "1.0.
|
|
6030
|
+
export declare const VERSION = "1.0.129";
|
|
6031
6031
|
/**
|
|
6032
6032
|
* cURL Options Configuration
|
|
6033
6033
|
*
|
|
@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
|
|
|
6027
6027
|
*
|
|
6028
6028
|
* IMPORTANT: Update these values when bumping package version.
|
|
6029
6029
|
*/
|
|
6030
|
-
export declare const VERSION = "1.0.
|
|
6030
|
+
export declare const VERSION = "1.0.129";
|
|
6031
6031
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
6032
6032
|
export declare const Cancel: typeof RezoError;
|
|
6033
6033
|
export declare const CancelToken: {
|
|
@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
|
|
|
6027
6027
|
*
|
|
6028
6028
|
* IMPORTANT: Update these values when bumping package version.
|
|
6029
6029
|
*/
|
|
6030
|
-
export declare const VERSION = "1.0.
|
|
6030
|
+
export declare const VERSION = "1.0.129";
|
|
6031
6031
|
/**
|
|
6032
6032
|
* Type guard to check if an error is a RezoError instance.
|
|
6033
6033
|
*/
|
|
@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
|
|
|
6027
6027
|
*
|
|
6028
6028
|
* IMPORTANT: Update these values when bumping package version.
|
|
6029
6029
|
*/
|
|
6030
|
-
export declare const VERSION = "1.0.
|
|
6030
|
+
export declare const VERSION = "1.0.129";
|
|
6031
6031
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
6032
6032
|
export declare const Cancel: typeof RezoError;
|
|
6033
6033
|
export declare const CancelToken: {
|
|
@@ -6,12 +6,12 @@ const { RezoFormData } = require('../../utils/form-data.cjs');
|
|
|
6
6
|
const { RezoCookieJar, Cookie } = require('../../cookies/cookie-jar.cjs');
|
|
7
7
|
const { createDefaultHooks, mergeHooks } = require('../../core/hooks.cjs');
|
|
8
8
|
const { VERSION } = require('../../version.cjs');
|
|
9
|
-
const
|
|
10
|
-
exports.createFetchStreamTransport =
|
|
11
|
-
exports.createExpoFileSystemAdapter =
|
|
12
|
-
exports.createReactNativeFsAdapter =
|
|
13
|
-
exports.createNetInfoProvider =
|
|
14
|
-
exports.createExpoBackgroundTaskProvider =
|
|
9
|
+
const _mod_fi01d9 = require('../../platform/react-native-providers.cjs');
|
|
10
|
+
exports.createFetchStreamTransport = _mod_fi01d9.createFetchStreamTransport;
|
|
11
|
+
exports.createExpoFileSystemAdapter = _mod_fi01d9.createExpoFileSystemAdapter;
|
|
12
|
+
exports.createReactNativeFsAdapter = _mod_fi01d9.createReactNativeFsAdapter;
|
|
13
|
+
exports.createNetInfoProvider = _mod_fi01d9.createNetInfoProvider;
|
|
14
|
+
exports.createExpoBackgroundTaskProvider = _mod_fi01d9.createExpoBackgroundTaskProvider;;
|
|
15
15
|
exports.Rezo = Rezo;
|
|
16
16
|
exports.RezoError = RezoError;
|
|
17
17
|
exports.RezoErrorCode = RezoErrorCode;
|
|
@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
|
|
|
6027
6027
|
*
|
|
6028
6028
|
* IMPORTANT: Update these values when bumping package version.
|
|
6029
6029
|
*/
|
|
6030
|
-
export declare const VERSION = "1.0.
|
|
6030
|
+
export declare const VERSION = "1.0.129";
|
|
6031
6031
|
export interface ExpoFileSystemFileLike {
|
|
6032
6032
|
uri?: string;
|
|
6033
6033
|
size?: number;
|
|
@@ -6027,7 +6027,7 @@ export interface RezoInstance extends Rezo, RezoCallable {
|
|
|
6027
6027
|
*
|
|
6028
6028
|
* IMPORTANT: Update these values when bumping package version.
|
|
6029
6029
|
*/
|
|
6030
|
-
export declare const VERSION = "1.0.
|
|
6030
|
+
export declare const VERSION = "1.0.129";
|
|
6031
6031
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
6032
6032
|
export declare const Cancel: typeof RezoError;
|
|
6033
6033
|
export declare const CancelToken: {
|
package/dist/adapters/http.cjs
CHANGED
|
@@ -1368,7 +1368,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, _resp
|
|
|
1368
1368
|
const uploadStart = performance.now();
|
|
1369
1369
|
for (let offset = 0;offset < totalSize; offset += chunkSize) {
|
|
1370
1370
|
const end = Math.min(offset + chunkSize, totalSize);
|
|
1371
|
-
const chunk = buffer.
|
|
1371
|
+
const chunk = buffer.subarray(offset, end);
|
|
1372
1372
|
req.write(chunk);
|
|
1373
1373
|
written += chunk.length;
|
|
1374
1374
|
const now = performance.now();
|
package/dist/adapters/http.js
CHANGED
|
@@ -1368,7 +1368,7 @@ async function request(config, fetchOptions, requestCount, timing, _stats, _resp
|
|
|
1368
1368
|
const uploadStart = performance.now();
|
|
1369
1369
|
for (let offset = 0;offset < totalSize; offset += chunkSize) {
|
|
1370
1370
|
const end = Math.min(offset + chunkSize, totalSize);
|
|
1371
|
-
const chunk = buffer.
|
|
1371
|
+
const chunk = buffer.subarray(offset, end);
|
|
1372
1372
|
req.write(chunk);
|
|
1373
1373
|
written += chunk.length;
|
|
1374
1374
|
const now = performance.now();
|
package/dist/adapters/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.detectRuntime =
|
|
3
|
-
exports.getAdapterCapabilities =
|
|
4
|
-
exports.buildAdapterContext =
|
|
5
|
-
exports.getAvailableAdapters =
|
|
6
|
-
exports.selectAdapter =
|
|
1
|
+
const _mod_bjhzfo = require('./picker.cjs');
|
|
2
|
+
exports.detectRuntime = _mod_bjhzfo.detectRuntime;
|
|
3
|
+
exports.getAdapterCapabilities = _mod_bjhzfo.getAdapterCapabilities;
|
|
4
|
+
exports.buildAdapterContext = _mod_bjhzfo.buildAdapterContext;
|
|
5
|
+
exports.getAvailableAdapters = _mod_bjhzfo.getAvailableAdapters;
|
|
6
|
+
exports.selectAdapter = _mod_bjhzfo.selectAdapter;;
|
package/dist/cache/dns-cache.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const { LRUCache } = require('./lru-cache.cjs');
|
|
2
|
-
const
|
|
2
|
+
const { requireNodeModule } = require('../utils/node-runtime.cjs');
|
|
3
3
|
const DEFAULT_DNS_TTL = 60000;
|
|
4
4
|
const DEFAULT_DNS_MAX_ENTRIES = 1000;
|
|
5
5
|
|
|
@@ -59,6 +59,11 @@ class DNSCache {
|
|
|
59
59
|
}
|
|
60
60
|
resolveDNS(hostname, family) {
|
|
61
61
|
return new Promise((resolve) => {
|
|
62
|
+
const dns = requireNodeModule("node:dns");
|
|
63
|
+
if (!dns?.lookup) {
|
|
64
|
+
resolve(undefined);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
62
67
|
const options = { family: family ?? 0 };
|
|
63
68
|
dns.lookup(hostname, options, (err, address, resultFamily) => {
|
|
64
69
|
if (err || !address || typeof address !== "string") {
|
|
@@ -71,6 +76,11 @@ class DNSCache {
|
|
|
71
76
|
}
|
|
72
77
|
resolveAllDNS(hostname, family) {
|
|
73
78
|
return new Promise((resolve) => {
|
|
79
|
+
const dns = requireNodeModule("node:dns");
|
|
80
|
+
if (!dns?.lookup) {
|
|
81
|
+
resolve([]);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
74
84
|
const options = { family: family ?? 0, all: true };
|
|
75
85
|
dns.lookup(hostname, options, (err, addresses) => {
|
|
76
86
|
if (err || !addresses || !Array.isArray(addresses)) {
|
package/dist/cache/dns-cache.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { LRUCache } from './lru-cache.js';
|
|
2
|
-
import
|
|
2
|
+
import { requireNodeModule } from '../utils/node-runtime.js';
|
|
3
3
|
const DEFAULT_DNS_TTL = 60000;
|
|
4
4
|
const DEFAULT_DNS_MAX_ENTRIES = 1000;
|
|
5
5
|
|
|
@@ -59,6 +59,11 @@ export class DNSCache {
|
|
|
59
59
|
}
|
|
60
60
|
resolveDNS(hostname, family) {
|
|
61
61
|
return new Promise((resolve) => {
|
|
62
|
+
const dns = requireNodeModule("node:dns");
|
|
63
|
+
if (!dns?.lookup) {
|
|
64
|
+
resolve(undefined);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
62
67
|
const options = { family: family ?? 0 };
|
|
63
68
|
dns.lookup(hostname, options, (err, address, resultFamily) => {
|
|
64
69
|
if (err || !address || typeof address !== "string") {
|
|
@@ -71,6 +76,11 @@ export class DNSCache {
|
|
|
71
76
|
}
|
|
72
77
|
resolveAllDNS(hostname, family) {
|
|
73
78
|
return new Promise((resolve) => {
|
|
79
|
+
const dns = requireNodeModule("node:dns");
|
|
80
|
+
if (!dns?.lookup) {
|
|
81
|
+
resolve([]);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
74
84
|
const options = { family: family ?? 0, all: true };
|
|
75
85
|
dns.lookup(hostname, options, (err, addresses) => {
|
|
76
86
|
if (err || !addresses || !Array.isArray(addresses)) {
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
1
|
+
const _mod_snxaxs = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_snxaxs.LRUCache;;
|
|
3
|
+
const _mod_e700bz = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_e700bz.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_e700bz.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_e700bz.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_4lcu8q = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_4lcu8q.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_4lcu8q.normalizeResponseCacheConfig;;
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
const { LRUCache } = require('./lru-cache.cjs');
|
|
2
|
-
const
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
const { requireNodeModule } = require('../utils/node-runtime.cjs');
|
|
3
|
+
function getFsModule() {
|
|
4
|
+
return requireNodeModule("node:fs");
|
|
5
|
+
}
|
|
6
|
+
function getPathModule() {
|
|
7
|
+
return requireNodeModule("node:path");
|
|
8
|
+
}
|
|
5
9
|
const DEFAULT_TTL = 3000000;
|
|
6
10
|
const DEFAULT_MAX_ENTRIES = 500;
|
|
7
11
|
const DEFAULT_METHODS = ["GET", "HEAD"];
|
|
@@ -22,7 +26,7 @@ class ResponseCache {
|
|
|
22
26
|
methods: config.methods ?? DEFAULT_METHODS,
|
|
23
27
|
respectHeaders: config.respectHeaders ?? true
|
|
24
28
|
};
|
|
25
|
-
this.persistenceEnabled = !!this.config.cacheDir;
|
|
29
|
+
this.persistenceEnabled = !!this.config.cacheDir && !!getFsModule() && !!getPathModule();
|
|
26
30
|
this.memoryCache = new LRUCache({
|
|
27
31
|
maxEntries: this.config.maxEntries,
|
|
28
32
|
ttl: this.config.ttl,
|
|
@@ -42,8 +46,13 @@ class ResponseCache {
|
|
|
42
46
|
async initializePersistenceAsync() {
|
|
43
47
|
if (!this.config.cacheDir)
|
|
44
48
|
return;
|
|
49
|
+
const fs = getFsModule();
|
|
50
|
+
if (!fs?.promises) {
|
|
51
|
+
this.persistenceEnabled = false;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
45
54
|
try {
|
|
46
|
-
await
|
|
55
|
+
await fs.promises.mkdir(this.config.cacheDir, { recursive: true });
|
|
47
56
|
await this.loadFromDiskAsync();
|
|
48
57
|
this.initialized = true;
|
|
49
58
|
} catch {
|
|
@@ -51,34 +60,42 @@ class ResponseCache {
|
|
|
51
60
|
}
|
|
52
61
|
}
|
|
53
62
|
getCacheFilePath(key) {
|
|
63
|
+
const path = getPathModule();
|
|
54
64
|
const safeKey = Buffer.from(key).toString("base64url");
|
|
55
|
-
return path.join(this.config.cacheDir, `${safeKey}.json`)
|
|
65
|
+
return path && this.config.cacheDir ? path.join(this.config.cacheDir, `${safeKey}.json`) : `${this.config.cacheDir || ""}/${safeKey}.json`;
|
|
56
66
|
}
|
|
57
67
|
persistToDisk(key, entry) {
|
|
58
68
|
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
59
69
|
return;
|
|
70
|
+
const fs = getFsModule();
|
|
71
|
+
if (!fs?.promises)
|
|
72
|
+
return;
|
|
60
73
|
const filePath = this.getCacheFilePath(key);
|
|
61
|
-
|
|
74
|
+
fs.promises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
|
|
62
75
|
}
|
|
63
76
|
async loadFromDiskAsync() {
|
|
64
77
|
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
65
78
|
return;
|
|
79
|
+
const fs = getFsModule();
|
|
80
|
+
const path = getPathModule();
|
|
81
|
+
if (!fs?.promises || !path)
|
|
82
|
+
return;
|
|
66
83
|
try {
|
|
67
|
-
const files = await
|
|
84
|
+
const files = await fs.promises.readdir(this.config.cacheDir);
|
|
68
85
|
const now = Date.now();
|
|
69
86
|
for (const file of files) {
|
|
70
87
|
if (!file.endsWith(".json"))
|
|
71
88
|
continue;
|
|
72
89
|
try {
|
|
73
90
|
const filePath = path.join(this.config.cacheDir, file);
|
|
74
|
-
const content = await
|
|
91
|
+
const content = await fs.promises.readFile(filePath, "utf-8");
|
|
75
92
|
const entry = JSON.parse(content);
|
|
76
93
|
if (entry.timestamp + entry.ttl > now) {
|
|
77
94
|
const key = Buffer.from(file.replace(".json", ""), "base64url").toString("utf-8");
|
|
78
95
|
const remainingTTL = entry.timestamp + entry.ttl - now;
|
|
79
96
|
this.memoryCache.set(key, entry, remainingTTL);
|
|
80
97
|
} else {
|
|
81
|
-
|
|
98
|
+
fs.promises.unlink(filePath).catch(() => {});
|
|
82
99
|
}
|
|
83
100
|
} catch {}
|
|
84
101
|
}
|
|
@@ -145,6 +162,9 @@ class ResponseCache {
|
|
|
145
162
|
loadSingleFromDisk(key) {
|
|
146
163
|
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
147
164
|
return;
|
|
165
|
+
const fs = getFsModule();
|
|
166
|
+
if (!fs)
|
|
167
|
+
return;
|
|
148
168
|
try {
|
|
149
169
|
const filePath = this.getCacheFilePath(key);
|
|
150
170
|
if (!fs.existsSync(filePath))
|
|
@@ -241,7 +261,8 @@ class ResponseCache {
|
|
|
241
261
|
this.memoryCache.delete(key);
|
|
242
262
|
if (this.persistenceEnabled) {
|
|
243
263
|
const filePath = this.getCacheFilePath(key);
|
|
244
|
-
|
|
264
|
+
const fs = getFsModule();
|
|
265
|
+
fs?.promises?.unlink(filePath).catch(() => {});
|
|
245
266
|
}
|
|
246
267
|
return;
|
|
247
268
|
}
|
|
@@ -270,18 +291,23 @@ class ResponseCache {
|
|
|
270
291
|
this.memoryCache.delete(key);
|
|
271
292
|
if (this.persistenceEnabled) {
|
|
272
293
|
const filePath = this.getCacheFilePath(key);
|
|
273
|
-
|
|
294
|
+
const fs = getFsModule();
|
|
295
|
+
fs?.promises?.unlink(filePath).catch(() => {});
|
|
274
296
|
}
|
|
275
297
|
}
|
|
276
298
|
}
|
|
277
299
|
clear() {
|
|
278
300
|
this.memoryCache.clear();
|
|
279
301
|
if (this.persistenceEnabled && this.config.cacheDir) {
|
|
302
|
+
const fs = getFsModule();
|
|
303
|
+
const path = getPathModule();
|
|
304
|
+
if (!fs?.promises || !path)
|
|
305
|
+
return;
|
|
280
306
|
const cacheDir = this.config.cacheDir;
|
|
281
|
-
|
|
307
|
+
fs.promises.readdir(cacheDir).then((files) => {
|
|
282
308
|
for (const file of files) {
|
|
283
309
|
if (file.endsWith(".json")) {
|
|
284
|
-
|
|
310
|
+
fs.promises.unlink(path.join(cacheDir, file)).catch(() => {});
|
|
285
311
|
}
|
|
286
312
|
}
|
|
287
313
|
}).catch(() => {});
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { LRUCache } from './lru-cache.js';
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
import { requireNodeModule } from '../utils/node-runtime.js';
|
|
3
|
+
function getFsModule() {
|
|
4
|
+
return requireNodeModule("node:fs");
|
|
5
|
+
}
|
|
6
|
+
function getPathModule() {
|
|
7
|
+
return requireNodeModule("node:path");
|
|
8
|
+
}
|
|
5
9
|
const DEFAULT_TTL = 3000000;
|
|
6
10
|
const DEFAULT_MAX_ENTRIES = 500;
|
|
7
11
|
const DEFAULT_METHODS = ["GET", "HEAD"];
|
|
@@ -22,7 +26,7 @@ export class ResponseCache {
|
|
|
22
26
|
methods: config.methods ?? DEFAULT_METHODS,
|
|
23
27
|
respectHeaders: config.respectHeaders ?? true
|
|
24
28
|
};
|
|
25
|
-
this.persistenceEnabled = !!this.config.cacheDir;
|
|
29
|
+
this.persistenceEnabled = !!this.config.cacheDir && !!getFsModule() && !!getPathModule();
|
|
26
30
|
this.memoryCache = new LRUCache({
|
|
27
31
|
maxEntries: this.config.maxEntries,
|
|
28
32
|
ttl: this.config.ttl,
|
|
@@ -42,8 +46,13 @@ export class ResponseCache {
|
|
|
42
46
|
async initializePersistenceAsync() {
|
|
43
47
|
if (!this.config.cacheDir)
|
|
44
48
|
return;
|
|
49
|
+
const fs = getFsModule();
|
|
50
|
+
if (!fs?.promises) {
|
|
51
|
+
this.persistenceEnabled = false;
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
45
54
|
try {
|
|
46
|
-
await
|
|
55
|
+
await fs.promises.mkdir(this.config.cacheDir, { recursive: true });
|
|
47
56
|
await this.loadFromDiskAsync();
|
|
48
57
|
this.initialized = true;
|
|
49
58
|
} catch {
|
|
@@ -51,34 +60,42 @@ export class ResponseCache {
|
|
|
51
60
|
}
|
|
52
61
|
}
|
|
53
62
|
getCacheFilePath(key) {
|
|
63
|
+
const path = getPathModule();
|
|
54
64
|
const safeKey = Buffer.from(key).toString("base64url");
|
|
55
|
-
return path.join(this.config.cacheDir, `${safeKey}.json`)
|
|
65
|
+
return path && this.config.cacheDir ? path.join(this.config.cacheDir, `${safeKey}.json`) : `${this.config.cacheDir || ""}/${safeKey}.json`;
|
|
56
66
|
}
|
|
57
67
|
persistToDisk(key, entry) {
|
|
58
68
|
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
59
69
|
return;
|
|
70
|
+
const fs = getFsModule();
|
|
71
|
+
if (!fs?.promises)
|
|
72
|
+
return;
|
|
60
73
|
const filePath = this.getCacheFilePath(key);
|
|
61
|
-
|
|
74
|
+
fs.promises.writeFile(filePath, JSON.stringify(entry), "utf-8").catch(() => {});
|
|
62
75
|
}
|
|
63
76
|
async loadFromDiskAsync() {
|
|
64
77
|
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
65
78
|
return;
|
|
79
|
+
const fs = getFsModule();
|
|
80
|
+
const path = getPathModule();
|
|
81
|
+
if (!fs?.promises || !path)
|
|
82
|
+
return;
|
|
66
83
|
try {
|
|
67
|
-
const files = await
|
|
84
|
+
const files = await fs.promises.readdir(this.config.cacheDir);
|
|
68
85
|
const now = Date.now();
|
|
69
86
|
for (const file of files) {
|
|
70
87
|
if (!file.endsWith(".json"))
|
|
71
88
|
continue;
|
|
72
89
|
try {
|
|
73
90
|
const filePath = path.join(this.config.cacheDir, file);
|
|
74
|
-
const content = await
|
|
91
|
+
const content = await fs.promises.readFile(filePath, "utf-8");
|
|
75
92
|
const entry = JSON.parse(content);
|
|
76
93
|
if (entry.timestamp + entry.ttl > now) {
|
|
77
94
|
const key = Buffer.from(file.replace(".json", ""), "base64url").toString("utf-8");
|
|
78
95
|
const remainingTTL = entry.timestamp + entry.ttl - now;
|
|
79
96
|
this.memoryCache.set(key, entry, remainingTTL);
|
|
80
97
|
} else {
|
|
81
|
-
|
|
98
|
+
fs.promises.unlink(filePath).catch(() => {});
|
|
82
99
|
}
|
|
83
100
|
} catch {}
|
|
84
101
|
}
|
|
@@ -145,6 +162,9 @@ export class ResponseCache {
|
|
|
145
162
|
loadSingleFromDisk(key) {
|
|
146
163
|
if (!this.persistenceEnabled || !this.config.cacheDir)
|
|
147
164
|
return;
|
|
165
|
+
const fs = getFsModule();
|
|
166
|
+
if (!fs)
|
|
167
|
+
return;
|
|
148
168
|
try {
|
|
149
169
|
const filePath = this.getCacheFilePath(key);
|
|
150
170
|
if (!fs.existsSync(filePath))
|
|
@@ -241,7 +261,8 @@ export class ResponseCache {
|
|
|
241
261
|
this.memoryCache.delete(key);
|
|
242
262
|
if (this.persistenceEnabled) {
|
|
243
263
|
const filePath = this.getCacheFilePath(key);
|
|
244
|
-
|
|
264
|
+
const fs = getFsModule();
|
|
265
|
+
fs?.promises?.unlink(filePath).catch(() => {});
|
|
245
266
|
}
|
|
246
267
|
return;
|
|
247
268
|
}
|
|
@@ -270,18 +291,23 @@ export class ResponseCache {
|
|
|
270
291
|
this.memoryCache.delete(key);
|
|
271
292
|
if (this.persistenceEnabled) {
|
|
272
293
|
const filePath = this.getCacheFilePath(key);
|
|
273
|
-
|
|
294
|
+
const fs = getFsModule();
|
|
295
|
+
fs?.promises?.unlink(filePath).catch(() => {});
|
|
274
296
|
}
|
|
275
297
|
}
|
|
276
298
|
}
|
|
277
299
|
clear() {
|
|
278
300
|
this.memoryCache.clear();
|
|
279
301
|
if (this.persistenceEnabled && this.config.cacheDir) {
|
|
302
|
+
const fs = getFsModule();
|
|
303
|
+
const path = getPathModule();
|
|
304
|
+
if (!fs?.promises || !path)
|
|
305
|
+
return;
|
|
280
306
|
const cacheDir = this.config.cacheDir;
|
|
281
|
-
|
|
307
|
+
fs.promises.readdir(cacheDir).then((files) => {
|
|
282
308
|
for (const file of files) {
|
|
283
309
|
if (file.endsWith(".json")) {
|
|
284
|
-
|
|
310
|
+
fs.promises.unlink(path.join(cacheDir, file)).catch(() => {});
|
|
285
311
|
}
|
|
286
312
|
}
|
|
287
313
|
}).catch(() => {});
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
const { CookieJar: TouchCookieJar } = require("tough-cookie");
|
|
2
2
|
const { Cookie } = require('./cookie.cjs');
|
|
3
|
-
const
|
|
4
|
-
|
|
3
|
+
const { requireNodeModule } = require('../utils/node-runtime.cjs');
|
|
4
|
+
const _mod_p2kzqp = require('./cookie.cjs');
|
|
5
|
+
exports.Cookie = _mod_p2kzqp.Cookie;;
|
|
5
6
|
|
|
6
7
|
class RezoCookieJar extends TouchCookieJar {
|
|
7
8
|
constructor(store, options) {
|
|
@@ -402,10 +403,8 @@ class RezoCookieJar extends TouchCookieJar {
|
|
|
402
403
|
return this._cookieFile;
|
|
403
404
|
}
|
|
404
405
|
loadFromFile(filePath, _defaultUrl) {
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
fs = require("node:fs");
|
|
408
|
-
} catch {
|
|
406
|
+
const fs = requireNodeModule("node:fs");
|
|
407
|
+
if (!fs) {
|
|
409
408
|
throw new Error("loadFromFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
|
|
410
409
|
}
|
|
411
410
|
if (!fs.existsSync(filePath)) {
|
|
@@ -456,10 +455,8 @@ class RezoCookieJar extends TouchCookieJar {
|
|
|
456
455
|
if (!targetPath) {
|
|
457
456
|
throw new Error("No cookie file path specified. Provide a path or load from a file first.");
|
|
458
457
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
fs = require("node:fs");
|
|
462
|
-
} catch {
|
|
458
|
+
const fs = requireNodeModule("node:fs");
|
|
459
|
+
if (!fs) {
|
|
463
460
|
throw new Error("saveToFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
|
|
464
461
|
}
|
|
465
462
|
const isJson = targetPath.toLowerCase().endsWith(".json");
|
|
@@ -484,7 +481,7 @@ class RezoCookieJar extends TouchCookieJar {
|
|
|
484
481
|
}
|
|
485
482
|
}
|
|
486
483
|
const CookieJar = exports.CookieJar = RezoCookieJar;
|
|
487
|
-
const
|
|
488
|
-
exports.Store =
|
|
484
|
+
const _mod_nfmlpp = require("tough-cookie");
|
|
485
|
+
exports.Store = _mod_nfmlpp.Store;;
|
|
489
486
|
|
|
490
487
|
exports.RezoCookieJar = RezoCookieJar;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CookieJar as TouchCookieJar } from "tough-cookie";
|
|
2
2
|
import { Cookie } from './cookie.js';
|
|
3
|
+
import { requireNodeModule } from '../utils/node-runtime.js';
|
|
3
4
|
export { Cookie } from './cookie.js';
|
|
4
5
|
|
|
5
6
|
export class RezoCookieJar extends TouchCookieJar {
|
|
@@ -401,10 +402,8 @@ export class RezoCookieJar extends TouchCookieJar {
|
|
|
401
402
|
return this._cookieFile;
|
|
402
403
|
}
|
|
403
404
|
loadFromFile(filePath, _defaultUrl) {
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
fs = require("node:fs");
|
|
407
|
-
} catch {
|
|
405
|
+
const fs = requireNodeModule("node:fs");
|
|
406
|
+
if (!fs) {
|
|
408
407
|
throw new Error("loadFromFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
|
|
409
408
|
}
|
|
410
409
|
if (!fs.existsSync(filePath)) {
|
|
@@ -455,10 +454,8 @@ export class RezoCookieJar extends TouchCookieJar {
|
|
|
455
454
|
if (!targetPath) {
|
|
456
455
|
throw new Error("No cookie file path specified. Provide a path or load from a file first.");
|
|
457
456
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
fs = require("node:fs");
|
|
461
|
-
} catch {
|
|
457
|
+
const fs = requireNodeModule("node:fs");
|
|
458
|
+
if (!fs) {
|
|
462
459
|
throw new Error("saveToFile() requires Node.js, Bun, or Deno. Not available in browsers or React Native.");
|
|
463
460
|
}
|
|
464
461
|
const isJson = targetPath.toLowerCase().endsWith(".json");
|
package/dist/cookies/index.cjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Cookie =
|
|
3
|
-
exports.RezoCookie =
|
|
4
|
-
const
|
|
5
|
-
exports.RezoCookieStore =
|
|
6
|
-
const
|
|
7
|
-
exports.RezoCookieJar =
|
|
8
|
-
exports.CookieJar =
|
|
9
|
-
const
|
|
10
|
-
exports.Store =
|
|
1
|
+
const _mod_5f7e7j = require('./cookie.cjs');
|
|
2
|
+
exports.Cookie = _mod_5f7e7j.Cookie;
|
|
3
|
+
exports.RezoCookie = _mod_5f7e7j.RezoCookie;;
|
|
4
|
+
const _mod_qgxkmc = require('./cookie-store.cjs');
|
|
5
|
+
exports.RezoCookieStore = _mod_qgxkmc.RezoCookieStore;;
|
|
6
|
+
const _mod_tenk2p = require('./cookie-jar.cjs');
|
|
7
|
+
exports.RezoCookieJar = _mod_tenk2p.RezoCookieJar;
|
|
8
|
+
exports.CookieJar = _mod_tenk2p.CookieJar;;
|
|
9
|
+
const _mod_01bx46 = require("tough-cookie");
|
|
10
|
+
exports.Store = _mod_01bx46.Store;;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
var{Rezo:h}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(
|
|
1
|
+
var{Rezo:h}=require("../../../core/rezo.cjs"),a=require("./options.cjs");exports.OXYLABS_BROWSER_TYPES=a.OXYLABS_BROWSER_TYPES;exports.OXYLABS_COMMON_LOCALES=a.OXYLABS_COMMON_LOCALES;exports.OXYLABS_COMMON_GEO_LOCATIONS=a.OXYLABS_COMMON_GEO_LOCATIONS;exports.OXYLABS_US_STATES=a.OXYLABS_US_STATES;exports.OXYLABS_EUROPEAN_COUNTRIES=a.OXYLABS_EUROPEAN_COUNTRIES;exports.OXYLABS_ASIAN_COUNTRIES=a.OXYLABS_ASIAN_COUNTRIES;exports.getRandomBrowserType=a.getRandomBrowserType;exports.getRandomLocale=a.getRandomLocale;exports.getRandomGeoLocation=a.getRandomGeoLocation;var i="https://realtime.oxylabs.io/v1/queries";class d{config;http;authHeader;constructor(t){if(!t.username||!t.password)throw Error("Oxylabs username and password are required");this.config={username:t.username,password:t.password,browserType:t.browserType??"desktop",locale:t.locale??"en-US",geoLocation:t.geoLocation??"",render:t.render??!1,context:t.context??{},timeout:t.timeout??120000},this.http=new h({baseURL:i,timeout:this.config.timeout}),this.authHeader=`Basic ${Buffer.from(`${t.username}:${t.password}`).toString("base64")}`}async scrape(t,e){let s={...this.config,...e},r=this.buildRequestBody(t,s),o=(await this.http.postJson(i,r,{headers:{Authorization:this.authHeader,"Content-Type":"application/json"}})).data;if(o.error)throw Error(`Oxylabs API error: ${o.error}`);if(!o.results||o.results.length===0)throw Error("Oxylabs API returned no results");let n=o.results[0],l=n._response?.cookies||[],_=n._response?.headers||{};return{statusCode:n.status_code,url:n.url,content:n.content,cookies:l,headers:_,jobId:o.job_id||n.job_id,rendered:s.render,geoLocation:s.geoLocation||void 0,locale:s.locale,browserType:s.browserType,raw:o}}async scrapeMany(t,e,s=1000){let r=[];for(let u=0;u<t.length;u++){let o=await this.scrape(t[u],e);if(r.push(o),u<t.length-1&&s>0)await new Promise((n)=>setTimeout(n,s))}return r}buildRequestBody(t,e){let s=[];if(s.push({key:"return_page_cookies",value:!0}),s.push({key:"return_page_headers",value:!0}),e.context)for(let[u,o]of Object.entries(e.context))s.push({key:u,value:o});if(e.headers&&Object.keys(e.headers).length>0)s.push({key:"headers",value:e.headers});if(e.cookies&&e.cookies.length>0)s.push({key:"cookies",value:e.cookies});if(e.session_id)s.push({key:"session_id",value:e.session_id});if(e.http_method)s.push({key:"http_method",value:e.http_method});if(e.base64Body)s.push({key:"content",value:e.base64Body});if(typeof e.follow_redirects==="boolean")s.push({key:"follow_redirects",value:e.follow_redirects});if(e.successful_status_codes&&e.successful_status_codes.length>0)s.push({key:"successful_status_codes",value:e.successful_status_codes});let r={source:"universal",url:t,context:s};if(e.render||e.javascript_rendering)r.render="html";if(e.browserType)r.user_agent_type=e.browserType;if(e.locale)r.locale=e.locale;if(e.geoLocation)r.geo_location=e.geoLocation;if(e.returnAsBase64)r.content_encoding="base64";return r}getConfig(){return{...this.config,password:"***"}}withConfig(t){return new d({...this.config,...t})}async testConnection(){try{return await this.scrape("https://httpbin.org/ip"),!0}catch(t){throw Error(`Oxylabs connection test failed: ${t.message}`)}}}exports.Oxylabs=d;exports.default=d;module.exports=Object.assign(d,exports);
|