rezo 1.0.136 → 1.0.138
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/adapters/entries/curl.d.ts +119 -82
- package/dist/adapters/entries/fetch.d.ts +119 -82
- package/dist/adapters/entries/http.d.ts +119 -82
- package/dist/adapters/entries/http2.d.ts +119 -82
- package/dist/adapters/entries/react-native.cjs +6 -6
- package/dist/adapters/entries/react-native.d.ts +119 -82
- package/dist/adapters/entries/xhr.d.ts +119 -82
- package/dist/adapters/http.cjs +26 -0
- package/dist/adapters/http.js +26 -0
- package/dist/adapters/index.cjs +10 -10
- package/dist/adapters/index.d.ts +52 -15
- package/dist/cache/index.cjs +9 -9
- package/dist/cookies/cookie-jar.cjs +154 -9
- package/dist/cookies/cookie-jar.js +150 -5
- package/dist/cookies/index.cjs +10 -10
- package/dist/crawler/index.cjs +42 -42
- package/dist/crawler/plugin/index.cjs +1 -1
- package/dist/crawler.d.ts +118 -81
- package/dist/entries/crawler.cjs +24 -24
- package/dist/errors/rezo-error.cjs +3 -2
- package/dist/errors/rezo-error.js +3 -2
- package/dist/index.cjs +58 -58
- package/dist/index.d.ts +119 -82
- package/dist/internal/agents/index.cjs +14 -14
- package/dist/platform/browser.d.ts +119 -82
- package/dist/platform/bun.d.ts +119 -82
- package/dist/platform/deno.d.ts +119 -82
- package/dist/platform/node.d.ts +119 -82
- package/dist/platform/react-native.cjs +6 -6
- package/dist/platform/react-native.d.ts +119 -82
- package/dist/platform/worker.d.ts +119 -82
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/stealth/index.cjs +17 -17
- package/dist/stealth/profiles/index.cjs +10 -10
- package/dist/utils/agent-pool.cjs +1 -1
- package/dist/utils/agent-pool.js +1 -1
- package/dist/utils/headers.cjs +3 -0
- package/dist/utils/headers.js +3 -0
- package/dist/utils/http-config.cjs +15 -2
- package/dist/utils/http-config.js +15 -2
- package/dist/utils/staged-timeout.cjs +1 -2
- package/dist/utils/staged-timeout.js +1 -2
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/wget/index.cjs +51 -51
- package/dist/wget/index.d.ts +118 -81
- package/package.json +1 -1
package/dist/adapters/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { Agent as HttpAgent, OutgoingHttpHeaders } from 'node:http';
|
|
|
2
2
|
import { Agent as HttpsAgent } from 'node:https';
|
|
3
3
|
import { Socket } from 'node:net';
|
|
4
4
|
import { SecureContext, TLSSocket } from 'node:tls';
|
|
5
|
-
import { Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieJarOptions, CreateCookieOptions, Nullable, Store } from 'tough-cookie';
|
|
5
|
+
import { Callback, Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieJarOptions, CreateCookieOptions, GetCookiesOptions, Nullable, SerializedCookieJar, SetCookieOptions, Store } from 'tough-cookie';
|
|
6
6
|
|
|
7
7
|
export interface RezoHttpHeaders {
|
|
8
8
|
accept?: string | undefined;
|
|
@@ -137,6 +137,9 @@ declare class RezoHeaders extends Headers {
|
|
|
137
137
|
* Used by stealth adapters to match browser header ordering.
|
|
138
138
|
*/
|
|
139
139
|
toOrderedObject(order: string[]): Record<string, string | string[]>;
|
|
140
|
+
toJSON(): Record<string, string> & {
|
|
141
|
+
"set-cookie"?: string[];
|
|
142
|
+
};
|
|
140
143
|
toObject(omit?: Array<keyof RezoHttpHeaders> | keyof RezoHttpHeaders): Record<string, string | string[]>;
|
|
141
144
|
toString(): string;
|
|
142
145
|
set(name: keyof RezoHttpHeaders, value: string): void;
|
|
@@ -214,6 +217,38 @@ declare class RezoCookieJar extends TouchCookieJar {
|
|
|
214
217
|
constructor(cookies: Cookie[], url: string);
|
|
215
218
|
constructor(store: Nullable<Store>, options?: CreateCookieJarOptions | boolean);
|
|
216
219
|
private generateCookies;
|
|
220
|
+
setCookieSync(cookie: string | Cookie, url: string, options?: SetCookieOptions): Cookie | undefined;
|
|
221
|
+
setCookie(cookie: string | TouchCookie, url: string | URL, callback: Callback<TouchCookie | undefined>): void;
|
|
222
|
+
setCookie(cookie: string | TouchCookie, url: string | URL, options: SetCookieOptions, callback: Callback<TouchCookie | undefined>): void;
|
|
223
|
+
setCookie(cookie: string | TouchCookie, url: string | URL, options?: SetCookieOptions): Promise<TouchCookie | undefined>;
|
|
224
|
+
setCookie(cookie: string | TouchCookie, url: string | URL, options: SetCookieOptions | Callback<TouchCookie | undefined>, callback?: Callback<TouchCookie | undefined>): unknown;
|
|
225
|
+
getCookies(url: string, callback: Callback<TouchCookie[]>): void;
|
|
226
|
+
getCookies(url: string | URL, options: GetCookiesOptions | undefined, callback: Callback<TouchCookie[]>): void;
|
|
227
|
+
getCookies(url: string | URL, options?: GetCookiesOptions): Promise<TouchCookie[]>;
|
|
228
|
+
getCookies(url: string | URL, options: GetCookiesOptions | undefined | Callback<TouchCookie[]>, callback?: Callback<TouchCookie[]>): unknown;
|
|
229
|
+
getCookiesSync(url: string, options?: GetCookiesOptions): Cookie[];
|
|
230
|
+
getCookieString(url: string, callback: Callback<string | undefined>): void;
|
|
231
|
+
getCookieString(url: string, options: GetCookiesOptions, callback: Callback<string | undefined>): void;
|
|
232
|
+
getCookieString(url: string, options?: GetCookiesOptions): Promise<string>;
|
|
233
|
+
getCookieString(url: string, options: GetCookiesOptions | Callback<string | undefined>, callback?: Callback<string | undefined>): unknown;
|
|
234
|
+
getCookieStringSync(url: string, options?: GetCookiesOptions): string;
|
|
235
|
+
getSetCookieStrings(url: string, callback: Callback<string[] | undefined>): void;
|
|
236
|
+
getSetCookieStrings(url: string, options: GetCookiesOptions, callback: Callback<string[] | undefined>): void;
|
|
237
|
+
getSetCookieStrings(url: string, options?: GetCookiesOptions): Promise<string[] | undefined>;
|
|
238
|
+
getSetCookieStrings(url: string, options: GetCookiesOptions, callback?: Callback<string[] | undefined>): unknown;
|
|
239
|
+
getSetCookieStringsSync(url: string, options?: GetCookiesOptions): string[];
|
|
240
|
+
serialize(callback: Callback<SerializedCookieJar>): void;
|
|
241
|
+
serialize(): Promise<SerializedCookieJar>;
|
|
242
|
+
serializeSync(): SerializedCookieJar | undefined;
|
|
243
|
+
toJSON(): ReturnType<TouchCookieJar["toJSON"]>;
|
|
244
|
+
private toPublicSerializedJar;
|
|
245
|
+
private toStoredCookie;
|
|
246
|
+
private toPublicCookies;
|
|
247
|
+
private toPublicCookie;
|
|
248
|
+
private toStoredCookieKey;
|
|
249
|
+
private toPublicCookieKey;
|
|
250
|
+
private cookiesToCookieString;
|
|
251
|
+
private cookiesToSetCookieStrings;
|
|
217
252
|
/**
|
|
218
253
|
* Get all cookies from the cookie jar.
|
|
219
254
|
*
|
|
@@ -3993,34 +4028,36 @@ export interface RezoRequestConfig<D = any> {
|
|
|
3993
4028
|
* TLS negotiation for HTTPS).
|
|
3994
4029
|
*
|
|
3995
4030
|
* **Behavior:**
|
|
3996
|
-
* - `
|
|
3997
|
-
*
|
|
4031
|
+
* - `true` (default) - Connections are pooled and reused across requests to the
|
|
4032
|
+
* same host. Idle pooled sockets are closed after ~5 seconds.
|
|
4033
|
+
* - `false` - A fresh connection is opened for every request and closed after
|
|
4034
|
+
* the response. No pooling, no reuse.
|
|
3998
4035
|
*
|
|
3999
|
-
* **When to
|
|
4036
|
+
* **When to keep the default (`keepAlive: true`):**
|
|
4000
4037
|
* - Making multiple requests to the same host in sequence
|
|
4001
4038
|
* - Long-running applications (servers, bots, scrapers)
|
|
4002
4039
|
* - Performance-critical applications where connection overhead matters
|
|
4003
4040
|
*
|
|
4004
|
-
* **When to use `keepAlive: false
|
|
4005
|
-
* -
|
|
4006
|
-
* -
|
|
4007
|
-
* -
|
|
4041
|
+
* **When to use `keepAlive: false`:**
|
|
4042
|
+
* - Hosts that reset reused connections aggressively
|
|
4043
|
+
* - Debugging connection-level issues (isolates every request)
|
|
4044
|
+
* - One-off requests where connection reuse buys nothing
|
|
4008
4045
|
*
|
|
4009
4046
|
* @example
|
|
4010
4047
|
* ```typescript
|
|
4011
|
-
* // Default:
|
|
4048
|
+
* // Default: pooled keep-alive connections
|
|
4012
4049
|
* const { data } = await rezo.get('https://api.example.com/data');
|
|
4013
4050
|
*
|
|
4014
|
-
* //
|
|
4015
|
-
* const client =
|
|
4051
|
+
* // Fresh connection per request for this instance
|
|
4052
|
+
* const client = rezo.create({ keepAlive: false });
|
|
4016
4053
|
* await client.get('https://api.example.com/users');
|
|
4017
|
-
* await client.get('https://api.example.com/posts'); //
|
|
4054
|
+
* await client.get('https://api.example.com/posts'); // New connection again
|
|
4018
4055
|
*
|
|
4019
|
-
* //
|
|
4020
|
-
*
|
|
4056
|
+
* // Or per request
|
|
4057
|
+
* await rezo.get('https://api.example.com/data', { keepAlive: false });
|
|
4021
4058
|
* ```
|
|
4022
4059
|
*
|
|
4023
|
-
* @default
|
|
4060
|
+
* @default true
|
|
4024
4061
|
*/
|
|
4025
4062
|
keepAlive?: boolean;
|
|
4026
4063
|
/**
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
1
|
+
const _mod_a07f40 = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_a07f40.LRUCache;;
|
|
3
|
+
const _mod_e8bo0m = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_e8bo0m.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_e8bo0m.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_e8bo0m.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_xklic4 = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_xklic4.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_xklic4.normalizeResponseCacheConfig;;
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
const { CookieJar: TouchCookieJar } = require("tough-cookie");
|
|
2
2
|
const { Cookie } = require('./cookie.cjs');
|
|
3
3
|
const { requireNodeModule } = require('../utils/node-runtime.cjs');
|
|
4
|
-
const
|
|
5
|
-
exports.Cookie =
|
|
4
|
+
const _mod_q2chxo = require('./cookie.cjs');
|
|
5
|
+
exports.Cookie = _mod_q2chxo.Cookie;;
|
|
6
|
+
const REZO_COOKIE_PREFIX_KEY = "Rezo_prx_";
|
|
7
|
+
const HOST_PREFIX = "__Host-";
|
|
8
|
+
const STORED_HOST_PREFIX = `${REZO_COOKIE_PREFIX_KEY}Host-`;
|
|
6
9
|
|
|
7
10
|
class RezoCookieJar extends TouchCookieJar {
|
|
8
11
|
constructor(store, options) {
|
|
@@ -22,9 +25,10 @@ class RezoCookieJar extends TouchCookieJar {
|
|
|
22
25
|
generateCookies(data, url) {
|
|
23
26
|
data = !data && url ? this.getCookiesSync(url) : data;
|
|
24
27
|
const cookies = !data ? (this.toJSON()?.cookies || []).map((cookie) => new Cookie(cookie)) : data[0] instanceof Cookie ? data : data.map((cookie) => new Cookie(cookie instanceof Cookie ? cookie : Cookie.fromJSON(cookie)));
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
const
|
|
28
|
+
const publicCookies = cookies.map((cookie) => this.toPublicCookie(cookie));
|
|
29
|
+
const netscape = publicCookies.map((cookie) => cookie.toNetscapeFormat());
|
|
30
|
+
const cookieString = publicCookies.map((cookie) => cookie.cookieString());
|
|
31
|
+
const setCookiesString = publicCookies.map((cookie) => cookie.toSetCookieString());
|
|
28
32
|
let netscapeString = `# Netscape HTTP Cookie File
|
|
29
33
|
`;
|
|
30
34
|
netscapeString += `# This file was generated by Rezo HTTP client
|
|
@@ -32,13 +36,154 @@ class RezoCookieJar extends TouchCookieJar {
|
|
|
32
36
|
netscapeString += netscape.join(`
|
|
33
37
|
`) || "";
|
|
34
38
|
return {
|
|
35
|
-
array:
|
|
36
|
-
serialized:
|
|
39
|
+
array: publicCookies,
|
|
40
|
+
serialized: publicCookies.map((cookie) => cookie.toJSON()) || [],
|
|
37
41
|
netscape: netscapeString,
|
|
38
42
|
string: cookieString.join("; "),
|
|
39
43
|
setCookiesString
|
|
40
44
|
};
|
|
41
45
|
}
|
|
46
|
+
setCookieSync(cookie, url, options) {
|
|
47
|
+
const storedCookie = this.toStoredCookie(cookie);
|
|
48
|
+
const result = super.setCookieSync(storedCookie, url, options);
|
|
49
|
+
return result ? this.toPublicCookie(result) : undefined;
|
|
50
|
+
}
|
|
51
|
+
setCookie(cookie, url, options, callback) {
|
|
52
|
+
const storedCookie = this.toStoredCookie(cookie);
|
|
53
|
+
const handleResult = (done) => (error, result) => {
|
|
54
|
+
if (error) {
|
|
55
|
+
done(error);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
done(null, result ? this.toPublicCookie(result) : result);
|
|
59
|
+
};
|
|
60
|
+
if (typeof options === "function") {
|
|
61
|
+
super.setCookie(storedCookie, url, handleResult(options));
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
if (callback) {
|
|
65
|
+
super.setCookie(storedCookie, url, options, handleResult(callback));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
return super.setCookie(storedCookie, url, options).then((result) => result ? this.toPublicCookie(result) : result);
|
|
69
|
+
}
|
|
70
|
+
getCookies(url, options, callback) {
|
|
71
|
+
const handleResult = (done) => (error, cookies) => {
|
|
72
|
+
if (error) {
|
|
73
|
+
done(error);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
done(null, this.toPublicCookies(cookies || []));
|
|
77
|
+
};
|
|
78
|
+
if (typeof options === "function") {
|
|
79
|
+
super.getCookies(url, handleResult(options));
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
if (callback) {
|
|
83
|
+
super.getCookies(url, options, handleResult(callback));
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
return super.getCookies(url, options).then((cookies) => this.toPublicCookies(cookies));
|
|
87
|
+
}
|
|
88
|
+
getCookiesSync(url, options) {
|
|
89
|
+
return this.toPublicCookies(super.getCookiesSync(url, options));
|
|
90
|
+
}
|
|
91
|
+
getCookieString(url, options, callback) {
|
|
92
|
+
if (typeof options === "function") {
|
|
93
|
+
this.getCookies(url).then((cookies) => options(null, this.cookiesToCookieString(cookies))).catch((error) => options(error));
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (callback) {
|
|
97
|
+
this.getCookies(url, options).then((cookies) => callback(null, this.cookiesToCookieString(cookies))).catch((error) => callback(error));
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
return this.getCookies(url, options).then((cookies) => this.cookiesToCookieString(cookies));
|
|
101
|
+
}
|
|
102
|
+
getCookieStringSync(url, options) {
|
|
103
|
+
return this.cookiesToCookieString(this.getCookiesSync(url, options));
|
|
104
|
+
}
|
|
105
|
+
getSetCookieStrings(url, options, callback) {
|
|
106
|
+
if (typeof options === "function") {
|
|
107
|
+
this.getCookies(url).then((cookies) => options(null, this.cookiesToSetCookieStrings(cookies))).catch((error) => options(error));
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
if (callback) {
|
|
111
|
+
this.getCookies(url, options).then((cookies) => callback(null, this.cookiesToSetCookieStrings(cookies))).catch((error) => callback(error));
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
return this.getCookies(url, options).then((cookies) => this.cookiesToSetCookieStrings(cookies));
|
|
115
|
+
}
|
|
116
|
+
getSetCookieStringsSync(url, options) {
|
|
117
|
+
return this.cookiesToSetCookieStrings(this.getCookiesSync(url, options));
|
|
118
|
+
}
|
|
119
|
+
serialize(callback) {
|
|
120
|
+
if (callback) {
|
|
121
|
+
super.serialize((error, serialized) => {
|
|
122
|
+
if (error) {
|
|
123
|
+
callback(error);
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
callback(null, this.toPublicSerializedJar(serialized));
|
|
127
|
+
});
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
return super.serialize().then((serialized) => this.toPublicSerializedJar(serialized));
|
|
131
|
+
}
|
|
132
|
+
serializeSync() {
|
|
133
|
+
return this.toPublicSerializedJar(super.serializeSync());
|
|
134
|
+
}
|
|
135
|
+
toJSON() {
|
|
136
|
+
const json = this.serializeSync();
|
|
137
|
+
if (!json?.cookies)
|
|
138
|
+
return json;
|
|
139
|
+
return json;
|
|
140
|
+
}
|
|
141
|
+
toPublicSerializedJar(json) {
|
|
142
|
+
if (!json?.cookies)
|
|
143
|
+
return json;
|
|
144
|
+
return {
|
|
145
|
+
...json,
|
|
146
|
+
cookies: json.cookies.map((cookie) => ({
|
|
147
|
+
...cookie,
|
|
148
|
+
key: this.toPublicCookieKey(cookie.key)
|
|
149
|
+
}))
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
toStoredCookie(cookie) {
|
|
153
|
+
const parsed = typeof cookie === "string" ? Cookie.parse(cookie) : cookie;
|
|
154
|
+
if (!parsed?.key?.startsWith(HOST_PREFIX)) {
|
|
155
|
+
return cookie;
|
|
156
|
+
}
|
|
157
|
+
const storedCookie = new Cookie(Cookie.fromJSON(parsed.toJSON()));
|
|
158
|
+
storedCookie.key = this.toStoredCookieKey(storedCookie.key);
|
|
159
|
+
return storedCookie;
|
|
160
|
+
}
|
|
161
|
+
toPublicCookies(cookies) {
|
|
162
|
+
return cookies.map((cookie) => this.toPublicCookie(cookie));
|
|
163
|
+
}
|
|
164
|
+
toPublicCookie(cookie) {
|
|
165
|
+
const publicCookie = new Cookie(Cookie.fromJSON(cookie.toJSON()));
|
|
166
|
+
const publicKey = this.toPublicCookieKey(publicCookie.key);
|
|
167
|
+
if (publicKey) {
|
|
168
|
+
publicCookie.key = publicKey;
|
|
169
|
+
}
|
|
170
|
+
return publicCookie;
|
|
171
|
+
}
|
|
172
|
+
toStoredCookieKey(key) {
|
|
173
|
+
return key.startsWith(HOST_PREFIX) ? `${REZO_COOKIE_PREFIX_KEY}${key.slice(2)}` : key;
|
|
174
|
+
}
|
|
175
|
+
toPublicCookieKey(key) {
|
|
176
|
+
if (!key) {
|
|
177
|
+
return key;
|
|
178
|
+
}
|
|
179
|
+
return key.startsWith(STORED_HOST_PREFIX) ? `__${key.slice(REZO_COOKIE_PREFIX_KEY.length)}` : key;
|
|
180
|
+
}
|
|
181
|
+
cookiesToCookieString(cookies) {
|
|
182
|
+
return cookies.map((cookie) => cookie.cookieString()).join("; ");
|
|
183
|
+
}
|
|
184
|
+
cookiesToSetCookieStrings(cookies) {
|
|
185
|
+
return cookies.map((cookie) => cookie.toSetCookieString());
|
|
186
|
+
}
|
|
42
187
|
cookies(url) {
|
|
43
188
|
return this.generateCookies(undefined, url);
|
|
44
189
|
}
|
|
@@ -491,7 +636,7 @@ class RezoCookieJar extends TouchCookieJar {
|
|
|
491
636
|
}
|
|
492
637
|
}
|
|
493
638
|
const CookieJar = exports.CookieJar = RezoCookieJar;
|
|
494
|
-
const
|
|
495
|
-
exports.Store =
|
|
639
|
+
const _mod_0uc7x2 = require("tough-cookie");
|
|
640
|
+
exports.Store = _mod_0uc7x2.Store;;
|
|
496
641
|
|
|
497
642
|
exports.RezoCookieJar = RezoCookieJar;
|
|
@@ -2,6 +2,9 @@ import { CookieJar as TouchCookieJar } from "tough-cookie";
|
|
|
2
2
|
import { Cookie } from './cookie.js';
|
|
3
3
|
import { requireNodeModule } from '../utils/node-runtime.js';
|
|
4
4
|
export { Cookie } from './cookie.js';
|
|
5
|
+
const REZO_COOKIE_PREFIX_KEY = "Rezo_prx_";
|
|
6
|
+
const HOST_PREFIX = "__Host-";
|
|
7
|
+
const STORED_HOST_PREFIX = `${REZO_COOKIE_PREFIX_KEY}Host-`;
|
|
5
8
|
|
|
6
9
|
export class RezoCookieJar extends TouchCookieJar {
|
|
7
10
|
constructor(store, options) {
|
|
@@ -21,9 +24,10 @@ export class RezoCookieJar extends TouchCookieJar {
|
|
|
21
24
|
generateCookies(data, url) {
|
|
22
25
|
data = !data && url ? this.getCookiesSync(url) : data;
|
|
23
26
|
const cookies = !data ? (this.toJSON()?.cookies || []).map((cookie) => new Cookie(cookie)) : data[0] instanceof Cookie ? data : data.map((cookie) => new Cookie(cookie instanceof Cookie ? cookie : Cookie.fromJSON(cookie)));
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
+
const publicCookies = cookies.map((cookie) => this.toPublicCookie(cookie));
|
|
28
|
+
const netscape = publicCookies.map((cookie) => cookie.toNetscapeFormat());
|
|
29
|
+
const cookieString = publicCookies.map((cookie) => cookie.cookieString());
|
|
30
|
+
const setCookiesString = publicCookies.map((cookie) => cookie.toSetCookieString());
|
|
27
31
|
let netscapeString = `# Netscape HTTP Cookie File
|
|
28
32
|
`;
|
|
29
33
|
netscapeString += `# This file was generated by Rezo HTTP client
|
|
@@ -31,13 +35,154 @@ export class RezoCookieJar extends TouchCookieJar {
|
|
|
31
35
|
netscapeString += netscape.join(`
|
|
32
36
|
`) || "";
|
|
33
37
|
return {
|
|
34
|
-
array:
|
|
35
|
-
serialized:
|
|
38
|
+
array: publicCookies,
|
|
39
|
+
serialized: publicCookies.map((cookie) => cookie.toJSON()) || [],
|
|
36
40
|
netscape: netscapeString,
|
|
37
41
|
string: cookieString.join("; "),
|
|
38
42
|
setCookiesString
|
|
39
43
|
};
|
|
40
44
|
}
|
|
45
|
+
setCookieSync(cookie, url, options) {
|
|
46
|
+
const storedCookie = this.toStoredCookie(cookie);
|
|
47
|
+
const result = super.setCookieSync(storedCookie, url, options);
|
|
48
|
+
return result ? this.toPublicCookie(result) : undefined;
|
|
49
|
+
}
|
|
50
|
+
setCookie(cookie, url, options, callback) {
|
|
51
|
+
const storedCookie = this.toStoredCookie(cookie);
|
|
52
|
+
const handleResult = (done) => (error, result) => {
|
|
53
|
+
if (error) {
|
|
54
|
+
done(error);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
done(null, result ? this.toPublicCookie(result) : result);
|
|
58
|
+
};
|
|
59
|
+
if (typeof options === "function") {
|
|
60
|
+
super.setCookie(storedCookie, url, handleResult(options));
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
if (callback) {
|
|
64
|
+
super.setCookie(storedCookie, url, options, handleResult(callback));
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
return super.setCookie(storedCookie, url, options).then((result) => result ? this.toPublicCookie(result) : result);
|
|
68
|
+
}
|
|
69
|
+
getCookies(url, options, callback) {
|
|
70
|
+
const handleResult = (done) => (error, cookies) => {
|
|
71
|
+
if (error) {
|
|
72
|
+
done(error);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
done(null, this.toPublicCookies(cookies || []));
|
|
76
|
+
};
|
|
77
|
+
if (typeof options === "function") {
|
|
78
|
+
super.getCookies(url, handleResult(options));
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
if (callback) {
|
|
82
|
+
super.getCookies(url, options, handleResult(callback));
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
return super.getCookies(url, options).then((cookies) => this.toPublicCookies(cookies));
|
|
86
|
+
}
|
|
87
|
+
getCookiesSync(url, options) {
|
|
88
|
+
return this.toPublicCookies(super.getCookiesSync(url, options));
|
|
89
|
+
}
|
|
90
|
+
getCookieString(url, options, callback) {
|
|
91
|
+
if (typeof options === "function") {
|
|
92
|
+
this.getCookies(url).then((cookies) => options(null, this.cookiesToCookieString(cookies))).catch((error) => options(error));
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (callback) {
|
|
96
|
+
this.getCookies(url, options).then((cookies) => callback(null, this.cookiesToCookieString(cookies))).catch((error) => callback(error));
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
return this.getCookies(url, options).then((cookies) => this.cookiesToCookieString(cookies));
|
|
100
|
+
}
|
|
101
|
+
getCookieStringSync(url, options) {
|
|
102
|
+
return this.cookiesToCookieString(this.getCookiesSync(url, options));
|
|
103
|
+
}
|
|
104
|
+
getSetCookieStrings(url, options, callback) {
|
|
105
|
+
if (typeof options === "function") {
|
|
106
|
+
this.getCookies(url).then((cookies) => options(null, this.cookiesToSetCookieStrings(cookies))).catch((error) => options(error));
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
if (callback) {
|
|
110
|
+
this.getCookies(url, options).then((cookies) => callback(null, this.cookiesToSetCookieStrings(cookies))).catch((error) => callback(error));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
return this.getCookies(url, options).then((cookies) => this.cookiesToSetCookieStrings(cookies));
|
|
114
|
+
}
|
|
115
|
+
getSetCookieStringsSync(url, options) {
|
|
116
|
+
return this.cookiesToSetCookieStrings(this.getCookiesSync(url, options));
|
|
117
|
+
}
|
|
118
|
+
serialize(callback) {
|
|
119
|
+
if (callback) {
|
|
120
|
+
super.serialize((error, serialized) => {
|
|
121
|
+
if (error) {
|
|
122
|
+
callback(error);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
callback(null, this.toPublicSerializedJar(serialized));
|
|
126
|
+
});
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
return super.serialize().then((serialized) => this.toPublicSerializedJar(serialized));
|
|
130
|
+
}
|
|
131
|
+
serializeSync() {
|
|
132
|
+
return this.toPublicSerializedJar(super.serializeSync());
|
|
133
|
+
}
|
|
134
|
+
toJSON() {
|
|
135
|
+
const json = this.serializeSync();
|
|
136
|
+
if (!json?.cookies)
|
|
137
|
+
return json;
|
|
138
|
+
return json;
|
|
139
|
+
}
|
|
140
|
+
toPublicSerializedJar(json) {
|
|
141
|
+
if (!json?.cookies)
|
|
142
|
+
return json;
|
|
143
|
+
return {
|
|
144
|
+
...json,
|
|
145
|
+
cookies: json.cookies.map((cookie) => ({
|
|
146
|
+
...cookie,
|
|
147
|
+
key: this.toPublicCookieKey(cookie.key)
|
|
148
|
+
}))
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
toStoredCookie(cookie) {
|
|
152
|
+
const parsed = typeof cookie === "string" ? Cookie.parse(cookie) : cookie;
|
|
153
|
+
if (!parsed?.key?.startsWith(HOST_PREFIX)) {
|
|
154
|
+
return cookie;
|
|
155
|
+
}
|
|
156
|
+
const storedCookie = new Cookie(Cookie.fromJSON(parsed.toJSON()));
|
|
157
|
+
storedCookie.key = this.toStoredCookieKey(storedCookie.key);
|
|
158
|
+
return storedCookie;
|
|
159
|
+
}
|
|
160
|
+
toPublicCookies(cookies) {
|
|
161
|
+
return cookies.map((cookie) => this.toPublicCookie(cookie));
|
|
162
|
+
}
|
|
163
|
+
toPublicCookie(cookie) {
|
|
164
|
+
const publicCookie = new Cookie(Cookie.fromJSON(cookie.toJSON()));
|
|
165
|
+
const publicKey = this.toPublicCookieKey(publicCookie.key);
|
|
166
|
+
if (publicKey) {
|
|
167
|
+
publicCookie.key = publicKey;
|
|
168
|
+
}
|
|
169
|
+
return publicCookie;
|
|
170
|
+
}
|
|
171
|
+
toStoredCookieKey(key) {
|
|
172
|
+
return key.startsWith(HOST_PREFIX) ? `${REZO_COOKIE_PREFIX_KEY}${key.slice(2)}` : key;
|
|
173
|
+
}
|
|
174
|
+
toPublicCookieKey(key) {
|
|
175
|
+
if (!key) {
|
|
176
|
+
return key;
|
|
177
|
+
}
|
|
178
|
+
return key.startsWith(STORED_HOST_PREFIX) ? `__${key.slice(REZO_COOKIE_PREFIX_KEY.length)}` : key;
|
|
179
|
+
}
|
|
180
|
+
cookiesToCookieString(cookies) {
|
|
181
|
+
return cookies.map((cookie) => cookie.cookieString()).join("; ");
|
|
182
|
+
}
|
|
183
|
+
cookiesToSetCookieStrings(cookies) {
|
|
184
|
+
return cookies.map((cookie) => cookie.toSetCookieString());
|
|
185
|
+
}
|
|
41
186
|
cookies(url) {
|
|
42
187
|
return this.generateCookies(undefined, url);
|
|
43
188
|
}
|
package/dist/cookies/index.cjs
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Cookie =
|
|
3
|
-
exports.RezoCookie =
|
|
4
|
-
const
|
|
5
|
-
exports.RezoCookieStore =
|
|
6
|
-
const
|
|
7
|
-
exports.RezoCookieJar =
|
|
8
|
-
exports.CookieJar =
|
|
9
|
-
const
|
|
10
|
-
exports.Store =
|
|
1
|
+
const _mod_wfv6rq = require('./cookie.cjs');
|
|
2
|
+
exports.Cookie = _mod_wfv6rq.Cookie;
|
|
3
|
+
exports.RezoCookie = _mod_wfv6rq.RezoCookie;;
|
|
4
|
+
const _mod_y3f199 = require('./cookie-store.cjs');
|
|
5
|
+
exports.RezoCookieStore = _mod_y3f199.RezoCookieStore;;
|
|
6
|
+
const _mod_n0ue96 = require('./cookie-jar.cjs');
|
|
7
|
+
exports.RezoCookieJar = _mod_n0ue96.RezoCookieJar;
|
|
8
|
+
exports.CookieJar = _mod_n0ue96.CookieJar;;
|
|
9
|
+
const _mod_87dkba = require("tough-cookie");
|
|
10
|
+
exports.Store = _mod_87dkba.Store;;
|
package/dist/crawler/index.cjs
CHANGED
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
const
|
|
6
|
-
exports.RobotsTxt =
|
|
7
|
-
const
|
|
8
|
-
exports.FileCacher =
|
|
9
|
-
const
|
|
10
|
-
exports.UrlStore =
|
|
11
|
-
const
|
|
12
|
-
exports.NavigationHistory =
|
|
13
|
-
const
|
|
14
|
-
exports.Oxylabs =
|
|
15
|
-
const
|
|
16
|
-
exports.OXYLABS_BROWSER_TYPES =
|
|
17
|
-
exports.OXYLABS_COMMON_LOCALES =
|
|
18
|
-
exports.OXYLABS_COMMON_GEO_LOCATIONS =
|
|
19
|
-
exports.OXYLABS_US_STATES =
|
|
20
|
-
exports.OXYLABS_EUROPEAN_COUNTRIES =
|
|
21
|
-
exports.OXYLABS_ASIAN_COUNTRIES =
|
|
22
|
-
exports.getRandomOxylabsBrowserType =
|
|
23
|
-
exports.getRandomOxylabsLocale =
|
|
24
|
-
exports.getRandomOxylabsGeoLocation =
|
|
25
|
-
const
|
|
26
|
-
exports.isRestrictedDomain =
|
|
27
|
-
const
|
|
28
|
-
exports.Decodo =
|
|
29
|
-
const
|
|
30
|
-
exports.DECODO_DEVICE_TYPES =
|
|
31
|
-
exports.DECODO_HEADLESS_MODES =
|
|
32
|
-
exports.DECODO_COMMON_LOCALES =
|
|
33
|
-
exports.DECODO_COMMON_COUNTRIES =
|
|
34
|
-
exports.DECODO_EUROPEAN_COUNTRIES =
|
|
35
|
-
exports.DECODO_ASIAN_COUNTRIES =
|
|
36
|
-
exports.DECODO_US_STATES =
|
|
37
|
-
exports.DECODO_COMMON_CITIES =
|
|
38
|
-
exports.getRandomDecodoDeviceType =
|
|
39
|
-
exports.getRandomDecodoLocale =
|
|
40
|
-
exports.getRandomDecodoCountry =
|
|
41
|
-
exports.getRandomDecodoCity =
|
|
42
|
-
exports.generateDecodoSessionId =
|
|
1
|
+
const _mod_5yvoio = require('./crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_5yvoio.Crawler;;
|
|
3
|
+
const _mod_21g7nz = require('./crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_21g7nz.CrawlerOptions;;
|
|
5
|
+
const _mod_0wpmd7 = require('./plugin/robots-txt.cjs');
|
|
6
|
+
exports.RobotsTxt = _mod_0wpmd7.RobotsTxt;;
|
|
7
|
+
const _mod_il7cvs = require('./plugin/file-cacher.cjs');
|
|
8
|
+
exports.FileCacher = _mod_il7cvs.FileCacher;;
|
|
9
|
+
const _mod_ot83h7 = require('./plugin/url-store.cjs');
|
|
10
|
+
exports.UrlStore = _mod_ot83h7.UrlStore;;
|
|
11
|
+
const _mod_lqg3vc = require('./plugin/navigation-history.cjs');
|
|
12
|
+
exports.NavigationHistory = _mod_lqg3vc.NavigationHistory;;
|
|
13
|
+
const _mod_1iw70u = require('./addon/oxylabs/index.cjs');
|
|
14
|
+
exports.Oxylabs = _mod_1iw70u.Oxylabs;;
|
|
15
|
+
const _mod_397yv8 = require('./addon/oxylabs/options.cjs');
|
|
16
|
+
exports.OXYLABS_BROWSER_TYPES = _mod_397yv8.OXYLABS_BROWSER_TYPES;
|
|
17
|
+
exports.OXYLABS_COMMON_LOCALES = _mod_397yv8.OXYLABS_COMMON_LOCALES;
|
|
18
|
+
exports.OXYLABS_COMMON_GEO_LOCATIONS = _mod_397yv8.OXYLABS_COMMON_GEO_LOCATIONS;
|
|
19
|
+
exports.OXYLABS_US_STATES = _mod_397yv8.OXYLABS_US_STATES;
|
|
20
|
+
exports.OXYLABS_EUROPEAN_COUNTRIES = _mod_397yv8.OXYLABS_EUROPEAN_COUNTRIES;
|
|
21
|
+
exports.OXYLABS_ASIAN_COUNTRIES = _mod_397yv8.OXYLABS_ASIAN_COUNTRIES;
|
|
22
|
+
exports.getRandomOxylabsBrowserType = _mod_397yv8.getRandomBrowserType;
|
|
23
|
+
exports.getRandomOxylabsLocale = _mod_397yv8.getRandomLocale;
|
|
24
|
+
exports.getRandomOxylabsGeoLocation = _mod_397yv8.getRandomGeoLocation;;
|
|
25
|
+
const _mod_gjy4k5 = require('./scraper.cjs');
|
|
26
|
+
exports.isRestrictedDomain = _mod_gjy4k5.isRestrictedDomain;;
|
|
27
|
+
const _mod_w3ghk5 = require('./addon/decodo/index.cjs');
|
|
28
|
+
exports.Decodo = _mod_w3ghk5.Decodo;;
|
|
29
|
+
const _mod_pjwi2e = require('./addon/decodo/options.cjs');
|
|
30
|
+
exports.DECODO_DEVICE_TYPES = _mod_pjwi2e.DECODO_DEVICE_TYPES;
|
|
31
|
+
exports.DECODO_HEADLESS_MODES = _mod_pjwi2e.DECODO_HEADLESS_MODES;
|
|
32
|
+
exports.DECODO_COMMON_LOCALES = _mod_pjwi2e.DECODO_COMMON_LOCALES;
|
|
33
|
+
exports.DECODO_COMMON_COUNTRIES = _mod_pjwi2e.DECODO_COMMON_COUNTRIES;
|
|
34
|
+
exports.DECODO_EUROPEAN_COUNTRIES = _mod_pjwi2e.DECODO_EUROPEAN_COUNTRIES;
|
|
35
|
+
exports.DECODO_ASIAN_COUNTRIES = _mod_pjwi2e.DECODO_ASIAN_COUNTRIES;
|
|
36
|
+
exports.DECODO_US_STATES = _mod_pjwi2e.DECODO_US_STATES;
|
|
37
|
+
exports.DECODO_COMMON_CITIES = _mod_pjwi2e.DECODO_COMMON_CITIES;
|
|
38
|
+
exports.getRandomDecodoDeviceType = _mod_pjwi2e.getRandomDeviceType;
|
|
39
|
+
exports.getRandomDecodoLocale = _mod_pjwi2e.getRandomLocale;
|
|
40
|
+
exports.getRandomDecodoCountry = _mod_pjwi2e.getRandomCountry;
|
|
41
|
+
exports.getRandomDecodoCity = _mod_pjwi2e.getRandomCity;
|
|
42
|
+
exports.generateDecodoSessionId = _mod_pjwi2e.generateSessionId;;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var
|
|
1
|
+
var e=require("./file-cacher.cjs");exports.FileCacher=e.FileCacher;var r=require("./url-store.cjs");exports.UrlStore=r.UrlStore;var o=require("./result-stream.cjs");exports.ResultStream=o.ResultStream;var t=require("./memory-monitor.cjs");exports.MemoryMonitor=t.MemoryMonitor;var a=require("./health-metrics.cjs");exports.HealthMetrics=a.HealthMetrics;var i=require("./capped-map.cjs");exports.CappedMap=i.CappedMap;var c=require("./capped-array.cjs");exports.CappedArray=c.CappedArray;
|