rezo 1.0.36 → 1.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +320 -9
- package/dist/adapters/curl.js +320 -9
- package/dist/adapters/entries/curl.d.ts +20 -2
- package/dist/adapters/entries/fetch.d.ts +20 -2
- package/dist/adapters/entries/http.d.ts +20 -2
- package/dist/adapters/entries/http2.d.ts +20 -2
- package/dist/adapters/entries/react-native.d.ts +20 -2
- package/dist/adapters/entries/xhr.d.ts +20 -2
- package/dist/adapters/fetch.cjs +10 -2
- package/dist/adapters/fetch.js +10 -2
- package/dist/adapters/http.cjs +206 -35
- package/dist/adapters/http.js +206 -35
- package/dist/adapters/http2.cjs +10 -2
- package/dist/adapters/http2.js +10 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/react-native.cjs +10 -2
- package/dist/adapters/react-native.js +10 -2
- package/dist/adapters/xhr.cjs +9 -1
- package/dist/adapters/xhr.js +9 -1
- package/dist/cache/index.cjs +13 -13
- package/dist/crawler.d.ts +20 -2
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +20 -2
- package/dist/platform/browser.d.ts +20 -2
- package/dist/platform/bun.d.ts +20 -2
- package/dist/platform/deno.d.ts +20 -2
- package/dist/platform/node.d.ts +20 -2
- package/dist/platform/react-native.d.ts +20 -2
- package/dist/platform/worker.d.ts +20 -2
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +204 -0
- package/dist/utils/agent-pool.js +201 -0
- package/dist/utils/http-config.cjs +24 -7
- package/dist/utils/http-config.js +24 -7
- package/dist/utils/index.cjs +2 -0
- package/dist/utils/index.js +2 -0
- package/dist/utils/staged-timeout.cjs +143 -0
- package/dist/utils/staged-timeout.js +139 -0
- package/package.json +1 -1
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
const http = require("node:http");
|
|
2
|
+
const https = require("node:https");
|
|
3
|
+
const dns = require("node:dns");
|
|
4
|
+
const { getGlobalDNSCache } = require('../cache/dns-cache.cjs');
|
|
5
|
+
const DEFAULT_CONFIG = {
|
|
6
|
+
keepAlive: true,
|
|
7
|
+
keepAliveMsecs: 1000,
|
|
8
|
+
maxSockets: 256,
|
|
9
|
+
maxFreeSockets: 64,
|
|
10
|
+
timeout: 30000,
|
|
11
|
+
scheduling: "lifo",
|
|
12
|
+
dnsCache: true,
|
|
13
|
+
idleEvictionMs: 60000
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
class AgentPool {
|
|
17
|
+
httpAgents = new Map;
|
|
18
|
+
httpsAgents = new Map;
|
|
19
|
+
config;
|
|
20
|
+
dnsCache = null;
|
|
21
|
+
evictionTimer = null;
|
|
22
|
+
constructor(config = {}) {
|
|
23
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
24
|
+
if (this.config.dnsCache) {
|
|
25
|
+
const dnsCacheOptions = typeof this.config.dnsCache === "object" ? this.config.dnsCache : {};
|
|
26
|
+
this.dnsCache = getGlobalDNSCache(dnsCacheOptions);
|
|
27
|
+
}
|
|
28
|
+
if (this.config.idleEvictionMs > 0) {
|
|
29
|
+
this.startEvictionTimer();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
buildAgentKey(options = {}) {
|
|
33
|
+
const parts = [];
|
|
34
|
+
if (options.proxy) {
|
|
35
|
+
parts.push(`proxy:${options.proxy.protocol}://${options.proxy.host}:${options.proxy.port}`);
|
|
36
|
+
if (options.proxy.auth) {
|
|
37
|
+
parts.push(`auth:${options.proxy.auth.username}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (options.rejectUnauthorized === false) {
|
|
41
|
+
parts.push("insecure");
|
|
42
|
+
}
|
|
43
|
+
if (options.ca) {
|
|
44
|
+
parts.push(`ca:${typeof options.ca === "string" ? options.ca.slice(0, 32) : "buffer"}`);
|
|
45
|
+
}
|
|
46
|
+
if (options.cert) {
|
|
47
|
+
parts.push(`cert:${typeof options.cert === "string" ? options.cert.slice(0, 32) : "buffer"}`);
|
|
48
|
+
}
|
|
49
|
+
if (options.servername) {
|
|
50
|
+
parts.push(`sni:${options.servername}`);
|
|
51
|
+
}
|
|
52
|
+
if (options.localAddress) {
|
|
53
|
+
parts.push(`local:${options.localAddress}`);
|
|
54
|
+
}
|
|
55
|
+
return parts.length > 0 ? parts.join("|") : "default";
|
|
56
|
+
}
|
|
57
|
+
createLookupFunction() {
|
|
58
|
+
if (!this.dnsCache) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const cache = this.dnsCache;
|
|
62
|
+
return (hostname, options, callback) => {
|
|
63
|
+
const family = options.family === 6 ? 6 : options.family === 4 ? 4 : undefined;
|
|
64
|
+
cache.lookup(hostname, family).then((result) => {
|
|
65
|
+
if (result) {
|
|
66
|
+
callback(null, result.address, result.family);
|
|
67
|
+
} else {
|
|
68
|
+
dns.lookup(hostname, options, callback);
|
|
69
|
+
}
|
|
70
|
+
}).catch(() => {
|
|
71
|
+
dns.lookup(hostname, options, callback);
|
|
72
|
+
});
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
createHttpAgent(key) {
|
|
76
|
+
const agentOptions = {
|
|
77
|
+
keepAlive: this.config.keepAlive,
|
|
78
|
+
keepAliveMsecs: this.config.keepAliveMsecs,
|
|
79
|
+
maxSockets: this.config.maxSockets,
|
|
80
|
+
maxFreeSockets: this.config.maxFreeSockets,
|
|
81
|
+
timeout: this.config.timeout,
|
|
82
|
+
scheduling: this.config.scheduling
|
|
83
|
+
};
|
|
84
|
+
const lookup = this.createLookupFunction();
|
|
85
|
+
if (lookup) {
|
|
86
|
+
agentOptions.lookup = lookup;
|
|
87
|
+
}
|
|
88
|
+
return new http.Agent(agentOptions);
|
|
89
|
+
}
|
|
90
|
+
createHttpsAgent(key, tlsOptions) {
|
|
91
|
+
const agentOptions = {
|
|
92
|
+
keepAlive: this.config.keepAlive,
|
|
93
|
+
keepAliveMsecs: this.config.keepAliveMsecs,
|
|
94
|
+
maxSockets: this.config.maxSockets,
|
|
95
|
+
maxFreeSockets: this.config.maxFreeSockets,
|
|
96
|
+
timeout: this.config.timeout,
|
|
97
|
+
scheduling: this.config.scheduling,
|
|
98
|
+
...tlsOptions
|
|
99
|
+
};
|
|
100
|
+
const lookup = this.createLookupFunction();
|
|
101
|
+
if (lookup) {
|
|
102
|
+
agentOptions.lookup = lookup;
|
|
103
|
+
}
|
|
104
|
+
return new https.Agent(agentOptions);
|
|
105
|
+
}
|
|
106
|
+
getHttpAgent(options) {
|
|
107
|
+
const key = this.buildAgentKey(options);
|
|
108
|
+
let pooled = this.httpAgents.get(key);
|
|
109
|
+
if (pooled) {
|
|
110
|
+
pooled.lastUsed = Date.now();
|
|
111
|
+
return pooled.agent;
|
|
112
|
+
}
|
|
113
|
+
const agent = this.createHttpAgent(key);
|
|
114
|
+
pooled = { agent, lastUsed: Date.now(), key };
|
|
115
|
+
this.httpAgents.set(key, pooled);
|
|
116
|
+
return agent;
|
|
117
|
+
}
|
|
118
|
+
getHttpsAgent(options) {
|
|
119
|
+
const key = this.buildAgentKey(options);
|
|
120
|
+
let pooled = this.httpsAgents.get(key);
|
|
121
|
+
if (pooled) {
|
|
122
|
+
pooled.lastUsed = Date.now();
|
|
123
|
+
return pooled.agent;
|
|
124
|
+
}
|
|
125
|
+
const agent = this.createHttpsAgent(key, options);
|
|
126
|
+
pooled = { agent, lastUsed: Date.now(), key };
|
|
127
|
+
this.httpsAgents.set(key, pooled);
|
|
128
|
+
return agent;
|
|
129
|
+
}
|
|
130
|
+
startEvictionTimer() {
|
|
131
|
+
if (this.evictionTimer) {
|
|
132
|
+
clearInterval(this.evictionTimer);
|
|
133
|
+
}
|
|
134
|
+
this.evictionTimer = setInterval(() => {
|
|
135
|
+
this.evictIdleAgents();
|
|
136
|
+
}, Math.min(this.config.idleEvictionMs / 2, 30000));
|
|
137
|
+
if (this.evictionTimer.unref) {
|
|
138
|
+
this.evictionTimer.unref();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
evictIdleAgents() {
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
const threshold = now - this.config.idleEvictionMs;
|
|
144
|
+
for (const [key, pooled] of this.httpAgents) {
|
|
145
|
+
if (pooled.lastUsed < threshold) {
|
|
146
|
+
pooled.agent.destroy();
|
|
147
|
+
this.httpAgents.delete(key);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (const [key, pooled] of this.httpsAgents) {
|
|
151
|
+
if (pooled.lastUsed < threshold) {
|
|
152
|
+
pooled.agent.destroy();
|
|
153
|
+
this.httpsAgents.delete(key);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
getStats() {
|
|
158
|
+
return {
|
|
159
|
+
httpAgents: this.httpAgents.size,
|
|
160
|
+
httpsAgents: this.httpsAgents.size,
|
|
161
|
+
dnsCacheSize: this.dnsCache?.size ?? 0,
|
|
162
|
+
config: this.config
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
destroy() {
|
|
166
|
+
if (this.evictionTimer) {
|
|
167
|
+
clearInterval(this.evictionTimer);
|
|
168
|
+
this.evictionTimer = null;
|
|
169
|
+
}
|
|
170
|
+
for (const pooled of this.httpAgents.values()) {
|
|
171
|
+
pooled.agent.destroy();
|
|
172
|
+
}
|
|
173
|
+
this.httpAgents.clear();
|
|
174
|
+
for (const pooled of this.httpsAgents.values()) {
|
|
175
|
+
pooled.agent.destroy();
|
|
176
|
+
}
|
|
177
|
+
this.httpsAgents.clear();
|
|
178
|
+
}
|
|
179
|
+
clear() {
|
|
180
|
+
this.destroy();
|
|
181
|
+
if (this.config.idleEvictionMs > 0) {
|
|
182
|
+
this.startEvictionTimer();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
let globalAgentPool = null;
|
|
187
|
+
function getGlobalAgentPool(config) {
|
|
188
|
+
if (!globalAgentPool) {
|
|
189
|
+
globalAgentPool = new AgentPool(config);
|
|
190
|
+
}
|
|
191
|
+
return globalAgentPool;
|
|
192
|
+
}
|
|
193
|
+
function resetGlobalAgentPool() {
|
|
194
|
+
if (globalAgentPool) {
|
|
195
|
+
globalAgentPool.destroy();
|
|
196
|
+
}
|
|
197
|
+
globalAgentPool = null;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
exports.AgentPool = AgentPool;
|
|
201
|
+
exports.getGlobalAgentPool = getGlobalAgentPool;
|
|
202
|
+
exports.resetGlobalAgentPool = resetGlobalAgentPool;
|
|
203
|
+
exports.default = AgentPool;
|
|
204
|
+
module.exports = Object.assign(AgentPool, exports);
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import * as http from "node:http";
|
|
2
|
+
import * as https from "node:https";
|
|
3
|
+
import dns from "node:dns";
|
|
4
|
+
import { getGlobalDNSCache } from '../cache/dns-cache.js';
|
|
5
|
+
const DEFAULT_CONFIG = {
|
|
6
|
+
keepAlive: true,
|
|
7
|
+
keepAliveMsecs: 1000,
|
|
8
|
+
maxSockets: 256,
|
|
9
|
+
maxFreeSockets: 64,
|
|
10
|
+
timeout: 30000,
|
|
11
|
+
scheduling: "lifo",
|
|
12
|
+
dnsCache: true,
|
|
13
|
+
idleEvictionMs: 60000
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
class AgentPool {
|
|
17
|
+
httpAgents = new Map;
|
|
18
|
+
httpsAgents = new Map;
|
|
19
|
+
config;
|
|
20
|
+
dnsCache = null;
|
|
21
|
+
evictionTimer = null;
|
|
22
|
+
constructor(config = {}) {
|
|
23
|
+
this.config = { ...DEFAULT_CONFIG, ...config };
|
|
24
|
+
if (this.config.dnsCache) {
|
|
25
|
+
const dnsCacheOptions = typeof this.config.dnsCache === "object" ? this.config.dnsCache : {};
|
|
26
|
+
this.dnsCache = getGlobalDNSCache(dnsCacheOptions);
|
|
27
|
+
}
|
|
28
|
+
if (this.config.idleEvictionMs > 0) {
|
|
29
|
+
this.startEvictionTimer();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
buildAgentKey(options = {}) {
|
|
33
|
+
const parts = [];
|
|
34
|
+
if (options.proxy) {
|
|
35
|
+
parts.push(`proxy:${options.proxy.protocol}://${options.proxy.host}:${options.proxy.port}`);
|
|
36
|
+
if (options.proxy.auth) {
|
|
37
|
+
parts.push(`auth:${options.proxy.auth.username}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (options.rejectUnauthorized === false) {
|
|
41
|
+
parts.push("insecure");
|
|
42
|
+
}
|
|
43
|
+
if (options.ca) {
|
|
44
|
+
parts.push(`ca:${typeof options.ca === "string" ? options.ca.slice(0, 32) : "buffer"}`);
|
|
45
|
+
}
|
|
46
|
+
if (options.cert) {
|
|
47
|
+
parts.push(`cert:${typeof options.cert === "string" ? options.cert.slice(0, 32) : "buffer"}`);
|
|
48
|
+
}
|
|
49
|
+
if (options.servername) {
|
|
50
|
+
parts.push(`sni:${options.servername}`);
|
|
51
|
+
}
|
|
52
|
+
if (options.localAddress) {
|
|
53
|
+
parts.push(`local:${options.localAddress}`);
|
|
54
|
+
}
|
|
55
|
+
return parts.length > 0 ? parts.join("|") : "default";
|
|
56
|
+
}
|
|
57
|
+
createLookupFunction() {
|
|
58
|
+
if (!this.dnsCache) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const cache = this.dnsCache;
|
|
62
|
+
return (hostname, options, callback) => {
|
|
63
|
+
const family = options.family === 6 ? 6 : options.family === 4 ? 4 : undefined;
|
|
64
|
+
cache.lookup(hostname, family).then((result) => {
|
|
65
|
+
if (result) {
|
|
66
|
+
callback(null, result.address, result.family);
|
|
67
|
+
} else {
|
|
68
|
+
dns.lookup(hostname, options, callback);
|
|
69
|
+
}
|
|
70
|
+
}).catch(() => {
|
|
71
|
+
dns.lookup(hostname, options, callback);
|
|
72
|
+
});
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
createHttpAgent(key) {
|
|
76
|
+
const agentOptions = {
|
|
77
|
+
keepAlive: this.config.keepAlive,
|
|
78
|
+
keepAliveMsecs: this.config.keepAliveMsecs,
|
|
79
|
+
maxSockets: this.config.maxSockets,
|
|
80
|
+
maxFreeSockets: this.config.maxFreeSockets,
|
|
81
|
+
timeout: this.config.timeout,
|
|
82
|
+
scheduling: this.config.scheduling
|
|
83
|
+
};
|
|
84
|
+
const lookup = this.createLookupFunction();
|
|
85
|
+
if (lookup) {
|
|
86
|
+
agentOptions.lookup = lookup;
|
|
87
|
+
}
|
|
88
|
+
return new http.Agent(agentOptions);
|
|
89
|
+
}
|
|
90
|
+
createHttpsAgent(key, tlsOptions) {
|
|
91
|
+
const agentOptions = {
|
|
92
|
+
keepAlive: this.config.keepAlive,
|
|
93
|
+
keepAliveMsecs: this.config.keepAliveMsecs,
|
|
94
|
+
maxSockets: this.config.maxSockets,
|
|
95
|
+
maxFreeSockets: this.config.maxFreeSockets,
|
|
96
|
+
timeout: this.config.timeout,
|
|
97
|
+
scheduling: this.config.scheduling,
|
|
98
|
+
...tlsOptions
|
|
99
|
+
};
|
|
100
|
+
const lookup = this.createLookupFunction();
|
|
101
|
+
if (lookup) {
|
|
102
|
+
agentOptions.lookup = lookup;
|
|
103
|
+
}
|
|
104
|
+
return new https.Agent(agentOptions);
|
|
105
|
+
}
|
|
106
|
+
getHttpAgent(options) {
|
|
107
|
+
const key = this.buildAgentKey(options);
|
|
108
|
+
let pooled = this.httpAgents.get(key);
|
|
109
|
+
if (pooled) {
|
|
110
|
+
pooled.lastUsed = Date.now();
|
|
111
|
+
return pooled.agent;
|
|
112
|
+
}
|
|
113
|
+
const agent = this.createHttpAgent(key);
|
|
114
|
+
pooled = { agent, lastUsed: Date.now(), key };
|
|
115
|
+
this.httpAgents.set(key, pooled);
|
|
116
|
+
return agent;
|
|
117
|
+
}
|
|
118
|
+
getHttpsAgent(options) {
|
|
119
|
+
const key = this.buildAgentKey(options);
|
|
120
|
+
let pooled = this.httpsAgents.get(key);
|
|
121
|
+
if (pooled) {
|
|
122
|
+
pooled.lastUsed = Date.now();
|
|
123
|
+
return pooled.agent;
|
|
124
|
+
}
|
|
125
|
+
const agent = this.createHttpsAgent(key, options);
|
|
126
|
+
pooled = { agent, lastUsed: Date.now(), key };
|
|
127
|
+
this.httpsAgents.set(key, pooled);
|
|
128
|
+
return agent;
|
|
129
|
+
}
|
|
130
|
+
startEvictionTimer() {
|
|
131
|
+
if (this.evictionTimer) {
|
|
132
|
+
clearInterval(this.evictionTimer);
|
|
133
|
+
}
|
|
134
|
+
this.evictionTimer = setInterval(() => {
|
|
135
|
+
this.evictIdleAgents();
|
|
136
|
+
}, Math.min(this.config.idleEvictionMs / 2, 30000));
|
|
137
|
+
if (this.evictionTimer.unref) {
|
|
138
|
+
this.evictionTimer.unref();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
evictIdleAgents() {
|
|
142
|
+
const now = Date.now();
|
|
143
|
+
const threshold = now - this.config.idleEvictionMs;
|
|
144
|
+
for (const [key, pooled] of this.httpAgents) {
|
|
145
|
+
if (pooled.lastUsed < threshold) {
|
|
146
|
+
pooled.agent.destroy();
|
|
147
|
+
this.httpAgents.delete(key);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (const [key, pooled] of this.httpsAgents) {
|
|
151
|
+
if (pooled.lastUsed < threshold) {
|
|
152
|
+
pooled.agent.destroy();
|
|
153
|
+
this.httpsAgents.delete(key);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
getStats() {
|
|
158
|
+
return {
|
|
159
|
+
httpAgents: this.httpAgents.size,
|
|
160
|
+
httpsAgents: this.httpsAgents.size,
|
|
161
|
+
dnsCacheSize: this.dnsCache?.size ?? 0,
|
|
162
|
+
config: this.config
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
destroy() {
|
|
166
|
+
if (this.evictionTimer) {
|
|
167
|
+
clearInterval(this.evictionTimer);
|
|
168
|
+
this.evictionTimer = null;
|
|
169
|
+
}
|
|
170
|
+
for (const pooled of this.httpAgents.values()) {
|
|
171
|
+
pooled.agent.destroy();
|
|
172
|
+
}
|
|
173
|
+
this.httpAgents.clear();
|
|
174
|
+
for (const pooled of this.httpsAgents.values()) {
|
|
175
|
+
pooled.agent.destroy();
|
|
176
|
+
}
|
|
177
|
+
this.httpsAgents.clear();
|
|
178
|
+
}
|
|
179
|
+
clear() {
|
|
180
|
+
this.destroy();
|
|
181
|
+
if (this.config.idleEvictionMs > 0) {
|
|
182
|
+
this.startEvictionTimer();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
let globalAgentPool = null;
|
|
187
|
+
export function getGlobalAgentPool(config) {
|
|
188
|
+
if (!globalAgentPool) {
|
|
189
|
+
globalAgentPool = new AgentPool(config);
|
|
190
|
+
}
|
|
191
|
+
return globalAgentPool;
|
|
192
|
+
}
|
|
193
|
+
export function resetGlobalAgentPool() {
|
|
194
|
+
if (globalAgentPool) {
|
|
195
|
+
globalAgentPool.destroy();
|
|
196
|
+
}
|
|
197
|
+
globalAgentPool = null;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export { AgentPool };
|
|
201
|
+
export default AgentPool;
|
|
@@ -191,7 +191,7 @@ async function getDefaultConfig(config = {}, proxyManager) {
|
|
|
191
191
|
retry: config.retry,
|
|
192
192
|
proxy: config.proxy,
|
|
193
193
|
followRedirects: config.followRedirects,
|
|
194
|
-
useCookies: config.enableCookieJar,
|
|
194
|
+
useCookies: config.disableCookieJar === true ? false : config.enableCookieJar !== false,
|
|
195
195
|
fs: await getFS(),
|
|
196
196
|
timeout: config.timeout ?? config.requestTimeout,
|
|
197
197
|
hooks: config.hooks,
|
|
@@ -236,7 +236,23 @@ function prepareHTTPOptions(options, jar, addedOptions, config) {
|
|
|
236
236
|
if (!options.responseType) {
|
|
237
237
|
options.responseType = "auto";
|
|
238
238
|
}
|
|
239
|
-
|
|
239
|
+
let cookieJar;
|
|
240
|
+
if (!config.disableCookieJar) {
|
|
241
|
+
cookieJar = jar;
|
|
242
|
+
if (addedOptions.isRedirected && config.responseCookies?.array && config.responseCookies.array.length > 0) {
|
|
243
|
+
const cookieSetUrl = addedOptions.redirectedUrl || (options.fullUrl || (options.url instanceof URL ? options.url.href : String(options.url)));
|
|
244
|
+
jar.setCookiesSync(config.responseCookies.array, cookieSetUrl);
|
|
245
|
+
}
|
|
246
|
+
} else {
|
|
247
|
+
cookieJar = new RezoCookieJar;
|
|
248
|
+
const cookieSetUrl = addedOptions.isRedirected && addedOptions.redirectedUrl ? addedOptions.redirectedUrl : options.fullUrl || (options.url instanceof URL ? options.url.href : String(options.url));
|
|
249
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
250
|
+
cookieJar.setCookiesSync(config.requestCookies, cookieSetUrl);
|
|
251
|
+
}
|
|
252
|
+
if (addedOptions.isRedirected && config.responseCookies?.array && config.responseCookies.array.length > 0) {
|
|
253
|
+
cookieJar.setCookiesSync(config.responseCookies.array, cookieSetUrl);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
240
256
|
let requestCookies = [];
|
|
241
257
|
fetchOptions.method = options.method;
|
|
242
258
|
if (options.fullUrl) {
|
|
@@ -584,7 +600,7 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
|
|
|
584
600
|
maxRedirects,
|
|
585
601
|
retryAttempts: 0,
|
|
586
602
|
timeout: typeof requestOptions.timeout === "number" ? requestOptions.timeout : null,
|
|
587
|
-
|
|
603
|
+
disableCookieJar: typeof defaultOptions.disableCookieJar === "boolean" ? defaultOptions.disableCookieJar : defaultOptions.enableCookieJar === false,
|
|
588
604
|
withCredentials: typeof withCredentials === "boolean" ? withCredentials : typeof defaultOptions.withCredentials === "boolean" ? defaultOptions.withCredentials : false,
|
|
589
605
|
useCookies: typeof requestOptions.useCookies === "boolean" ? requestOptions.useCookies : true,
|
|
590
606
|
cookieJar: requestOptions.jar || jar,
|
|
@@ -609,7 +625,8 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
|
|
|
609
625
|
rejectUnauthorized: typeof rejectUnauthorized === "boolean" ? rejectUnauthorized : true,
|
|
610
626
|
decompress: typeof requestOptions.decompress === "boolean" ? requestOptions.decompress : typeof defaultOptions.decompress === "boolean" ? defaultOptions.decompress : true,
|
|
611
627
|
debug: requestOptions.debug === true || defaultOptions.debug === true,
|
|
612
|
-
trackUrl: requestOptions.trackUrl === true || defaultOptions.trackUrl === true
|
|
628
|
+
trackUrl: requestOptions.trackUrl === true || defaultOptions.trackUrl === true,
|
|
629
|
+
originalBody: requestOptions.body
|
|
613
630
|
};
|
|
614
631
|
config.setSignal = setSignal.bind(config);
|
|
615
632
|
if (requestOptions.encoding || defaultOptions.encoding) {
|
|
@@ -636,10 +653,10 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
|
|
|
636
653
|
const baseHooks = mergeHooks(createDefaultHooks(), defaultOptions.hooks || {});
|
|
637
654
|
const mergedHooks = mergeHooks(baseHooks, requestOptions.hooks || {});
|
|
638
655
|
config.hooks = serializeHooks(mergedHooks);
|
|
639
|
-
if (typeof config.
|
|
640
|
-
config.
|
|
656
|
+
if (typeof config.disableCookieJar !== "boolean") {
|
|
657
|
+
config.disableCookieJar = false;
|
|
641
658
|
}
|
|
642
|
-
if (
|
|
659
|
+
if (config.disableCookieJar) {
|
|
643
660
|
config.cookieJar = new RezoCookieJar;
|
|
644
661
|
}
|
|
645
662
|
if (options.httpAgent) {
|
|
@@ -191,7 +191,7 @@ export async function getDefaultConfig(config = {}, proxyManager) {
|
|
|
191
191
|
retry: config.retry,
|
|
192
192
|
proxy: config.proxy,
|
|
193
193
|
followRedirects: config.followRedirects,
|
|
194
|
-
useCookies: config.enableCookieJar,
|
|
194
|
+
useCookies: config.disableCookieJar === true ? false : config.enableCookieJar !== false,
|
|
195
195
|
fs: await getFS(),
|
|
196
196
|
timeout: config.timeout ?? config.requestTimeout,
|
|
197
197
|
hooks: config.hooks,
|
|
@@ -236,7 +236,23 @@ export function prepareHTTPOptions(options, jar, addedOptions, config) {
|
|
|
236
236
|
if (!options.responseType) {
|
|
237
237
|
options.responseType = "auto";
|
|
238
238
|
}
|
|
239
|
-
|
|
239
|
+
let cookieJar;
|
|
240
|
+
if (!config.disableCookieJar) {
|
|
241
|
+
cookieJar = jar;
|
|
242
|
+
if (addedOptions.isRedirected && config.responseCookies?.array && config.responseCookies.array.length > 0) {
|
|
243
|
+
const cookieSetUrl = addedOptions.redirectedUrl || (options.fullUrl || (options.url instanceof URL ? options.url.href : String(options.url)));
|
|
244
|
+
jar.setCookiesSync(config.responseCookies.array, cookieSetUrl);
|
|
245
|
+
}
|
|
246
|
+
} else {
|
|
247
|
+
cookieJar = new RezoCookieJar;
|
|
248
|
+
const cookieSetUrl = addedOptions.isRedirected && addedOptions.redirectedUrl ? addedOptions.redirectedUrl : options.fullUrl || (options.url instanceof URL ? options.url.href : String(options.url));
|
|
249
|
+
if (config.requestCookies && config.requestCookies.length > 0) {
|
|
250
|
+
cookieJar.setCookiesSync(config.requestCookies, cookieSetUrl);
|
|
251
|
+
}
|
|
252
|
+
if (addedOptions.isRedirected && config.responseCookies?.array && config.responseCookies.array.length > 0) {
|
|
253
|
+
cookieJar.setCookiesSync(config.responseCookies.array, cookieSetUrl);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
240
256
|
let requestCookies = [];
|
|
241
257
|
fetchOptions.method = options.method;
|
|
242
258
|
if (options.fullUrl) {
|
|
@@ -584,7 +600,7 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
|
|
|
584
600
|
maxRedirects,
|
|
585
601
|
retryAttempts: 0,
|
|
586
602
|
timeout: typeof requestOptions.timeout === "number" ? requestOptions.timeout : null,
|
|
587
|
-
|
|
603
|
+
disableCookieJar: typeof defaultOptions.disableCookieJar === "boolean" ? defaultOptions.disableCookieJar : defaultOptions.enableCookieJar === false,
|
|
588
604
|
withCredentials: typeof withCredentials === "boolean" ? withCredentials : typeof defaultOptions.withCredentials === "boolean" ? defaultOptions.withCredentials : false,
|
|
589
605
|
useCookies: typeof requestOptions.useCookies === "boolean" ? requestOptions.useCookies : true,
|
|
590
606
|
cookieJar: requestOptions.jar || jar,
|
|
@@ -609,7 +625,8 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
|
|
|
609
625
|
rejectUnauthorized: typeof rejectUnauthorized === "boolean" ? rejectUnauthorized : true,
|
|
610
626
|
decompress: typeof requestOptions.decompress === "boolean" ? requestOptions.decompress : typeof defaultOptions.decompress === "boolean" ? defaultOptions.decompress : true,
|
|
611
627
|
debug: requestOptions.debug === true || defaultOptions.debug === true,
|
|
612
|
-
trackUrl: requestOptions.trackUrl === true || defaultOptions.trackUrl === true
|
|
628
|
+
trackUrl: requestOptions.trackUrl === true || defaultOptions.trackUrl === true,
|
|
629
|
+
originalBody: requestOptions.body
|
|
613
630
|
};
|
|
614
631
|
config.setSignal = setSignal.bind(config);
|
|
615
632
|
if (requestOptions.encoding || defaultOptions.encoding) {
|
|
@@ -636,10 +653,10 @@ As a workaround, process.env.NODE_TLS_REJECT_UNAUTHORIZED is being set to '0'.
|
|
|
636
653
|
const baseHooks = mergeHooks(createDefaultHooks(), defaultOptions.hooks || {});
|
|
637
654
|
const mergedHooks = mergeHooks(baseHooks, requestOptions.hooks || {});
|
|
638
655
|
config.hooks = serializeHooks(mergedHooks);
|
|
639
|
-
if (typeof config.
|
|
640
|
-
config.
|
|
656
|
+
if (typeof config.disableCookieJar !== "boolean") {
|
|
657
|
+
config.disableCookieJar = false;
|
|
641
658
|
}
|
|
642
|
-
if (
|
|
659
|
+
if (config.disableCookieJar) {
|
|
643
660
|
config.cookieJar = new RezoCookieJar;
|
|
644
661
|
}
|
|
645
662
|
if (options.httpAgent) {
|
package/dist/utils/index.cjs
CHANGED
|
@@ -5,3 +5,5 @@ Object.assign(exports, require('./form-data.cjs'));
|
|
|
5
5
|
Object.assign(exports, require('./headers.cjs'));
|
|
6
6
|
Object.assign(exports, require('./http-config.cjs'));
|
|
7
7
|
Object.assign(exports, require('./tools.cjs'));
|
|
8
|
+
Object.assign(exports, require('./agent-pool.cjs'));
|
|
9
|
+
Object.assign(exports, require('./staged-timeout.cjs'));
|
package/dist/utils/index.js
CHANGED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
2
|
+
|
|
3
|
+
class StagedTimeoutManager {
|
|
4
|
+
phases = new Map;
|
|
5
|
+
socket = null;
|
|
6
|
+
request = null;
|
|
7
|
+
abortController = null;
|
|
8
|
+
config = null;
|
|
9
|
+
requestConfig = null;
|
|
10
|
+
onTimeout = null;
|
|
11
|
+
constructor(timeoutConfig, config, requestConfig) {
|
|
12
|
+
this.config = config || null;
|
|
13
|
+
this.requestConfig = requestConfig || null;
|
|
14
|
+
if (timeoutConfig.connect) {
|
|
15
|
+
this.phases.set("connect", {
|
|
16
|
+
name: "connect",
|
|
17
|
+
timeout: timeoutConfig.connect,
|
|
18
|
+
timer: null,
|
|
19
|
+
startTime: 0
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
if (timeoutConfig.headers) {
|
|
23
|
+
this.phases.set("headers", {
|
|
24
|
+
name: "headers",
|
|
25
|
+
timeout: timeoutConfig.headers,
|
|
26
|
+
timer: null,
|
|
27
|
+
startTime: 0
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
if (timeoutConfig.body) {
|
|
31
|
+
this.phases.set("body", {
|
|
32
|
+
name: "body",
|
|
33
|
+
timeout: timeoutConfig.body,
|
|
34
|
+
timer: null,
|
|
35
|
+
startTime: 0
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
if (timeoutConfig.total) {
|
|
39
|
+
this.phases.set("total", {
|
|
40
|
+
name: "total",
|
|
41
|
+
timeout: timeoutConfig.total,
|
|
42
|
+
timer: null,
|
|
43
|
+
startTime: 0
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
setSocket(socket) {
|
|
48
|
+
this.socket = socket;
|
|
49
|
+
}
|
|
50
|
+
setRequest(request) {
|
|
51
|
+
this.request = request;
|
|
52
|
+
}
|
|
53
|
+
setAbortController(controller) {
|
|
54
|
+
this.abortController = controller;
|
|
55
|
+
}
|
|
56
|
+
setTimeoutCallback(callback) {
|
|
57
|
+
this.onTimeout = callback;
|
|
58
|
+
}
|
|
59
|
+
startPhase(phaseName) {
|
|
60
|
+
const phase = this.phases.get(phaseName);
|
|
61
|
+
if (!phase)
|
|
62
|
+
return;
|
|
63
|
+
this.clearPhase(phaseName);
|
|
64
|
+
phase.startTime = Date.now();
|
|
65
|
+
phase.timer = setTimeout(() => {
|
|
66
|
+
this.handleTimeout(phaseName);
|
|
67
|
+
}, phase.timeout);
|
|
68
|
+
}
|
|
69
|
+
clearPhase(phaseName) {
|
|
70
|
+
const phase = this.phases.get(phaseName);
|
|
71
|
+
if (phase?.timer) {
|
|
72
|
+
clearTimeout(phase.timer);
|
|
73
|
+
phase.timer = null;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
clearAll() {
|
|
77
|
+
for (const phaseName of this.phases.keys()) {
|
|
78
|
+
this.clearPhase(phaseName);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
handleTimeout(phaseName) {
|
|
82
|
+
const phase = this.phases.get(phaseName);
|
|
83
|
+
if (!phase)
|
|
84
|
+
return;
|
|
85
|
+
const elapsed = Date.now() - phase.startTime;
|
|
86
|
+
if (this.socket) {
|
|
87
|
+
try {
|
|
88
|
+
this.socket.destroy();
|
|
89
|
+
} catch {}
|
|
90
|
+
}
|
|
91
|
+
if (this.request) {
|
|
92
|
+
try {
|
|
93
|
+
this.request.destroy();
|
|
94
|
+
} catch {}
|
|
95
|
+
}
|
|
96
|
+
if (this.abortController) {
|
|
97
|
+
try {
|
|
98
|
+
this.abortController.abort();
|
|
99
|
+
} catch {}
|
|
100
|
+
}
|
|
101
|
+
if (this.onTimeout) {
|
|
102
|
+
this.onTimeout(phaseName, elapsed);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
createTimeoutError(phaseName, elapsed) {
|
|
106
|
+
const phaseMessages = {
|
|
107
|
+
connect: `Connection timeout: Failed to establish TCP connection within ${elapsed}ms`,
|
|
108
|
+
headers: `Headers timeout: Server did not send response headers within ${elapsed}ms`,
|
|
109
|
+
body: `Body timeout: Response body transfer stalled for ${elapsed}ms`,
|
|
110
|
+
total: `Total timeout: Request exceeded maximum duration of ${elapsed}ms`
|
|
111
|
+
};
|
|
112
|
+
const message = phaseMessages[phaseName] || `Timeout in ${phaseName} phase after ${elapsed}ms`;
|
|
113
|
+
const error = new RezoError(message, this.config || {}, phaseName === "connect" ? "ETIMEDOUT" : phaseName === "headers" ? "ESOCKETTIMEDOUT" : "ECONNRESET", this.requestConfig || undefined);
|
|
114
|
+
error.phase = phaseName;
|
|
115
|
+
error.elapsed = elapsed;
|
|
116
|
+
error.isRetryable = phaseName === "connect" || phaseName === "headers";
|
|
117
|
+
return error;
|
|
118
|
+
}
|
|
119
|
+
getPhaseTimeout(phaseName) {
|
|
120
|
+
return this.phases.get(phaseName)?.timeout;
|
|
121
|
+
}
|
|
122
|
+
hasPhase(phaseName) {
|
|
123
|
+
return this.phases.has(phaseName);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function parseStagedTimeouts(timeout) {
|
|
127
|
+
if (!timeout) {
|
|
128
|
+
return {};
|
|
129
|
+
}
|
|
130
|
+
if (typeof timeout === "number") {
|
|
131
|
+
return {
|
|
132
|
+
total: timeout,
|
|
133
|
+
connect: Math.min(timeout, 1e4),
|
|
134
|
+
headers: Math.min(timeout, 30000)
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
return timeout;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
exports.StagedTimeoutManager = StagedTimeoutManager;
|
|
141
|
+
exports.parseStagedTimeouts = parseStagedTimeouts;
|
|
142
|
+
exports.default = StagedTimeoutManager;
|
|
143
|
+
module.exports = Object.assign(StagedTimeoutManager, exports);
|