rock-req 5.2.1 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.js +62 -12
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ In most existing libraries (2025):
|
|
|
32
32
|
It also supports many features:
|
|
33
33
|
|
|
34
34
|
- Follows redirects
|
|
35
|
-
- Handles **gzip/deflate/brotli** responses
|
|
35
|
+
- Handles **gzip/deflate/brotli** responses (or returns the raw body with `decompress: false`)
|
|
36
36
|
- Modify defaults
|
|
37
37
|
- Extend and create new instances
|
|
38
38
|
- Automatically destroy input/output **stream** on error and premature close event
|
package/index.js
CHANGED
|
@@ -9,26 +9,71 @@ const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === '
|
|
|
9
9
|
const isFnStream = o => o instanceof Function
|
|
10
10
|
function applyDefault (t, d) { for (const a in d) { if (t[a] === undefined) t[a] = d[a] } return t }
|
|
11
11
|
function cloneLowerCase (s) { const n = {}; for (const a in s) { n[a.toLowerCase()] = s[a] } return n }
|
|
12
|
+
const destroy = http.ClientRequest.prototype.destroy
|
|
13
|
+
function abort (err) { this.rockAborted = true; return destroy.call(this, err) } // req.destroy() or req.abort() by the user: never retried
|
|
14
|
+
function sweep (agent) { // destroy the free sockets idle for too long, then wake up for the next one. In-flight sockets are not in freeSockets
|
|
15
|
+
const now = Date.now()
|
|
16
|
+
let next = Infinity
|
|
17
|
+
for (const name in agent.freeSockets) {
|
|
18
|
+
for (const socket of agent.freeSockets[name]) {
|
|
19
|
+
if (socket.rockIdleEnd <= now) socket.destroy()
|
|
20
|
+
else if (socket.rockIdleEnd < next) next = socket.rockIdleEnd
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
agent.rockSweep = next === Infinity ? undefined : setTimeout(sweep, next - now, agent).unref()
|
|
24
|
+
agent.rockSweepAt = next
|
|
25
|
+
}
|
|
26
|
+
function retryAfter (res) { // RFC 9110 §10.2.3: delay-seconds or HTTP-date, in ms. Missing, invalid or past: 0
|
|
27
|
+
const h = res.statusCode === 429 || res.statusCode === 503 ? res.headers['retry-after'] : undefined
|
|
28
|
+
const ms = h === undefined ? 0 : /^\d+$/.test(h) ? h * 1000 : Date.parse(h) - Date.now()
|
|
29
|
+
return ms > 0 ? ms : 0
|
|
30
|
+
}
|
|
12
31
|
|
|
13
32
|
function extend (defaultOptions = {}) {
|
|
14
33
|
let _default = {
|
|
15
34
|
headers: { 'accept-encoding': 'gzip, deflate, br' },
|
|
16
35
|
maxRedirects: 10,
|
|
17
36
|
maxRetry: 1,
|
|
18
|
-
retryDelay: 10, // ms
|
|
37
|
+
retryDelay: 10, // ms, doubled on each retry, plus jitter
|
|
19
38
|
keepAliveDuration: 3000, // ms
|
|
20
39
|
retryOnCode: [408, 429, 502, 503, 504, 521, 522, 524],
|
|
21
|
-
retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'],
|
|
40
|
+
retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EHOSTUNREACH', 'EAI_AGAIN'],
|
|
22
41
|
beforeRequest: o => o
|
|
23
42
|
}
|
|
24
43
|
defaultOptions.headers = applyDefault(cloneLowerCase(defaultOptions.headers), _default.headers)
|
|
25
44
|
_default = applyDefault(defaultOptions, _default) // inherits of parent options
|
|
26
|
-
const
|
|
45
|
+
const idleTimeout = Number(_default.keepAliveTimeout ?? 5000) // ms, like Node's globalAgent. Like decompress (true), not in defaults: not copied in each request (faster)
|
|
46
|
+
const agents = [http, https].map(h => {
|
|
47
|
+
if (!(_default.keepAliveDuration > 0)) return undefined
|
|
48
|
+
class RockAgent extends h.Agent {
|
|
49
|
+
keepSocketAlive (socket) { // the socket goes to the free pool. No socket.setTimeout per request (slow): one sweep timer per agent
|
|
50
|
+
if (super.keepSocketAlive(socket) === false) return false
|
|
51
|
+
const hint = socket._httpMessage?.res?.headers['keep-alive'] // server hint minus 1s, like Node
|
|
52
|
+
const timeout = hint === undefined ? idleTimeout : Math.min(idleTimeout, (/^timeout=(\d+)/.exec(hint)?.[1] ?? Infinity) * 1000 - 1000)
|
|
53
|
+
if (timeout <= 0) return false
|
|
54
|
+
socket.rockIdleEnd = Date.now() + timeout
|
|
55
|
+
if (this.rockSweep === undefined || socket.rockIdleEnd < this.rockSweepAt) {
|
|
56
|
+
clearTimeout(this.rockSweep)
|
|
57
|
+
this.rockSweep = setTimeout(sweep, timeout, this).unref()
|
|
58
|
+
this.rockSweepAt = socket.rockIdleEnd
|
|
59
|
+
}
|
|
60
|
+
return true
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return new (idleTimeout > 0 ? RockAgent : h.Agent)({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration })
|
|
64
|
+
})
|
|
65
|
+
|
|
66
|
+
function retry (opts, cb, wait = 0) { // failure only: retryDelay * 2^n + jitter, capped by retryMaxDelay, at least Retry-After
|
|
67
|
+
const d = opts.retryDelay * 2 ** (opts.maxRetry - opts.remainingRetry - 1)
|
|
68
|
+
setTimeout(rock, Math.max(wait, Math.min(opts.retryMaxDelay ?? 2000, d + Math.random() * d)), opts, cb)
|
|
69
|
+
}
|
|
27
70
|
|
|
28
71
|
function rock (opts, directBody, cb) {
|
|
29
72
|
if (typeof opts === 'string') opts = { url: opts }
|
|
30
73
|
if (!cb) { cb = directBody } else { opts.body = directBody }
|
|
31
|
-
|
|
74
|
+
const headers = cloneLowerCase(opts.headers)
|
|
75
|
+
if (headers.range !== undefined && headers['accept-encoding'] === undefined) headers['accept-encoding'] = 'identity' // a range of the compressed body cannot be decompressed
|
|
76
|
+
opts.headers = applyDefault(headers, _default.headers)
|
|
32
77
|
opts = applyDefault(opts, _default)
|
|
33
78
|
opts.remainingRetry = opts.remainingRetry ?? opts.maxRetry
|
|
34
79
|
opts.remainingRedirects = opts.remainingRedirects ?? opts.maxRedirects
|
|
@@ -68,10 +113,11 @@ function extend (defaultOptions = {}) {
|
|
|
68
113
|
if (requestAbortedOrEnded === true) return
|
|
69
114
|
requestAbortedOrEnded = true
|
|
70
115
|
if (err) {
|
|
116
|
+
const aborted = req.rockAborted === true // read before destroying the streams below (req included)
|
|
71
117
|
streamToCleanOnError.forEach(s => s.destroy(err))
|
|
72
|
-
if (opts.retryOnError.indexOf(err?.code) !== -1 &&
|
|
118
|
+
if (aborted === false && opts.retryOnError.indexOf(err?.code) !== -1 && opts.remainingRetry-- > 0) {
|
|
73
119
|
opts.prevError = err
|
|
74
|
-
return
|
|
120
|
+
return retry(opts, cb)
|
|
75
121
|
}
|
|
76
122
|
return cb(err)
|
|
77
123
|
}
|
|
@@ -92,11 +138,12 @@ function extend (defaultOptions = {}) {
|
|
|
92
138
|
}
|
|
93
139
|
const req = protocol.request(opts, res => {
|
|
94
140
|
opts.prevStatusCode = res.statusCode
|
|
141
|
+
let retryWait
|
|
95
142
|
// retry and leave
|
|
96
|
-
if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && opts.remainingRetry-- > 0) {
|
|
143
|
+
if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && (retryWait = retryAfter(res)) <= (opts.retryMaxDelay ?? 2000) && opts.remainingRetry-- > 0) {
|
|
97
144
|
requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
|
|
98
145
|
res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
|
|
99
|
-
return
|
|
146
|
+
return retry(opts, cb, retryWait) // retry later
|
|
100
147
|
}
|
|
101
148
|
// or redirect and leave
|
|
102
149
|
if (opts.followRedirects !== false && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
@@ -113,8 +160,8 @@ function extend (defaultOptions = {}) {
|
|
|
113
160
|
delete opts.headers.cookie
|
|
114
161
|
delete opts.headers.authorization
|
|
115
162
|
}
|
|
116
|
-
if (opts.method === 'POST' && [301, 302].includes(res.statusCode)) {
|
|
117
|
-
opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35)
|
|
163
|
+
if ((opts.method === 'POST' && [301, 302].includes(res.statusCode)) || (res.statusCode === 303 && opts.method && opts.method !== 'GET' && opts.method !== 'HEAD')) {
|
|
164
|
+
opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35). On 303, change all methods except HEAD to GET (RFC 9110)
|
|
118
165
|
delete opts.headers['content-length']; delete opts.headers['content-type']; delete opts.body; delete opts.form
|
|
119
166
|
}
|
|
120
167
|
if (opts.remainingRedirects-- === 0) {
|
|
@@ -125,21 +172,24 @@ function extend (defaultOptions = {}) {
|
|
|
125
172
|
}
|
|
126
173
|
// or read response and leave at the end
|
|
127
174
|
response = res
|
|
128
|
-
const
|
|
175
|
+
const status = res.statusCode // no body (HEAD, 204, 205, 304. Node never gives 1xx here) or a range of the compressed body (206): never decompress
|
|
176
|
+
const contentEncoding = opts.decompress !== false && opts.method !== 'HEAD' && status !== 204 && status !== 205 && status !== 206 && status !== 304 ? (res.headers['content-encoding'] || '').toLowerCase() : ''
|
|
129
177
|
const output = opts.output ? opts.output(opts, res) : new Writable({ write (chunk, enc, wcb) { chunks.push(chunk); wcb() } })
|
|
130
178
|
switch (contentEncoding) {
|
|
131
179
|
case 'br':
|
|
132
180
|
listen(res).pipe(listen(zlib.createBrotliDecompress())).pipe(listen(output, true)); break
|
|
133
181
|
case 'gzip':
|
|
182
|
+
case 'x-gzip':
|
|
134
183
|
case 'deflate':
|
|
135
184
|
listen(res).pipe(listen(zlib.createUnzip())).pipe(listen(output, true)); break
|
|
136
185
|
default:
|
|
137
186
|
listen(res).pipe(listen(output, true)); break
|
|
138
187
|
}
|
|
139
188
|
})
|
|
189
|
+
req.destroy = abort
|
|
140
190
|
req.once('timeout', () => {
|
|
141
191
|
const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT'
|
|
142
|
-
|
|
192
|
+
destroy.call(req, _error) // we must destroy manually and send the error to the error listener to call onRequestEnd. Not a user abort: retried
|
|
143
193
|
})
|
|
144
194
|
|
|
145
195
|
if (isFnStream(body) === true) listen(body(opts)).pipe(listen(req))
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rock-req",
|
|
3
3
|
"description": "Ultra-light (150 LOC, No dependencies) & Ultra-fast request library with reliable retry on failure, http/https, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.3.0",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "David Grelaud & Feross Aboukhadijeh"
|
|
7
7
|
},
|