rock-req 5.2.1 → 5.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/index.js +72 -17
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ In most existing libraries (2025):
|
|
|
32
32
|
It also supports many features:
|
|
33
33
|
|
|
34
34
|
- Follows redirects
|
|
35
|
-
- Handles **gzip/deflate/brotli** responses
|
|
35
|
+
- Handles **gzip/deflate/brotli** responses (or returns the raw body with `decompress: false`)
|
|
36
36
|
- Modify defaults
|
|
37
37
|
- Extend and create new instances
|
|
38
38
|
- Automatically destroy input/output **stream** on error and premature close event
|
package/index.js
CHANGED
|
@@ -9,26 +9,76 @@ const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === '
|
|
|
9
9
|
const isFnStream = o => o instanceof Function
|
|
10
10
|
function applyDefault (t, d) { for (const a in d) { if (t[a] === undefined) t[a] = d[a] } return t }
|
|
11
11
|
function cloneLowerCase (s) { const n = {}; for (const a in s) { n[a.toLowerCase()] = s[a] } return n }
|
|
12
|
+
const destroy = http.ClientRequest.prototype.destroy
|
|
13
|
+
function abort (err) { this.rockAborted = true; return destroy.call(this, err) } // req.destroy() or req.abort() by the user: never retried
|
|
14
|
+
function sweep (agent) { // destroy the free sockets idle for too long, then wake up for the next one. In-flight sockets are not in freeSockets
|
|
15
|
+
const now = Date.now()
|
|
16
|
+
let next = Infinity
|
|
17
|
+
for (const name in agent.freeSockets) {
|
|
18
|
+
for (const socket of agent.freeSockets[name]) {
|
|
19
|
+
if (socket.rockIdleEnd <= now) socket.destroy()
|
|
20
|
+
else if (socket.rockIdleEnd < next) next = socket.rockIdleEnd
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
agent.rockSweep = next === Infinity ? undefined : setTimeout(sweep, next - now, agent).unref()
|
|
24
|
+
agent.rockSweepAt = next
|
|
25
|
+
}
|
|
26
|
+
function retryAfter (res) { // RFC 9110 §10.2.3: delay-seconds or HTTP-date, in ms. Missing, invalid or past: 0
|
|
27
|
+
const h = res.statusCode === 429 || res.statusCode === 503 ? res.headers['retry-after'] : undefined
|
|
28
|
+
const ms = h === undefined ? 0 : /^\d+$/.test(h) ? h * 1000 : Date.parse(h) - Date.now()
|
|
29
|
+
return ms > 0 ? ms : 0
|
|
30
|
+
}
|
|
12
31
|
|
|
13
32
|
function extend (defaultOptions = {}) {
|
|
14
33
|
let _default = {
|
|
15
34
|
headers: { 'accept-encoding': 'gzip, deflate, br' },
|
|
16
35
|
maxRedirects: 10,
|
|
17
36
|
maxRetry: 1,
|
|
18
|
-
retryDelay: 10, // ms
|
|
37
|
+
retryDelay: 10, // ms, doubled on each retry, plus jitter
|
|
19
38
|
keepAliveDuration: 3000, // ms
|
|
20
39
|
retryOnCode: [408, 429, 502, 503, 504, 521, 522, 524],
|
|
21
|
-
retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'],
|
|
40
|
+
retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EHOSTUNREACH', 'EAI_AGAIN'],
|
|
22
41
|
beforeRequest: o => o
|
|
23
42
|
}
|
|
24
43
|
defaultOptions.headers = applyDefault(cloneLowerCase(defaultOptions.headers), _default.headers)
|
|
25
44
|
_default = applyDefault(defaultOptions, _default) // inherits of parent options
|
|
26
|
-
const
|
|
45
|
+
const idleTimeout = Number(_default.keepAliveTimeout ?? 5000) // ms, like Node's globalAgent. Like decompress (true), not in defaults: not copied in each request (faster)
|
|
46
|
+
const sensitiveHeaders = _default.sensitiveHeaders ?? ['authorization', 'cookie', 'proxy-authorization', 'x-auth-token'] // dropped on redirect to another host or from https to http
|
|
47
|
+
const agents = [http, https].map(h => {
|
|
48
|
+
if (!(_default.keepAliveDuration > 0)) return undefined
|
|
49
|
+
class RockAgent extends h.Agent {
|
|
50
|
+
keepSocketAlive (socket) { // the socket goes to the free pool. No socket.setTimeout per request (slow): one sweep timer per agent
|
|
51
|
+
if (super.keepSocketAlive(socket) === false) return false
|
|
52
|
+
const hint = socket._httpMessage?.res?.headers['keep-alive'] // server hint minus 1s, like Node
|
|
53
|
+
const timeout = hint === undefined ? idleTimeout : Math.min(idleTimeout, (/^timeout=(\d+)/.exec(hint)?.[1] ?? Infinity) * 1000 - 1000)
|
|
54
|
+
if (timeout <= 0) return false
|
|
55
|
+
socket.rockIdleEnd = Date.now() + timeout
|
|
56
|
+
if (this.rockSweep === undefined || socket.rockIdleEnd < this.rockSweepAt) {
|
|
57
|
+
clearTimeout(this.rockSweep)
|
|
58
|
+
this.rockSweep = setTimeout(sweep, timeout, this).unref()
|
|
59
|
+
this.rockSweepAt = socket.rockIdleEnd
|
|
60
|
+
}
|
|
61
|
+
return true
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return new (idleTimeout > 0 ? RockAgent : h.Agent)({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration })
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
function retry (opts, cb, wait = 0) { // failure only: retryDelay * 2^n + jitter, capped by retryMaxDelay, at least Retry-After
|
|
68
|
+
const d = opts.retryDelay * 2 ** (opts.maxRetry - opts.remainingRetry - 1)
|
|
69
|
+
setTimeout(next, Math.max(wait, Math.min(opts.retryMaxDelay ?? 2000, d + Math.random() * d)), opts, cb)
|
|
70
|
+
}
|
|
71
|
+
function next (opts, cb) { // retry or redirect: called from an event handler, so an invalid request (bad location, port...) must not throw
|
|
72
|
+
let called = false
|
|
73
|
+
try { rock(opts, (err, res, data) => { called = true; return cb(err, res, data) }) } catch (e) { if (called) throw e; cb(e) } // an exception of the caller's callback is not swallowed, never a second call
|
|
74
|
+
}
|
|
27
75
|
|
|
28
76
|
function rock (opts, directBody, cb) {
|
|
29
77
|
if (typeof opts === 'string') opts = { url: opts }
|
|
30
78
|
if (!cb) { cb = directBody } else { opts.body = directBody }
|
|
31
|
-
|
|
79
|
+
const headers = cloneLowerCase(opts.headers)
|
|
80
|
+
if (headers.range !== undefined && headers['accept-encoding'] === undefined) headers['accept-encoding'] = 'identity' // a range of the compressed body cannot be decompressed
|
|
81
|
+
opts.headers = applyDefault(headers, _default.headers)
|
|
32
82
|
opts = applyDefault(opts, _default)
|
|
33
83
|
opts.remainingRetry = opts.remainingRetry ?? opts.maxRetry
|
|
34
84
|
opts.remainingRedirects = opts.remainingRedirects ?? opts.maxRedirects
|
|
@@ -68,10 +118,11 @@ function extend (defaultOptions = {}) {
|
|
|
68
118
|
if (requestAbortedOrEnded === true) return
|
|
69
119
|
requestAbortedOrEnded = true
|
|
70
120
|
if (err) {
|
|
121
|
+
const aborted = req.rockAborted === true // read before destroying the streams below (req included)
|
|
71
122
|
streamToCleanOnError.forEach(s => s.destroy(err))
|
|
72
|
-
if (opts.retryOnError.indexOf(err?.code) !== -1 &&
|
|
123
|
+
if (aborted === false && opts.retryOnError.indexOf(err?.code) !== -1 && opts.remainingRetry-- > 0) {
|
|
73
124
|
opts.prevError = err
|
|
74
|
-
return
|
|
125
|
+
return retry(opts, cb)
|
|
75
126
|
}
|
|
76
127
|
return cb(err)
|
|
77
128
|
}
|
|
@@ -92,11 +143,12 @@ function extend (defaultOptions = {}) {
|
|
|
92
143
|
}
|
|
93
144
|
const req = protocol.request(opts, res => {
|
|
94
145
|
opts.prevStatusCode = res.statusCode
|
|
146
|
+
let retryWait
|
|
95
147
|
// retry and leave
|
|
96
|
-
if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && opts.remainingRetry-- > 0) {
|
|
148
|
+
if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && (retryWait = retryAfter(res)) <= (opts.retryMaxDelay ?? 2000) && opts.remainingRetry-- > 0) {
|
|
97
149
|
requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
|
|
98
150
|
res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
|
|
99
|
-
return
|
|
151
|
+
return retry(opts, cb, retryWait) // retry later
|
|
100
152
|
}
|
|
101
153
|
// or redirect and leave
|
|
102
154
|
if (opts.followRedirects !== false && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
@@ -104,42 +156,45 @@ function extend (defaultOptions = {}) {
|
|
|
104
156
|
res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
|
|
105
157
|
delete opts.headers.host // Discard `host` header on redirect (see #32)
|
|
106
158
|
opts.url = res.headers.location
|
|
107
|
-
|
|
159
|
+
let redirectTo
|
|
160
|
+
try { redirectTo = url.parse(opts.url) } catch (e) { return cb(e) } // eslint-disable-line node/no-deprecated-api
|
|
108
161
|
if (redirectTo.hostname === null) { // relative redirect
|
|
109
162
|
opts.url = null
|
|
110
163
|
Object.assign(opts, originalRequest)
|
|
111
164
|
opts.path = redirectTo.path
|
|
112
|
-
} else if (redirectTo.hostname !== originalRequest.hostname
|
|
113
|
-
delete opts.headers.
|
|
114
|
-
delete opts.headers.authorization
|
|
165
|
+
} else if (redirectTo.hostname !== originalRequest.hostname || (redirectTo.protocol === 'http:' && originalRequest.protocol === 'https:')) {
|
|
166
|
+
for (const h of sensitiveHeaders) delete opts.headers[h] // other host or https to http: drop credentials (#73). The body of a 307/308 is still sent
|
|
115
167
|
}
|
|
116
|
-
if (opts.method === 'POST' && [301, 302].includes(res.statusCode)) {
|
|
117
|
-
opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35)
|
|
168
|
+
if ((opts.method === 'POST' && [301, 302].includes(res.statusCode)) || (res.statusCode === 303 && opts.method && opts.method !== 'GET' && opts.method !== 'HEAD')) {
|
|
169
|
+
opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35). On 303, change all methods except HEAD to GET (RFC 9110)
|
|
118
170
|
delete opts.headers['content-length']; delete opts.headers['content-type']; delete opts.body; delete opts.form
|
|
119
171
|
}
|
|
120
172
|
if (opts.remainingRedirects-- === 0) {
|
|
121
173
|
requestAbortedOrEnded = false
|
|
122
174
|
return onRequestEnd(new Error('too many redirects'))
|
|
123
175
|
}
|
|
124
|
-
return
|
|
176
|
+
return next(opts, cb)
|
|
125
177
|
}
|
|
126
178
|
// or read response and leave at the end
|
|
127
179
|
response = res
|
|
128
|
-
const
|
|
180
|
+
const status = res.statusCode // no body (HEAD, 204, 205, 304. Node never gives 1xx here) or a range of the compressed body (206): never decompress
|
|
181
|
+
const contentEncoding = opts.decompress !== false && opts.method !== 'HEAD' && status !== 204 && status !== 205 && status !== 206 && status !== 304 ? (res.headers['content-encoding'] || '').toLowerCase() : ''
|
|
129
182
|
const output = opts.output ? opts.output(opts, res) : new Writable({ write (chunk, enc, wcb) { chunks.push(chunk); wcb() } })
|
|
130
183
|
switch (contentEncoding) {
|
|
131
184
|
case 'br':
|
|
132
185
|
listen(res).pipe(listen(zlib.createBrotliDecompress())).pipe(listen(output, true)); break
|
|
133
186
|
case 'gzip':
|
|
187
|
+
case 'x-gzip':
|
|
134
188
|
case 'deflate':
|
|
135
189
|
listen(res).pipe(listen(zlib.createUnzip())).pipe(listen(output, true)); break
|
|
136
190
|
default:
|
|
137
191
|
listen(res).pipe(listen(output, true)); break
|
|
138
192
|
}
|
|
139
193
|
})
|
|
194
|
+
req.destroy = abort
|
|
140
195
|
req.once('timeout', () => {
|
|
141
196
|
const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT'
|
|
142
|
-
|
|
197
|
+
destroy.call(req, _error) // we must destroy manually and send the error to the error listener to call onRequestEnd. Not a user abort: retried
|
|
143
198
|
})
|
|
144
199
|
|
|
145
200
|
if (isFnStream(body) === true) listen(body(opts)).pipe(listen(req))
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rock-req",
|
|
3
3
|
"description": "Ultra-light (150 LOC, No dependencies) & Ultra-fast request library with reliable retry on failure, http/https, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.3.1",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "David Grelaud & Feross Aboukhadijeh"
|
|
7
7
|
},
|