rock-req 5.2.1 → 5.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +1 -1
  2. package/index.js +72 -17
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -32,7 +32,7 @@ In most existing libraries (2025):
32
32
  It also supports many features:
33
33
 
34
34
  - Follows redirects
35
- - Handles **gzip/deflate/brotli** responses
35
+ - Handles **gzip/deflate/brotli** responses (or returns the raw body with `decompress: false`)
36
36
  - Modify defaults
37
37
  - Extend and create new instances
38
38
  - Automatically destroy input/output **stream** on error and premature close event
package/index.js CHANGED
@@ -9,26 +9,76 @@ const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === '
9
9
  const isFnStream = o => o instanceof Function
10
10
  function applyDefault (t, d) { for (const a in d) { if (t[a] === undefined) t[a] = d[a] } return t }
11
11
  function cloneLowerCase (s) { const n = {}; for (const a in s) { n[a.toLowerCase()] = s[a] } return n }
12
+ const destroy = http.ClientRequest.prototype.destroy
13
+ function abort (err) { this.rockAborted = true; return destroy.call(this, err) } // req.destroy() or req.abort() by the user: never retried
14
+ function sweep (agent) { // destroy the free sockets idle for too long, then wake up for the next one. In-flight sockets are not in freeSockets
15
+ const now = Date.now()
16
+ let next = Infinity
17
+ for (const name in agent.freeSockets) {
18
+ for (const socket of agent.freeSockets[name]) {
19
+ if (socket.rockIdleEnd <= now) socket.destroy()
20
+ else if (socket.rockIdleEnd < next) next = socket.rockIdleEnd
21
+ }
22
+ }
23
+ agent.rockSweep = next === Infinity ? undefined : setTimeout(sweep, next - now, agent).unref()
24
+ agent.rockSweepAt = next
25
+ }
26
+ function retryAfter (res) { // RFC 9110 §10.2.3: delay-seconds or HTTP-date, in ms. Missing, invalid or past: 0
27
+ const h = res.statusCode === 429 || res.statusCode === 503 ? res.headers['retry-after'] : undefined
28
+ const ms = h === undefined ? 0 : /^\d+$/.test(h) ? h * 1000 : Date.parse(h) - Date.now()
29
+ return ms > 0 ? ms : 0
30
+ }
12
31
 
13
32
  function extend (defaultOptions = {}) {
14
33
  let _default = {
15
34
  headers: { 'accept-encoding': 'gzip, deflate, br' },
16
35
  maxRedirects: 10,
17
36
  maxRetry: 1,
18
- retryDelay: 10, // ms
37
+ retryDelay: 10, // ms, doubled on each retry, plus jitter
19
38
  keepAliveDuration: 3000, // ms
20
39
  retryOnCode: [408, 429, 502, 503, 504, 521, 522, 524],
21
- retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'],
40
+ retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EHOSTUNREACH', 'EAI_AGAIN'],
22
41
  beforeRequest: o => o
23
42
  }
24
43
  defaultOptions.headers = applyDefault(cloneLowerCase(defaultOptions.headers), _default.headers)
25
44
  _default = applyDefault(defaultOptions, _default) // inherits of parent options
26
- const agents = [http, https].map(h => (_default.keepAliveDuration > 0) ? new h.Agent({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration }) : undefined)
45
+ const idleTimeout = Number(_default.keepAliveTimeout ?? 5000) // ms, like Node's globalAgent. Like decompress (true), not in defaults: not copied in each request (faster)
46
+ const sensitiveHeaders = _default.sensitiveHeaders ?? ['authorization', 'cookie', 'proxy-authorization', 'x-auth-token'] // dropped on redirect to another host or from https to http
47
+ const agents = [http, https].map(h => {
48
+ if (!(_default.keepAliveDuration > 0)) return undefined
49
+ class RockAgent extends h.Agent {
50
+ keepSocketAlive (socket) { // the socket goes to the free pool. No socket.setTimeout per request (slow): one sweep timer per agent
51
+ if (super.keepSocketAlive(socket) === false) return false
52
+ const hint = socket._httpMessage?.res?.headers['keep-alive'] // server hint minus 1s, like Node
53
+ const timeout = hint === undefined ? idleTimeout : Math.min(idleTimeout, (/^timeout=(\d+)/.exec(hint)?.[1] ?? Infinity) * 1000 - 1000)
54
+ if (timeout <= 0) return false
55
+ socket.rockIdleEnd = Date.now() + timeout
56
+ if (this.rockSweep === undefined || socket.rockIdleEnd < this.rockSweepAt) {
57
+ clearTimeout(this.rockSweep)
58
+ this.rockSweep = setTimeout(sweep, timeout, this).unref()
59
+ this.rockSweepAt = socket.rockIdleEnd
60
+ }
61
+ return true
62
+ }
63
+ }
64
+ return new (idleTimeout > 0 ? RockAgent : h.Agent)({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration })
65
+ })
66
+
67
+ function retry (opts, cb, wait = 0) { // failure only: retryDelay * 2^n + jitter, capped by retryMaxDelay, at least Retry-After
68
+ const d = opts.retryDelay * 2 ** (opts.maxRetry - opts.remainingRetry - 1)
69
+ setTimeout(next, Math.max(wait, Math.min(opts.retryMaxDelay ?? 2000, d + Math.random() * d)), opts, cb)
70
+ }
71
+ function next (opts, cb) { // retry or redirect: called from an event handler, so an invalid request (bad location, port...) must not throw
72
+ let called = false
73
+ try { rock(opts, (err, res, data) => { called = true; return cb(err, res, data) }) } catch (e) { if (called) throw e; cb(e) } // an exception of the caller's callback is not swallowed, never a second call
74
+ }
27
75
 
28
76
  function rock (opts, directBody, cb) {
29
77
  if (typeof opts === 'string') opts = { url: opts }
30
78
  if (!cb) { cb = directBody } else { opts.body = directBody }
31
- opts.headers = applyDefault(cloneLowerCase(opts.headers), _default.headers)
79
+ const headers = cloneLowerCase(opts.headers)
80
+ if (headers.range !== undefined && headers['accept-encoding'] === undefined) headers['accept-encoding'] = 'identity' // a range of the compressed body cannot be decompressed
81
+ opts.headers = applyDefault(headers, _default.headers)
32
82
  opts = applyDefault(opts, _default)
33
83
  opts.remainingRetry = opts.remainingRetry ?? opts.maxRetry
34
84
  opts.remainingRedirects = opts.remainingRedirects ?? opts.maxRedirects
@@ -68,10 +118,11 @@ function extend (defaultOptions = {}) {
68
118
  if (requestAbortedOrEnded === true) return
69
119
  requestAbortedOrEnded = true
70
120
  if (err) {
121
+ const aborted = req.rockAborted === true // read before destroying the streams below (req included)
71
122
  streamToCleanOnError.forEach(s => s.destroy(err))
72
- if (opts.retryOnError.indexOf(err?.code) !== -1 && --opts.remainingRetry > 0) {
123
+ if (aborted === false && opts.retryOnError.indexOf(err?.code) !== -1 && opts.remainingRetry-- > 0) {
73
124
  opts.prevError = err
74
- return setTimeout(rock, opts.retryDelay, opts, cb) // retry
125
+ return retry(opts, cb)
75
126
  }
76
127
  return cb(err)
77
128
  }
@@ -92,11 +143,12 @@ function extend (defaultOptions = {}) {
92
143
  }
93
144
  const req = protocol.request(opts, res => {
94
145
  opts.prevStatusCode = res.statusCode
146
+ let retryWait
95
147
  // retry and leave
96
- if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && opts.remainingRetry-- > 0) {
148
+ if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && (retryWait = retryAfter(res)) <= (opts.retryMaxDelay ?? 2000) && opts.remainingRetry-- > 0) {
97
149
  requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
98
150
  res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
99
- return setTimeout(rock, opts.retryDelay, opts, cb) // retry later
151
+ return retry(opts, cb, retryWait) // retry later
100
152
  }
101
153
  // or redirect and leave
102
154
  if (opts.followRedirects !== false && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
@@ -104,42 +156,45 @@ function extend (defaultOptions = {}) {
104
156
  res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
105
157
  delete opts.headers.host // Discard `host` header on redirect (see #32)
106
158
  opts.url = res.headers.location
107
- const redirectTo = url.parse(opts.url) // eslint-disable-line node/no-deprecated-api
159
+ let redirectTo
160
+ try { redirectTo = url.parse(opts.url) } catch (e) { return cb(e) } // eslint-disable-line node/no-deprecated-api
108
161
  if (redirectTo.hostname === null) { // relative redirect
109
162
  opts.url = null
110
163
  Object.assign(opts, originalRequest)
111
164
  opts.path = redirectTo.path
112
- } else if (redirectTo.hostname !== originalRequest.hostname) { // If redirected host is different than original host, drop headers to prevent cookie leak (#73)
113
- delete opts.headers.cookie
114
- delete opts.headers.authorization
165
+ } else if (redirectTo.hostname !== originalRequest.hostname || (redirectTo.protocol === 'http:' && originalRequest.protocol === 'https:')) {
166
+ for (const h of sensitiveHeaders) delete opts.headers[h] // other host or https to http: drop credentials (#73). The body of a 307/308 is still sent
115
167
  }
116
- if (opts.method === 'POST' && [301, 302].includes(res.statusCode)) {
117
- opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35)
168
+ if ((opts.method === 'POST' && [301, 302].includes(res.statusCode)) || (res.statusCode === 303 && opts.method && opts.method !== 'GET' && opts.method !== 'HEAD')) {
169
+ opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35). On 303, change all methods except HEAD to GET (RFC 9110)
118
170
  delete opts.headers['content-length']; delete opts.headers['content-type']; delete opts.body; delete opts.form
119
171
  }
120
172
  if (opts.remainingRedirects-- === 0) {
121
173
  requestAbortedOrEnded = false
122
174
  return onRequestEnd(new Error('too many redirects'))
123
175
  }
124
- return rock(opts, cb)
176
+ return next(opts, cb)
125
177
  }
126
178
  // or read response and leave at the end
127
179
  response = res
128
- const contentEncoding = opts.method !== 'HEAD' ? (res.headers['content-encoding'] || '').toLowerCase() : ''
180
+ const status = res.statusCode // no body (HEAD, 204, 205, 304. Node never gives 1xx here) or a range of the compressed body (206): never decompress
181
+ const contentEncoding = opts.decompress !== false && opts.method !== 'HEAD' && status !== 204 && status !== 205 && status !== 206 && status !== 304 ? (res.headers['content-encoding'] || '').toLowerCase() : ''
129
182
  const output = opts.output ? opts.output(opts, res) : new Writable({ write (chunk, enc, wcb) { chunks.push(chunk); wcb() } })
130
183
  switch (contentEncoding) {
131
184
  case 'br':
132
185
  listen(res).pipe(listen(zlib.createBrotliDecompress())).pipe(listen(output, true)); break
133
186
  case 'gzip':
187
+ case 'x-gzip':
134
188
  case 'deflate':
135
189
  listen(res).pipe(listen(zlib.createUnzip())).pipe(listen(output, true)); break
136
190
  default:
137
191
  listen(res).pipe(listen(output, true)); break
138
192
  }
139
193
  })
194
+ req.destroy = abort
140
195
  req.once('timeout', () => {
141
196
  const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT'
142
- req.destroy(_error) // we must destroy manually and send the error to the error listener to call onRequestEnd
197
+ destroy.call(req, _error) // we must destroy manually and send the error to the error listener to call onRequestEnd. Not a user abort: retried
143
198
  })
144
199
 
145
200
  if (isFnStream(body) === true) listen(body(opts)).pipe(listen(req))
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "rock-req",
3
3
  "description": "Ultra-light (150 LOC, No dependencies) & Ultra-fast request library with reliable retry on failure, http/https, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout",
4
- "version": "5.2.1",
4
+ "version": "5.3.1",
5
5
  "author": {
6
6
  "name": "David Grelaud & Feross Aboukhadijeh"
7
7
  },