rock-req 5.2.1 → 5.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +1 -1
  2. package/index.js +62 -12
  3. package/package.json +1 -1
package/README.md CHANGED
@@ -32,7 +32,7 @@ In most existing libraries (2025):
32
32
  It also supports many features:
33
33
 
34
34
  - Follows redirects
35
- - Handles **gzip/deflate/brotli** responses
35
+ - Handles **gzip/deflate/brotli** responses (or returns the raw body with `decompress: false`)
36
36
  - Modify defaults
37
37
  - Extend and create new instances
38
38
  - Automatically destroy input/output **stream** on error and premature close event
package/index.js CHANGED
@@ -9,26 +9,71 @@ const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === '
9
9
  const isFnStream = o => o instanceof Function
10
10
  function applyDefault (t, d) { for (const a in d) { if (t[a] === undefined) t[a] = d[a] } return t }
11
11
  function cloneLowerCase (s) { const n = {}; for (const a in s) { n[a.toLowerCase()] = s[a] } return n }
12
+ const destroy = http.ClientRequest.prototype.destroy
13
+ function abort (err) { this.rockAborted = true; return destroy.call(this, err) } // req.destroy() or req.abort() by the user: never retried
14
+ function sweep (agent) { // destroy the free sockets idle for too long, then wake up for the next one. In-flight sockets are not in freeSockets
15
+ const now = Date.now()
16
+ let next = Infinity
17
+ for (const name in agent.freeSockets) {
18
+ for (const socket of agent.freeSockets[name]) {
19
+ if (socket.rockIdleEnd <= now) socket.destroy()
20
+ else if (socket.rockIdleEnd < next) next = socket.rockIdleEnd
21
+ }
22
+ }
23
+ agent.rockSweep = next === Infinity ? undefined : setTimeout(sweep, next - now, agent).unref()
24
+ agent.rockSweepAt = next
25
+ }
26
+ function retryAfter (res) { // RFC 9110 §10.2.3: delay-seconds or HTTP-date, in ms. Missing, invalid or past: 0
27
+ const h = res.statusCode === 429 || res.statusCode === 503 ? res.headers['retry-after'] : undefined
28
+ const ms = h === undefined ? 0 : /^\d+$/.test(h) ? h * 1000 : Date.parse(h) - Date.now()
29
+ return ms > 0 ? ms : 0
30
+ }
12
31
 
13
32
  function extend (defaultOptions = {}) {
14
33
  let _default = {
15
34
  headers: { 'accept-encoding': 'gzip, deflate, br' },
16
35
  maxRedirects: 10,
17
36
  maxRetry: 1,
18
- retryDelay: 10, // ms
37
+ retryDelay: 10, // ms, doubled on each retry, plus jitter
19
38
  keepAliveDuration: 3000, // ms
20
39
  retryOnCode: [408, 429, 502, 503, 504, 521, 522, 524],
21
- retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'],
40
+ retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EHOSTUNREACH', 'EAI_AGAIN'],
22
41
  beforeRequest: o => o
23
42
  }
24
43
  defaultOptions.headers = applyDefault(cloneLowerCase(defaultOptions.headers), _default.headers)
25
44
  _default = applyDefault(defaultOptions, _default) // inherits of parent options
26
- const agents = [http, https].map(h => (_default.keepAliveDuration > 0) ? new h.Agent({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration }) : undefined)
45
+ const idleTimeout = Number(_default.keepAliveTimeout ?? 5000) // ms, like Node's globalAgent. Like decompress (true), not in defaults: not copied in each request (faster)
46
+ const agents = [http, https].map(h => {
47
+ if (!(_default.keepAliveDuration > 0)) return undefined
48
+ class RockAgent extends h.Agent {
49
+ keepSocketAlive (socket) { // the socket goes to the free pool. No socket.setTimeout per request (slow): one sweep timer per agent
50
+ if (super.keepSocketAlive(socket) === false) return false
51
+ const hint = socket._httpMessage?.res?.headers['keep-alive'] // server hint minus 1s, like Node
52
+ const timeout = hint === undefined ? idleTimeout : Math.min(idleTimeout, (/^timeout=(\d+)/.exec(hint)?.[1] ?? Infinity) * 1000 - 1000)
53
+ if (timeout <= 0) return false
54
+ socket.rockIdleEnd = Date.now() + timeout
55
+ if (this.rockSweep === undefined || socket.rockIdleEnd < this.rockSweepAt) {
56
+ clearTimeout(this.rockSweep)
57
+ this.rockSweep = setTimeout(sweep, timeout, this).unref()
58
+ this.rockSweepAt = socket.rockIdleEnd
59
+ }
60
+ return true
61
+ }
62
+ }
63
+ return new (idleTimeout > 0 ? RockAgent : h.Agent)({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration })
64
+ })
65
+
66
+ function retry (opts, cb, wait = 0) { // failure only: retryDelay * 2^n + jitter, capped by retryMaxDelay, at least Retry-After
67
+ const d = opts.retryDelay * 2 ** (opts.maxRetry - opts.remainingRetry - 1)
68
+ setTimeout(rock, Math.max(wait, Math.min(opts.retryMaxDelay ?? 2000, d + Math.random() * d)), opts, cb)
69
+ }
27
70
 
28
71
  function rock (opts, directBody, cb) {
29
72
  if (typeof opts === 'string') opts = { url: opts }
30
73
  if (!cb) { cb = directBody } else { opts.body = directBody }
31
- opts.headers = applyDefault(cloneLowerCase(opts.headers), _default.headers)
74
+ const headers = cloneLowerCase(opts.headers)
75
+ if (headers.range !== undefined && headers['accept-encoding'] === undefined) headers['accept-encoding'] = 'identity' // a range of the compressed body cannot be decompressed
76
+ opts.headers = applyDefault(headers, _default.headers)
32
77
  opts = applyDefault(opts, _default)
33
78
  opts.remainingRetry = opts.remainingRetry ?? opts.maxRetry
34
79
  opts.remainingRedirects = opts.remainingRedirects ?? opts.maxRedirects
@@ -68,10 +113,11 @@ function extend (defaultOptions = {}) {
68
113
  if (requestAbortedOrEnded === true) return
69
114
  requestAbortedOrEnded = true
70
115
  if (err) {
116
+ const aborted = req.rockAborted === true // read before destroying the streams below (req included)
71
117
  streamToCleanOnError.forEach(s => s.destroy(err))
72
- if (opts.retryOnError.indexOf(err?.code) !== -1 && --opts.remainingRetry > 0) {
118
+ if (aborted === false && opts.retryOnError.indexOf(err?.code) !== -1 && opts.remainingRetry-- > 0) {
73
119
  opts.prevError = err
74
- return setTimeout(rock, opts.retryDelay, opts, cb) // retry
120
+ return retry(opts, cb)
75
121
  }
76
122
  return cb(err)
77
123
  }
@@ -92,11 +138,12 @@ function extend (defaultOptions = {}) {
92
138
  }
93
139
  const req = protocol.request(opts, res => {
94
140
  opts.prevStatusCode = res.statusCode
141
+ let retryWait
95
142
  // retry and leave
96
- if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && opts.remainingRetry-- > 0) {
143
+ if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && (retryWait = retryAfter(res)) <= (opts.retryMaxDelay ?? 2000) && opts.remainingRetry-- > 0) {
97
144
  requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
98
145
  res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
99
- return setTimeout(rock, opts.retryDelay, opts, cb) // retry later
146
+ return retry(opts, cb, retryWait) // retry later
100
147
  }
101
148
  // or redirect and leave
102
149
  if (opts.followRedirects !== false && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
@@ -113,8 +160,8 @@ function extend (defaultOptions = {}) {
113
160
  delete opts.headers.cookie
114
161
  delete opts.headers.authorization
115
162
  }
116
- if (opts.method === 'POST' && [301, 302].includes(res.statusCode)) {
117
- opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35)
163
+ if ((opts.method === 'POST' && [301, 302].includes(res.statusCode)) || (res.statusCode === 303 && opts.method && opts.method !== 'GET' && opts.method !== 'HEAD')) {
164
+ opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35). On 303, change all methods except HEAD to GET (RFC 9110)
118
165
  delete opts.headers['content-length']; delete opts.headers['content-type']; delete opts.body; delete opts.form
119
166
  }
120
167
  if (opts.remainingRedirects-- === 0) {
@@ -125,21 +172,24 @@ function extend (defaultOptions = {}) {
125
172
  }
126
173
  // or read response and leave at the end
127
174
  response = res
128
- const contentEncoding = opts.method !== 'HEAD' ? (res.headers['content-encoding'] || '').toLowerCase() : ''
175
+ const status = res.statusCode // no body (HEAD, 204, 205, 304. Node never gives 1xx here) or a range of the compressed body (206): never decompress
176
+ const contentEncoding = opts.decompress !== false && opts.method !== 'HEAD' && status !== 204 && status !== 205 && status !== 206 && status !== 304 ? (res.headers['content-encoding'] || '').toLowerCase() : ''
129
177
  const output = opts.output ? opts.output(opts, res) : new Writable({ write (chunk, enc, wcb) { chunks.push(chunk); wcb() } })
130
178
  switch (contentEncoding) {
131
179
  case 'br':
132
180
  listen(res).pipe(listen(zlib.createBrotliDecompress())).pipe(listen(output, true)); break
133
181
  case 'gzip':
182
+ case 'x-gzip':
134
183
  case 'deflate':
135
184
  listen(res).pipe(listen(zlib.createUnzip())).pipe(listen(output, true)); break
136
185
  default:
137
186
  listen(res).pipe(listen(output, true)); break
138
187
  }
139
188
  })
189
+ req.destroy = abort
140
190
  req.once('timeout', () => {
141
191
  const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT'
142
- req.destroy(_error) // we must destroy manually and send the error to the error listener to call onRequestEnd
192
+ destroy.call(req, _error) // we must destroy manually and send the error to the error listener to call onRequestEnd. Not a user abort: retried
143
193
  })
144
194
 
145
195
  if (isFnStream(body) === true) listen(body(opts)).pipe(listen(req))
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "rock-req",
3
3
  "description": "Ultra-light (150 LOC, No dependencies) & Ultra-fast request library with reliable retry on failure, http/https, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout",
4
- "version": "5.2.1",
4
+ "version": "5.3.0",
5
5
  "author": {
6
6
  "name": "David Grelaud & Feross Aboukhadijeh"
7
7
  },