rock-req 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/LICENSE +20 -0
  2. package/README.md +580 -0
  3. package/index.js +155 -0
  4. package/package.json +47 -0
package/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) Feross Aboukhadijeh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
6
+ this software and associated documentation files (the "Software"), to deal in
7
+ the Software without restriction, including without limitation the rights to
8
+ use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9
+ the Software, and to permit persons to whom the Software is furnished to do so,
10
+ subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17
+ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18
+ COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19
+ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,580 @@
1
+
2
+ <p align="center">
3
+ <img src="https://raw.githubusercontent.com/carboneio/rock-req/master/doc/rock-req-logo.svg" alt="rock-req logo" height="120"/>
4
+ </p>
5
+
6
+ <h1 align="center" style="border-bottom:none; font-size: 2.2em;">Rock-req.js</h1>
7
+
8
+ <p align="center">Ensure your HTTP requests always reach their destination!</p>
9
+
10
+ [![javascript style guide][standard-image]][standard-url]
11
+
12
+ [standard-image]: https://img.shields.io/badge/code_style-standard-brightgreen.svg
13
+ [standard-url]: https://standardjs.com
14
+
15
+
16
+ ## 🔥 Why should you need this?
17
+
18
+ In most existing libraries (2023):
19
+
20
+ - Managing **reliable retries** is difficult, tricky with streams and not battle-tested
21
+ - Using **multiple forward proxies** has several benefits like **higher availability** and **increased bandwidth** but
22
+ **Intercepting retries** to use another Egress controller between two requests is not possible.
23
+ - Many request libraries are heavy: node-fetch, superagent, needle, got, axios, request
24
+ - Lightweight alternatives are not as light as they claim due to dependencies (simple-get, tiny-req, puny-req, ...)
25
+
26
+ ⚡️ **Rock-req** solves these problems with only **160 lines of code** and **zero dependencies**
27
+
28
+ It also supports many features:
29
+
30
+ - Follows redirects
31
+ - Handles gzip/deflate/brotli responses
32
+ - Modify defaults
33
+ - Extend and create new instances
34
+ - Automatically destroy input/output stream on error (pipeline)
35
+ - Composable
36
+ - Timeouts
37
+ - HTTPS / HTTP
38
+ - Composes well with npm packages for features like cookies, proxies, form data, & OAuth
39
+ - Typescript definition
40
+ - Keep 98% of the `simple-get` API (fork source)
41
+
42
+ Like NodeJS pipeline, when the callback is called, the request is 100% finished, even with streams.
43
+
44
+ ## Install
45
+
46
+
47
+ ```
48
+ npm install rock-req
49
+ ```
50
+
51
+ ## Usage
52
+
53
+ All functions accept two or three parameters:
54
+
55
+ ```js
56
+ const rock = require('rock-req')
57
+ rock(optsOrURL [, bodyOrStreamFn], callback)
58
+ ````
59
+
60
+ - `optsOrURL` can be an object or a the URL
61
+ - `bodyOrStreamFn` can be a buffer/string/object or a function returning an input stream for sending the body of POST/PUT/PATCH/DELETE requests
62
+ - `callback(err, res, data)` called only when everything is finished (even with streams).
63
+
64
+
65
+ ### GET, HEAD requests
66
+
67
+ ```js
68
+ // res is the server response, already consumed by rock-req. The result is in data
69
+ rock.get('http://ex.com', (err, res, data) => {
70
+ console.log(res.statusCode) // 200
71
+ console.log(data) // Buffer('server response')
72
+ })
73
+ ```
74
+
75
+ Alternative syntax:
76
+
77
+ ```js
78
+ rock({ method: 'GET', url: 'http://ex.com' }, function (err, res, data) {} )
79
+ // OR
80
+ rock.concat({ method: 'GET', url: 'http://ex.com' }, function (err, res, data) {} )
81
+ ```
82
+
83
+ Head requests:
84
+
85
+ ```js
86
+ rock.head('http://example.com', (err, res, data) => {})
87
+ ```
88
+
89
+ ### POST, PUT, PATCH, DELETE requests
90
+
91
+ Use the second paramater to pass the body:
92
+
93
+ ```js
94
+ rock.post('http://ex.com', 'POST body', (err, res, data) => {})
95
+ ```
96
+
97
+ Alternative syntax:
98
+
99
+ ```js
100
+ rock({ method: 'POST', url : 'http://ex.com', body : 'POST body' }, function (err, res, data) {} )
101
+ ```
102
+
103
+ ### JSON request shortcuts
104
+
105
+ Automatically serialize/deserialize request and response with JSON with `getJSON`, `putJSON`, `postJSON`, `deleteJSON`, ...
106
+
107
+ ```js
108
+ rock.putJSON('http://ex.com', { id : 123 }, (err, res, data) => {
109
+ console.log(data) // already JSON.parsed
110
+ })
111
+ ```
112
+
113
+ Alternative syntax:
114
+
115
+ ```js
116
+ rock({ method: 'PUT', url: 'http://ex.com', body: { id : 123 }, json: true }, function (err, res, data) {} )
117
+ ```
118
+
119
+ ### All options:
120
+
121
+
122
+ ```js
123
+ const opts = {
124
+ url : 'http://example.com',
125
+ method : 'POST',
126
+ body : 'this is the POST body',
127
+ headers: {
128
+ 'user-agent': 'my cool app'
129
+ }
130
+ }
131
+ rock(opts, function (err, res, data) {} )
132
+ ```
133
+
134
+ **opts** can contain any value of NodeJS http.request with rock-req parameters. Here are the most used one:
135
+
136
+ - `maxRedirects <number>`overwrite global maximum number of redirects. Defaults to 10
137
+ - `maxRetry <number>` overwrite global maximum number of retries. Defaults to 0
138
+ - `followRedirects <boolean>` do not follow redirects
139
+ - `body <buffer> | <string> | <object> | <function>` body to post
140
+ - `json <boolean>` automatically stringify/parse request/response Default : false
141
+ - `url <string>` the destination URL
142
+ - `method <string>` A string specifying the HTTP request method. Default: 'GET'.
143
+ - `headers <object>` An object containing request headers. Default: 'accept-encoding': 'gzip, deflate, br'
144
+ - `timeout <number>`: A number specifying the socket timeout in milliseconds.
145
+ - `auth <string>` Basic authentication ('user:password') to compute an Authorization header.
146
+ - `port <number>` Port of remote server. Default: defaultPort if set, else 80.
147
+ - `host <string>` A domain name or IP address of the server to issue the request to. Default: 'localhost'.
148
+ - `hostname <string>` Alias for host.
149
+ - `path <string>` Request path. Should include query string if any. E.G. '/index.html?page=12'.
150
+ - `protocol <string>` Protocol to use. Default: 'http:'.
151
+ - `setHost <boolean>`: Specifies whether or not to automatically add the Host header. Defaults to true.
152
+ - `agent <http.Agent> | <boolean>` Controls Agent behavior. Possible values:
153
+ - `undefined` (default): use http.globalAgent for this host and port.
154
+ - `Agent object`: explicitly use the passed in Agent.
155
+ - `false` causes a new Agent with default values to be used.
156
+
157
+
158
+ ### Input Stream
159
+
160
+ Rock-req requires that input stream is initialized in a function.
161
+
162
+ This function is invoked by rock-req for every request retry.
163
+ If something goes wrong, the old stream is destroyed.
164
+
165
+ ```js
166
+ const rock = require('rock-req')
167
+ const fs = require('fs')
168
+
169
+ // opts contains options passed in rock(opts). DO NOT MODIFY IT
170
+ function createInputStream(opts) {
171
+ return fs.createReadStream('input.txt');
172
+ }
173
+
174
+ const opts = {
175
+ url : 'http://example.com',
176
+ body: createInputStream
177
+ }
178
+ rock(opts, function (err, res, data) {})
179
+ ```
180
+
181
+ Alternative syntax:
182
+
183
+ ```js
184
+ rock.post('http://example.com', createInputStream, function (err, res, data) {})
185
+ // or with more object options:
186
+ rock.post(opts, createInputStream, function (err, res, data) {})
187
+ ```
188
+
189
+ ### Output Stream
190
+
191
+ Rock-req requires that output stream is initialized in a function.
192
+ This function is invoked by rock-req for every request retry.
193
+
194
+ ```js
195
+ const rock = require('rock-req')
196
+ const fs = require('fs')
197
+ const { finished } = require('stream')
198
+
199
+ // opts contains options passed in rock(opts). DO NOT MODIFY IT
200
+ // res if the http response (res.statusCode, ...). DO NOT MODIFY IT and DO NOT CONSUME THE RES STREAM YOURSELF
201
+ function createOutputStream(opts, res) {
202
+
203
+ const writer = fs.createWriteStream('test_gfg.txt')
204
+ // Internally, rock-req uses pipeline. If something goes wrong, the stream is destroyed automatically.
205
+ // If you need to do some action (removing temporary files, ...), uses this native NodeJS method:
206
+ const cleanup = finished(writer, (err) => {
207
+ if (err) {
208
+ // clean up things
209
+ }
210
+ // When using the finished() method in NodeJS, it's important to be aware that it can leave some event listeners
211
+ // (specifically, the 'error', 'end', 'finish', and 'close' events) hanging around even after this callback function has been called.
212
+ // This is intentional, as it helps prevent unexpected crashes if an error occurs due to incorrect stream implementations.
213
+ // However, if you don't want these event listeners to stick around after the callback function has been called,
214
+ // you can use the cleanup function that's returned by stream.finished() to remove them.
215
+ // You'll need to explicitly call this cleanup function within your callback function to ensure that the event listeners get removed properly.
216
+ cleanup();
217
+ });
218
+ // It must return a Writable stream. Otherwise, the request is cancel with an error
219
+ return writer
220
+ }
221
+
222
+ const opts = {
223
+ url : 'http://example.com',
224
+ output : createOutputStream
225
+ }
226
+ rock(opts, function (err, res) {})
227
+ ```
228
+
229
+ ### Retry on failure
230
+
231
+ By default, rock-req retries with the following errors if `maxRetry > 1`.
232
+
233
+ The callback is called when all the request succeed or all retries are done
234
+
235
+ ```js
236
+ const rock = require('rock-req');
237
+
238
+ // default values can be overwritten like this:
239
+ rock.defaults.retryOnCode = [
240
+ 408, /* Request Timeout */
241
+ 429, /* Too Many Requests */
242
+ 500, /* Internal Server Error */
243
+ 502, /* Bad Gateway */
244
+ 503, /* Service Unavailable */
245
+ 504, /* Gateway Timeout*/
246
+ 521, /* Web Server Is Down*/
247
+ 522, /* Cloudflare Connection Timed Out */
248
+ 524 /* Cloudflare A Timeout Occurred */
249
+ ];
250
+ rock.defaults.retryOnError = [
251
+ 'ETIMEDOUT', /* One of the timeout limits was reached. */
252
+ 'ECONNRESET', /* The connection was forcibly closed. */
253
+ 'EADDRINUSE', /* Could not bind to any free port */
254
+ 'ECONNREFUSED', /* The connection was refused by the server. */
255
+ 'EPIPE', /* The remote side of the stream being written has been closed. */
256
+ 'ENOTFOUND', /* Could not resolve the hostname to an IP address. */
257
+ 'ENETUNREACH', /* No internet connection. */
258
+ 'EAI_AGAIN' /* DNS lookup timed out. */
259
+ ];
260
+
261
+ const opts = {
262
+ url : 'http://example.com',
263
+ body : 'this is the POST body',
264
+ maxRetry : 2 // 0 is the default value (= no retries)
265
+ }
266
+ rock(opts, function (err, res, data) {} );
267
+ ```
268
+
269
+
270
+ ### Global options
271
+
272
+ Change default parameters globally (not recommended), or create a new instance with specific paramaters (see below)
273
+
274
+ ```js
275
+ rock.defaults = {
276
+ headers : { 'accept-encoding': 'gzip, deflate, br' },
277
+ maxRedirects : 10,
278
+ maxRetry : 0,
279
+ retryDelay : 100, //ms
280
+ retryOnCode : [408, 429, 500, 502, 503, 504, 521, 522, 524 ],
281
+ retryOnError : ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED','EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN' ],
282
+ // beforeRequest is called for each request, retry and redirect
283
+ beforeRequest : (opts) => {
284
+ // There options can be overwritted (= parsed opts.url)
285
+ opts.protocol = 'https:' // or 'http:'
286
+ opts.hostname = 'google.com';
287
+ opts.port = 443;
288
+ opts.path = '/mypage.html?bla=1#hash';
289
+ opts.auth = '';
290
+ opts.headers = {};
291
+ opts.body = {};
292
+ opts.method = 'POST';
293
+ opts.remainingRetry;
294
+ opts.remainingRedirects;
295
+
296
+ // READ-ONLY options (not exhaustive)
297
+ opts.url; // DOT NOT OVERWRITE
298
+ opts.maxRetry;
299
+ opts.maxRedirects;
300
+ opts.prevError; // error of previous request on retry
301
+ opts.prevStatusCode; // HTTP status code of previous request on retry/redirect
302
+
303
+ // opts must be returned
304
+ return opts;
305
+ },
306
+ }
307
+ ```
308
+
309
+ ### Extend and intercept retries
310
+
311
+ Create a new instance with specific parameter instead of modifying `rock.defaults`
312
+
313
+ By default, this new instance inherits values of the instance source if options are not overwritten.
314
+ Headers are merged. Then only the first level of the options object is merged (no deep travelling in sub-objects or arrays).
315
+
316
+ Here is a basic example of `beforeRequest` interceptor to use [HAProxy as a forward proxy](https://www.haproxy.com/user-spotlight-series/haproxy-as-egress-controller/).
317
+
318
+ `beforeRequest` is always called on each redirect/retry.
319
+ - on redirect, `opts.url` (and `hostname`, `port`, `protocol`, `path`) is updated to the new location
320
+ - on retry, `opts.url` (and `hostname`, `port`, `protocol`, `path`) have the same value as they did
321
+ when the rock-req was initially called.
322
+
323
+
324
+ ```js
325
+ const myInstance = rock.extend({
326
+ beforeRequest: (opts) => {
327
+ const { hostname, port, protocol, path } = opts;
328
+ opts.protocol = 'http:';
329
+ opts.hostname = '10.0.0.1';
330
+ opts.port = 80;
331
+ opts.path = `${encodeURIComponent(hostname)}/${port}${path}`;
332
+ return opts;
333
+ },
334
+ headers: {
335
+ 'Custom-header': 'x-for-proxy'
336
+ },
337
+ timeout : 1000
338
+ });
339
+
340
+ myInstance.get('http://example.com', function (err, res, data) {})
341
+
342
+ ```
343
+
344
+
345
+
346
+ ### Timeout
347
+
348
+ You can set a timeout (in milliseconds) on the request with the `timeout` option.
349
+ If the request takes longer than `timeout` to complete, then the entire request
350
+ will fail with an `Error`.
351
+
352
+ ```js
353
+ const rock = require('rock-req')
354
+
355
+ const opts = {
356
+ url: 'http://example.com',
357
+ timeout: 2000 // 2 second timeout
358
+ }
359
+
360
+ rock(opts, function (err, res, data) {})
361
+ ```
362
+
363
+
364
+ ### One Quick Tip
365
+
366
+ It's a good idea to set the `'user-agent'` header so the provider can more easily
367
+ see how their resource is used.
368
+
369
+ ```js
370
+ const rock = require('rock-req')
371
+ const pkg = require('./package.json')
372
+
373
+ rock({
374
+ url : 'http://example.com',
375
+ headers: {
376
+ 'user-agent': `my-module/${pkg.version} (https://github.com/username/my-module)`
377
+ }
378
+ }, function (err, res, data) {})
379
+ ```
380
+
381
+ ### Proxies
382
+
383
+ You can use the [`tunnel`](https://github.com/koichik/node-tunnel) module with the
384
+ `agent` option to work with proxies:
385
+
386
+ ```js
387
+ const rock = require('rock-req')
388
+ const tunnel = require('tunnel')
389
+
390
+ const opts = {
391
+ url: 'http://example.com',
392
+ agent: tunnel.httpOverHttp({
393
+ proxy: {
394
+ host: 'localhost'
395
+ }
396
+ })
397
+ }
398
+
399
+ rock(opts, function (err, res, data) {})
400
+ ```
401
+
402
+ ### Cookies
403
+
404
+ You can use the [`cookie`](https://github.com/jshttp/cookie) module to include
405
+ cookies in a request:
406
+
407
+ You can extend and create a new instance of rock-req to keep the cookie in header for each request.
408
+
409
+ ```js
410
+ const rock = require('rock-req')
411
+ const cookie = require('cookie')
412
+
413
+ const opts = {
414
+ url: 'http://example.com',
415
+ headers: {
416
+ cookie: cookie.serialize('foo', 'bar')
417
+ }
418
+ }
419
+
420
+ rock(opts, function (err, res, data) {})
421
+ ```
422
+
423
+ ### Form data
424
+
425
+ You can use the [`form-data`](https://github.com/form-data/form-data) module to
426
+ create POST request with form data:
427
+
428
+ ```js
429
+ const fs = require('fs')
430
+ const rock = require('rock-req')
431
+ const FormData = require('form-data')
432
+ const form = new FormData()
433
+
434
+ const opts = {
435
+ url: 'http://example.com',
436
+ body: () => {
437
+ form.append('my_file', fs.createReadStream('/foo/bar.jpg'))
438
+ }
439
+ }
440
+
441
+ rock.post(opts, function (err, res, data) {})
442
+ ```
443
+
444
+ #### Or, include `application/x-www-form-urlencoded` form data manually:
445
+
446
+ ```js
447
+ const rock = require('rock-req')
448
+
449
+ const opts = {
450
+ url: 'http://example.com',
451
+ form: {
452
+ key: 'value'
453
+ }
454
+ }
455
+ rock.post(opts, function (err, res, data) {})
456
+ ```
457
+
458
+ ### Specifically disallowing redirects
459
+
460
+ ```js
461
+ const rock = require('rock-req')
462
+
463
+ const opts = {
464
+ url: 'http://example.com/will-redirect-elsewhere',
465
+ followRedirects: false
466
+ }
467
+ // res.statusCode will be 301, no error thrown
468
+ rock(opts, function (err, res, data) {})
469
+ ```
470
+
471
+ ### Basic Auth
472
+
473
+ ```js
474
+ const user = 'someuser'
475
+ const pass = 'pa$$word'
476
+ const encodedAuth = Buffer.from(`${user}:${pass}`).toString('base64')
477
+
478
+ rock('http://example.com', {
479
+ headers: {
480
+ authorization: `Basic ${encodedAuth}`
481
+ }
482
+ })
483
+ ```
484
+
485
+ ### OAuth
486
+
487
+ You can use the [`oauth-1.0a`](https://github.com/ddo/oauth-1.0a) module to create
488
+ a signed OAuth request:
489
+
490
+ ```js
491
+ const rock = require('rock-req')
492
+ const crypto = require('crypto')
493
+ const OAuth = require('oauth-1.0a')
494
+
495
+ const oauth = OAuth({
496
+ consumer: {
497
+ key: process.env.CONSUMER_KEY,
498
+ secret: process.env.CONSUMER_SECRET
499
+ },
500
+ signature_method: 'HMAC-SHA1',
501
+ hash_function: (baseString, key) => crypto.createHmac('sha1', key).update(baseString).digest('base64')
502
+ })
503
+
504
+ const token = {
505
+ key: process.env.ACCESS_TOKEN,
506
+ secret: process.env.ACCESS_TOKEN_SECRET
507
+ }
508
+
509
+ const url = 'https://api.twitter.com/1.1/statuses/home_timeline.json'
510
+
511
+ const opts = {
512
+ url: url,
513
+ headers: oauth.toHeader(oauth.authorize({url, method: 'GET'}, token)),
514
+ json: true
515
+ }
516
+
517
+ rock(opts, function (err, res) {})
518
+ ```
519
+
520
+ ### Throttle requests
521
+
522
+ You can use [limiter](https://github.com/jhurliman/node-rate-limiter) to throttle requests. This is useful when calling an API that is rate limited.
523
+
524
+ ```js
525
+ const rock = require('rock-get')
526
+ const RateLimiter = require('limiter').RateLimiter
527
+ const limiter = new RateLimiter(1, 'second')
528
+
529
+ const rock = (opts, cb) => limiter.removeTokens(1, () => rock(opts, cb))
530
+ rock.concat = (opts, cb) => limiter.removeTokens(1, () => rock.concat(opts, cb))
531
+
532
+ var opts = {
533
+ url: 'http://example.com'
534
+ }
535
+
536
+ rock.concat(opts, processResult)
537
+ rock.concat(opts, processResult)
538
+
539
+ function processResult (err, res, data) {
540
+ if (err) throw err
541
+ console.log(data.toString())
542
+ }
543
+ ```
544
+
545
+
546
+ ## Breaking change for people coming from simplet-get
547
+
548
+ Rock-req is a fork of [simple-get](https://github.com/feross/simple-get)
549
+
550
+ - Rock-req concatenates chunks and returns the concatenated buffer/JSON in the third paramater of the callback.
551
+ Why? because 80% of the time, we need to do simple request without streams.
552
+ When you need to pipe the result to another stream, you must use `opts.output` parameter (see "Output Stream" in the doc)
553
+ - All streams must be created in a function. Rock-req return an error if it is not the case
554
+ - before: `body = stream`
555
+ - after `body = () => { const myStream = create(); return myStream; }`
556
+
557
+
558
+ ## Notes:
559
+
560
+ - [ ] replace deprecated `url.parse` by `new URL` but new URL is slower than url.parse. Let's see if Node 20 LTS is faster
561
+ - [ ] agent keep Alive
562
+ - [ ] add advanced timeout (response timeout)
563
+ - [ ] test prevError
564
+
565
+
566
+ # Supporters
567
+
568
+ This packaged in maintained by Carbone:
569
+
570
+ <p>
571
+ <a href="https://carbone.io" alt="Carbone.io - Efficient PDF / DOCX / XLSX / CSV / HTML / XML generator with templates and JSON">
572
+ <img src="https://raw.githubusercontent.com/carboneio/rock-req/master/doc/carbone-logo.svg" alt="Carbone.io logo" height="60"/>
573
+ </a>
574
+ </p>
575
+
576
+
577
+ Thank you [Feross Aboukhadijeh](https://github.com/feross), creator of `simple-get`
578
+
579
+
580
+
package/index.js ADDED
@@ -0,0 +1,155 @@
1
+ module.exports = extend()
2
+
3
+ const http = require('http')
4
+ const https = require('https')
5
+ const querystring = require('querystring')
6
+ const url = require('url')
7
+ const zlib = require('zlib')
8
+ const { pipeline, Writable, Readable } = require('stream')
9
+
10
+ const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === 'function'
11
+ const isFnStream = o => o instanceof Function
12
+ function applyDefault (t, d) { for (const a in d) { if (t[a] === undefined) t[a] = d[a] } return t }
13
+ function cloneLowerCase (s) { const n = {}; for (const a in s) { n[a.toLowerCase()] = s[a] } return n }
14
+
15
+ function extend (defaultOptions = {}) {
16
+ let _default = {
17
+ headers: { 'accept-encoding': 'gzip, deflate, br' },
18
+ maxRedirects: 10,
19
+ maxRetry: 0,
20
+ retryDelay: 100, // ms
21
+ retryOnCode: [408, 429, 500, 502, 503, 504, 521, 522, 524],
22
+ retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'],
23
+ beforeRequest: o => o
24
+ }
25
+ defaultOptions.headers = applyDefault(cloneLowerCase(defaultOptions.headers), _default.headers)
26
+ _default = applyDefault(defaultOptions, _default) // inherits of parent options
27
+
28
+ // all options https://nodejs.org/dist/latest-v18.x/docs/api/http.html#http_http_request_url_options_callback
29
+ function rock (opts, directBody, cb) {
30
+ if (typeof opts === 'string') opts = { url: opts }
31
+ if (!cb) { cb = directBody } else { opts.body = directBody }
32
+ opts.headers = applyDefault(cloneLowerCase(opts.headers), _default.headers)
33
+ opts = applyDefault(opts, _default)
34
+ opts.remainingRetry = opts.remainingRetry ?? opts.maxRetry
35
+ opts.remainingRedirects = opts.remainingRedirects ?? opts.maxRedirects
36
+
37
+ if (opts.url) {
38
+ const { hostname, port, protocol, auth, path } = url.parse(opts.url) // eslint-disable-line node/no-deprecated-api
39
+ if (!hostname && !port && !protocol && !auth) opts.path = path // Relative redirect
40
+ else Object.assign(opts, { hostname, port, protocol, auth, path }) // Absolute redirect
41
+ }
42
+ opts = opts.beforeRequest(opts)
43
+
44
+ let body
45
+ if (opts.body) {
46
+ body = opts.json && !isFnStream(opts.body) && !isStream(opts.body) ? JSON.stringify(opts.body) : opts.body
47
+ } else if (opts.form) {
48
+ body = typeof opts.form === 'string' ? opts.form : querystring.stringify(opts.form)
49
+ opts.headers['content-type'] = 'application/x-www-form-urlencoded'
50
+ }
51
+ if (body) {
52
+ if (isStream(body)) return cb(new Error('opts.body must be a function returning a Readable stream. RTFM'))
53
+ if (!opts.method) opts.method = 'POST'
54
+ if (!isFnStream(body)) opts.headers['content-length'] = Buffer.byteLength(body)
55
+ if (opts.json && !opts.form) opts.headers['content-type'] = 'application/json'
56
+ }
57
+ if (opts.output && (isStream(opts.output) || !isFnStream(opts.output))) return cb(new Error('opts.output must be a function returning a Writable stream. RTFM'))
58
+ if (opts.json) opts.headers.accept = 'application/json'
59
+ if (opts.method) opts.method = opts.method.toUpperCase()
60
+
61
+ const originalHost = opts.hostname // hostname before potential redirect
62
+ const protocol = opts.protocol === 'https:' ? https : http // Support http/https urls
63
+ const chunks = []
64
+ let requestAbortedOrEnded = false
65
+ let response = null
66
+ function onRequestEnd (err) {
67
+ if (requestAbortedOrEnded === true) return
68
+ requestAbortedOrEnded = true
69
+ if (opts.retryOnError.indexOf(err?.code) !== -1 && --opts.remainingRetry > 0) {
70
+ opts.prevError = err
71
+ return setTimeout(rock, opts.retryDelay, opts, cb) // retry in 100ms
72
+ }
73
+ if (err) return cb(err)
74
+ let data = Buffer.concat(chunks)
75
+ if (opts.json) {
76
+ try { data = JSON.parse(data.toString()) } catch (e) { return cb(e, response, data) }
77
+ }
78
+ cb(null, response, data)
79
+ }
80
+ const req = protocol.request(opts, res => {
81
+ opts.prevStatusCode = res.statusCode
82
+ // retry and leave
83
+ if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && opts.remainingRetry-- > 0) {
84
+ requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
85
+ res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
86
+ return setTimeout(rock, opts.retryDelay, opts, cb) // retry later
87
+ }
88
+
89
+ // or redirect and leave
90
+ if (opts.followRedirects !== false && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
91
+ requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
92
+ res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
93
+ opts.url = res.headers.location // Follow 3xx redirects
94
+ delete opts.headers.host // Discard `host` header on redirect (see #32)
95
+ const redirectHost = url.parse(opts.url).hostname // eslint-disable-line node/no-deprecated-api
96
+ // If redirected host is different than original host, drop headers to prevent cookie leak (#73)
97
+ if (redirectHost !== null && redirectHost !== originalHost) {
98
+ delete opts.headers.cookie
99
+ delete opts.headers.authorization
100
+ }
101
+ if (opts.method === 'POST' && [301, 302].includes(res.statusCode)) {
102
+ opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35)
103
+ delete opts.headers['content-length']; delete opts.headers['content-type']; delete opts.body; delete opts.form
104
+ }
105
+ if (opts.remainingRedirects-- === 0) {
106
+ requestAbortedOrEnded = false
107
+ return onRequestEnd(new Error('too many redirects'))
108
+ }
109
+ return rock(opts, cb)
110
+ }
111
+
112
+ // or read response and leave at the end
113
+ response = res
114
+ const contentEncoding = opts.method !== 'HEAD' ? (res.headers['content-encoding'] || '').toLowerCase() : ''
115
+ const output = opts.output ? opts.output(opts, res) : new Writable({ write (chunk, enc, wcb) { chunks.push(chunk); wcb() } })
116
+ switch (contentEncoding) {
117
+ case 'br':
118
+ pipeline(res, zlib.createBrotliDecompress(), output, onRequestEnd); break
119
+ case 'gzip':
120
+ case 'deflate':
121
+ pipeline(res, zlib.createUnzip(), output, onRequestEnd); break
122
+ default:
123
+ pipeline(res, output, onRequestEnd); break
124
+ }
125
+ })
126
+ req.once('timeout', () => {
127
+ const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT'
128
+ req.destroy() // we must destroy manually
129
+ onRequestEnd(_error) // This timeout event can come after the input pipeline is finished (ex. timeout with no body)
130
+ })
131
+
132
+ const _inputStream = isFnStream(body) ? body(opts) : Readable.from([body], { objectMode: false })
133
+ pipeline(_inputStream, req, (e) => {
134
+ if (e) onRequestEnd(e)
135
+ })
136
+
137
+ return req
138
+ }
139
+
140
+ ;['get', 'post', 'put', 'patch', 'head', 'delete', 'getJSON', 'postJSON', 'putJSON', 'patchJSON', 'headJSON', 'deleteJSON'].forEach(method => {
141
+ const jsonShortcut = /JSON$/.test(method) === true
142
+ const methodShortcut = jsonShortcut === true ? method.toUpperCase().slice(0, -4) : method.toUpperCase()
143
+ rock[method] = (opts, body, cb) => {
144
+ if (typeof opts === 'string') opts = { url: opts }
145
+ opts.method = methodShortcut
146
+ opts.json = jsonShortcut
147
+ return rock(opts, body, cb)
148
+ }
149
+ })
150
+ rock.concat = rock
151
+ rock.defaults = _default
152
+ rock.extend = extend
153
+
154
+ return rock
155
+ }
package/package.json ADDED
@@ -0,0 +1,47 @@
1
+ {
2
+ "name": "rock-req",
3
+ "description": "Zero dependencies (160 LOC) & rock-solid request library: http/https, reliable retry on failure, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout",
4
+ "version": "5.0.0",
5
+ "author": {
6
+ "name": "David Grelaud & Feross Aboukhadijeh"
7
+ },
8
+ "devDependencies": {
9
+ "self-signed-https": "1.0.5",
10
+ "standard": "*",
11
+ "string-to-stream": "3.0.1",
12
+ "tape": "5.6.3"
13
+ },
14
+ "engines": {
15
+ "node": ">=16"
16
+ },
17
+ "files": [
18
+ "index.js"
19
+ ],
20
+ "keywords": [
21
+ "request",
22
+ "http",
23
+ "get request",
24
+ "http.get",
25
+ "redirects",
26
+ "retry",
27
+ "follow redirects",
28
+ "gzip",
29
+ "deflate",
30
+ "https",
31
+ "stream",
32
+ "simple request",
33
+ "light request",
34
+ "kiss request",
35
+ "tiny request",
36
+ "simple get"
37
+ ],
38
+ "license": "MIT",
39
+ "main": "index.js",
40
+ "repository": {
41
+ "type": "git",
42
+ "url": "git://github.com/carboneio/rock-req.git"
43
+ },
44
+ "scripts": {
45
+ "test": "tape test/*.js"
46
+ }
47
+ }