rock-req 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +20 -0
- package/README.md +580 -0
- package/index.js +155 -0
- package/package.json +47 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) Feross Aboukhadijeh
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
|
7
|
+
the Software without restriction, including without limitation the rights to
|
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
|
10
|
+
subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,580 @@
|
|
|
1
|
+
|
|
2
|
+
<p align="center">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/carboneio/rock-req/master/doc/rock-req-logo.svg" alt="rock-req logo" height="120"/>
|
|
4
|
+
</p>
|
|
5
|
+
|
|
6
|
+
<h1 align="center" style="border-bottom:none; font-size: 2.2em;">Rock-req.js</h1>
|
|
7
|
+
|
|
8
|
+
<p align="center">Ensure your HTTP requests always reach their destination!</p>
|
|
9
|
+
|
|
10
|
+
[![javascript style guide][standard-image]][standard-url]
|
|
11
|
+
|
|
12
|
+
[standard-image]: https://img.shields.io/badge/code_style-standard-brightgreen.svg
|
|
13
|
+
[standard-url]: https://standardjs.com
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
## 🔥 Why should you need this?
|
|
17
|
+
|
|
18
|
+
In most existing libraries (2023):
|
|
19
|
+
|
|
20
|
+
- Managing **reliable retries** is difficult, tricky with streams and not battle-tested
|
|
21
|
+
- Using **multiple forward proxies** has several benefits like **higher availability** and **increased bandwidth** but
|
|
22
|
+
**Intercepting retries** to use another Egress controller between two requests is not possible.
|
|
23
|
+
- Many request libraries are heavy: node-fetch, superagent, needle, got, axios, request
|
|
24
|
+
- Lightweight alternatives are not as light as they claim due to dependencies (simple-get, tiny-req, puny-req, ...)
|
|
25
|
+
|
|
26
|
+
⚡️ **Rock-req** solves these problems with only **160 lines of code** and **zero dependencies**
|
|
27
|
+
|
|
28
|
+
It also supports many features:
|
|
29
|
+
|
|
30
|
+
- Follows redirects
|
|
31
|
+
- Handles gzip/deflate/brotli responses
|
|
32
|
+
- Modify defaults
|
|
33
|
+
- Extend and create new instances
|
|
34
|
+
- Automatically destroy input/output stream on error (pipeline)
|
|
35
|
+
- Composable
|
|
36
|
+
- Timeouts
|
|
37
|
+
- HTTPS / HTTP
|
|
38
|
+
- Composes well with npm packages for features like cookies, proxies, form data, & OAuth
|
|
39
|
+
- Typescript definition
|
|
40
|
+
- Keep 98% of the `simple-get` API (fork source)
|
|
41
|
+
|
|
42
|
+
Like NodeJS pipeline, when the callback is called, the request is 100% finished, even with streams.
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
```
|
|
48
|
+
npm install rock-req
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Usage
|
|
52
|
+
|
|
53
|
+
All functions accept two or three parameters:
|
|
54
|
+
|
|
55
|
+
```js
|
|
56
|
+
const rock = require('rock-req')
|
|
57
|
+
rock(optsOrURL [, bodyOrStreamFn], callback)
|
|
58
|
+
````
|
|
59
|
+
|
|
60
|
+
- `optsOrURL` can be an object or a the URL
|
|
61
|
+
- `bodyOrStreamFn` can be a buffer/string/object or a function returning an input stream for sending the body of POST/PUT/PATCH/DELETE requests
|
|
62
|
+
- `callback(err, res, data)` called only when everything is finished (even with streams).
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
### GET, HEAD requests
|
|
66
|
+
|
|
67
|
+
```js
|
|
68
|
+
// res is the server response, already consumed by rock-req. The result is in data
|
|
69
|
+
rock.get('http://ex.com', (err, res, data) => {
|
|
70
|
+
console.log(res.statusCode) // 200
|
|
71
|
+
console.log(data) // Buffer('server response')
|
|
72
|
+
})
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Alternative syntax:
|
|
76
|
+
|
|
77
|
+
```js
|
|
78
|
+
rock({ method: 'GET', url: 'http://ex.com' }, function (err, res, data) {} )
|
|
79
|
+
// OR
|
|
80
|
+
rock.concat({ method: 'GET', url: 'http://ex.com' }, function (err, res, data) {} )
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Head requests:
|
|
84
|
+
|
|
85
|
+
```js
|
|
86
|
+
rock.head('http://example.com', (err, res, data) => {})
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### POST, PUT, PATCH, DELETE requests
|
|
90
|
+
|
|
91
|
+
Use the second paramater to pass the body:
|
|
92
|
+
|
|
93
|
+
```js
|
|
94
|
+
rock.post('http://ex.com', 'POST body', (err, res, data) => {})
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Alternative syntax:
|
|
98
|
+
|
|
99
|
+
```js
|
|
100
|
+
rock({ method: 'POST', url : 'http://ex.com', body : 'POST body' }, function (err, res, data) {} )
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### JSON request shortcuts
|
|
104
|
+
|
|
105
|
+
Automatically serialize/deserialize request and response with JSON with `getJSON`, `putJSON`, `postJSON`, `deleteJSON`, ...
|
|
106
|
+
|
|
107
|
+
```js
|
|
108
|
+
rock.putJSON('http://ex.com', { id : 123 }, (err, res, data) => {
|
|
109
|
+
console.log(data) // already JSON.parsed
|
|
110
|
+
})
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Alternative syntax:
|
|
114
|
+
|
|
115
|
+
```js
|
|
116
|
+
rock({ method: 'PUT', url: 'http://ex.com', body: { id : 123 }, json: true }, function (err, res, data) {} )
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
### All options:
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
```js
|
|
123
|
+
const opts = {
|
|
124
|
+
url : 'http://example.com',
|
|
125
|
+
method : 'POST',
|
|
126
|
+
body : 'this is the POST body',
|
|
127
|
+
headers: {
|
|
128
|
+
'user-agent': 'my cool app'
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
rock(opts, function (err, res, data) {} )
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
**opts** can contain any value of NodeJS http.request with rock-req parameters. Here are the most used one:
|
|
135
|
+
|
|
136
|
+
- `maxRedirects <number>`overwrite global maximum number of redirects. Defaults to 10
|
|
137
|
+
- `maxRetry <number>` overwrite global maximum number of retries. Defaults to 0
|
|
138
|
+
- `followRedirects <boolean>` do not follow redirects
|
|
139
|
+
- `body <buffer> | <string> | <object> | <function>` body to post
|
|
140
|
+
- `json <boolean>` automatically stringify/parse request/response Default : false
|
|
141
|
+
- `url <string>` the destination URL
|
|
142
|
+
- `method <string>` A string specifying the HTTP request method. Default: 'GET'.
|
|
143
|
+
- `headers <object>` An object containing request headers. Default: 'accept-encoding': 'gzip, deflate, br'
|
|
144
|
+
- `timeout <number>`: A number specifying the socket timeout in milliseconds.
|
|
145
|
+
- `auth <string>` Basic authentication ('user:password') to compute an Authorization header.
|
|
146
|
+
- `port <number>` Port of remote server. Default: defaultPort if set, else 80.
|
|
147
|
+
- `host <string>` A domain name or IP address of the server to issue the request to. Default: 'localhost'.
|
|
148
|
+
- `hostname <string>` Alias for host.
|
|
149
|
+
- `path <string>` Request path. Should include query string if any. E.G. '/index.html?page=12'.
|
|
150
|
+
- `protocol <string>` Protocol to use. Default: 'http:'.
|
|
151
|
+
- `setHost <boolean>`: Specifies whether or not to automatically add the Host header. Defaults to true.
|
|
152
|
+
- `agent <http.Agent> | <boolean>` Controls Agent behavior. Possible values:
|
|
153
|
+
- `undefined` (default): use http.globalAgent for this host and port.
|
|
154
|
+
- `Agent object`: explicitly use the passed in Agent.
|
|
155
|
+
- `false` causes a new Agent with default values to be used.
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
### Input Stream
|
|
159
|
+
|
|
160
|
+
Rock-req requires that input stream is initialized in a function.
|
|
161
|
+
|
|
162
|
+
This function is invoked by rock-req for every request retry.
|
|
163
|
+
If something goes wrong, the old stream is destroyed.
|
|
164
|
+
|
|
165
|
+
```js
|
|
166
|
+
const rock = require('rock-req')
|
|
167
|
+
const fs = require('fs')
|
|
168
|
+
|
|
169
|
+
// opts contains options passed in rock(opts). DO NOT MODIFY IT
|
|
170
|
+
function createInputStream(opts) {
|
|
171
|
+
return fs.createReadStream('input.txt');
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const opts = {
|
|
175
|
+
url : 'http://example.com',
|
|
176
|
+
body: createInputStream
|
|
177
|
+
}
|
|
178
|
+
rock(opts, function (err, res, data) {})
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Alternative syntax:
|
|
182
|
+
|
|
183
|
+
```js
|
|
184
|
+
rock.post('http://example.com', createInputStream, function (err, res, data) {})
|
|
185
|
+
// or with more object options:
|
|
186
|
+
rock.post(opts, createInputStream, function (err, res, data) {})
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Output Stream
|
|
190
|
+
|
|
191
|
+
Rock-req requires that output stream is initialized in a function.
|
|
192
|
+
This function is invoked by rock-req for every request retry.
|
|
193
|
+
|
|
194
|
+
```js
|
|
195
|
+
const rock = require('rock-req')
|
|
196
|
+
const fs = require('fs')
|
|
197
|
+
const { finished } = require('stream')
|
|
198
|
+
|
|
199
|
+
// opts contains options passed in rock(opts). DO NOT MODIFY IT
|
|
200
|
+
// res if the http response (res.statusCode, ...). DO NOT MODIFY IT and DO NOT CONSUME THE RES STREAM YOURSELF
|
|
201
|
+
function createOutputStream(opts, res) {
|
|
202
|
+
|
|
203
|
+
const writer = fs.createWriteStream('test_gfg.txt')
|
|
204
|
+
// Internally, rock-req uses pipeline. If something goes wrong, the stream is destroyed automatically.
|
|
205
|
+
// If you need to do some action (removing temporary files, ...), uses this native NodeJS method:
|
|
206
|
+
const cleanup = finished(writer, (err) => {
|
|
207
|
+
if (err) {
|
|
208
|
+
// clean up things
|
|
209
|
+
}
|
|
210
|
+
// When using the finished() method in NodeJS, it's important to be aware that it can leave some event listeners
|
|
211
|
+
// (specifically, the 'error', 'end', 'finish', and 'close' events) hanging around even after this callback function has been called.
|
|
212
|
+
// This is intentional, as it helps prevent unexpected crashes if an error occurs due to incorrect stream implementations.
|
|
213
|
+
// However, if you don't want these event listeners to stick around after the callback function has been called,
|
|
214
|
+
// you can use the cleanup function that's returned by stream.finished() to remove them.
|
|
215
|
+
// You'll need to explicitly call this cleanup function within your callback function to ensure that the event listeners get removed properly.
|
|
216
|
+
cleanup();
|
|
217
|
+
});
|
|
218
|
+
// It must return a Writable stream. Otherwise, the request is cancel with an error
|
|
219
|
+
return writer
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const opts = {
|
|
223
|
+
url : 'http://example.com',
|
|
224
|
+
output : createOutputStream
|
|
225
|
+
}
|
|
226
|
+
rock(opts, function (err, res) {})
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Retry on failure
|
|
230
|
+
|
|
231
|
+
By default, rock-req retries with the following errors if `maxRetry > 1`.
|
|
232
|
+
|
|
233
|
+
The callback is called when all the request succeed or all retries are done
|
|
234
|
+
|
|
235
|
+
```js
|
|
236
|
+
const rock = require('rock-req');
|
|
237
|
+
|
|
238
|
+
// default values can be overwritten like this:
|
|
239
|
+
rock.defaults.retryOnCode = [
|
|
240
|
+
408, /* Request Timeout */
|
|
241
|
+
429, /* Too Many Requests */
|
|
242
|
+
500, /* Internal Server Error */
|
|
243
|
+
502, /* Bad Gateway */
|
|
244
|
+
503, /* Service Unavailable */
|
|
245
|
+
504, /* Gateway Timeout*/
|
|
246
|
+
521, /* Web Server Is Down*/
|
|
247
|
+
522, /* Cloudflare Connection Timed Out */
|
|
248
|
+
524 /* Cloudflare A Timeout Occurred */
|
|
249
|
+
];
|
|
250
|
+
rock.defaults.retryOnError = [
|
|
251
|
+
'ETIMEDOUT', /* One of the timeout limits was reached. */
|
|
252
|
+
'ECONNRESET', /* The connection was forcibly closed. */
|
|
253
|
+
'EADDRINUSE', /* Could not bind to any free port */
|
|
254
|
+
'ECONNREFUSED', /* The connection was refused by the server. */
|
|
255
|
+
'EPIPE', /* The remote side of the stream being written has been closed. */
|
|
256
|
+
'ENOTFOUND', /* Could not resolve the hostname to an IP address. */
|
|
257
|
+
'ENETUNREACH', /* No internet connection. */
|
|
258
|
+
'EAI_AGAIN' /* DNS lookup timed out. */
|
|
259
|
+
];
|
|
260
|
+
|
|
261
|
+
const opts = {
|
|
262
|
+
url : 'http://example.com',
|
|
263
|
+
body : 'this is the POST body',
|
|
264
|
+
maxRetry : 2 // 0 is the default value (= no retries)
|
|
265
|
+
}
|
|
266
|
+
rock(opts, function (err, res, data) {} );
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
### Global options
|
|
271
|
+
|
|
272
|
+
Change default parameters globally (not recommended), or create a new instance with specific paramaters (see below)
|
|
273
|
+
|
|
274
|
+
```js
|
|
275
|
+
rock.defaults = {
|
|
276
|
+
headers : { 'accept-encoding': 'gzip, deflate, br' },
|
|
277
|
+
maxRedirects : 10,
|
|
278
|
+
maxRetry : 0,
|
|
279
|
+
retryDelay : 100, //ms
|
|
280
|
+
retryOnCode : [408, 429, 500, 502, 503, 504, 521, 522, 524 ],
|
|
281
|
+
retryOnError : ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED','EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN' ],
|
|
282
|
+
// beforeRequest is called for each request, retry and redirect
|
|
283
|
+
beforeRequest : (opts) => {
|
|
284
|
+
// There options can be overwritted (= parsed opts.url)
|
|
285
|
+
opts.protocol = 'https:' // or 'http:'
|
|
286
|
+
opts.hostname = 'google.com';
|
|
287
|
+
opts.port = 443;
|
|
288
|
+
opts.path = '/mypage.html?bla=1#hash';
|
|
289
|
+
opts.auth = '';
|
|
290
|
+
opts.headers = {};
|
|
291
|
+
opts.body = {};
|
|
292
|
+
opts.method = 'POST';
|
|
293
|
+
opts.remainingRetry;
|
|
294
|
+
opts.remainingRedirects;
|
|
295
|
+
|
|
296
|
+
// READ-ONLY options (not exhaustive)
|
|
297
|
+
opts.url; // DOT NOT OVERWRITE
|
|
298
|
+
opts.maxRetry;
|
|
299
|
+
opts.maxRedirects;
|
|
300
|
+
opts.prevError; // error of previous request on retry
|
|
301
|
+
opts.prevStatusCode; // HTTP status code of previous request on retry/redirect
|
|
302
|
+
|
|
303
|
+
// opts must be returned
|
|
304
|
+
return opts;
|
|
305
|
+
},
|
|
306
|
+
}
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
### Extend and intercept retries
|
|
310
|
+
|
|
311
|
+
Create a new instance with specific parameter instead of modifying `rock.defaults`
|
|
312
|
+
|
|
313
|
+
By default, this new instance inherits values of the instance source if options are not overwritten.
|
|
314
|
+
Headers are merged. Then only the first level of the options object is merged (no deep travelling in sub-objects or arrays).
|
|
315
|
+
|
|
316
|
+
Here is a basic example of `beforeRequest` interceptor to use [HAProxy as a forward proxy](https://www.haproxy.com/user-spotlight-series/haproxy-as-egress-controller/).
|
|
317
|
+
|
|
318
|
+
`beforeRequest` is always called on each redirect/retry.
|
|
319
|
+
- on redirect, `opts.url` (and `hostname`, `port`, `protocol`, `path`) is updated to the new location
|
|
320
|
+
- on retry, `opts.url` (and `hostname`, `port`, `protocol`, `path`) have the same value as they did
|
|
321
|
+
when the rock-req was initially called.
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
```js
|
|
325
|
+
const myInstance = rock.extend({
|
|
326
|
+
beforeRequest: (opts) => {
|
|
327
|
+
const { hostname, port, protocol, path } = opts;
|
|
328
|
+
opts.protocol = 'http:';
|
|
329
|
+
opts.hostname = '10.0.0.1';
|
|
330
|
+
opts.port = 80;
|
|
331
|
+
opts.path = `${encodeURIComponent(hostname)}/${port}${path}`;
|
|
332
|
+
return opts;
|
|
333
|
+
},
|
|
334
|
+
headers: {
|
|
335
|
+
'Custom-header': 'x-for-proxy'
|
|
336
|
+
},
|
|
337
|
+
timeout : 1000
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
myInstance.get('http://example.com', function (err, res, data) {})
|
|
341
|
+
|
|
342
|
+
```
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
### Timeout
|
|
347
|
+
|
|
348
|
+
You can set a timeout (in milliseconds) on the request with the `timeout` option.
|
|
349
|
+
If the request takes longer than `timeout` to complete, then the entire request
|
|
350
|
+
will fail with an `Error`.
|
|
351
|
+
|
|
352
|
+
```js
|
|
353
|
+
const rock = require('rock-req')
|
|
354
|
+
|
|
355
|
+
const opts = {
|
|
356
|
+
url: 'http://example.com',
|
|
357
|
+
timeout: 2000 // 2 second timeout
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
rock(opts, function (err, res, data) {})
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
### One Quick Tip
|
|
365
|
+
|
|
366
|
+
It's a good idea to set the `'user-agent'` header so the provider can more easily
|
|
367
|
+
see how their resource is used.
|
|
368
|
+
|
|
369
|
+
```js
|
|
370
|
+
const rock = require('rock-req')
|
|
371
|
+
const pkg = require('./package.json')
|
|
372
|
+
|
|
373
|
+
rock({
|
|
374
|
+
url : 'http://example.com',
|
|
375
|
+
headers: {
|
|
376
|
+
'user-agent': `my-module/${pkg.version} (https://github.com/username/my-module)`
|
|
377
|
+
}
|
|
378
|
+
}, function (err, res, data) {})
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
### Proxies
|
|
382
|
+
|
|
383
|
+
You can use the [`tunnel`](https://github.com/koichik/node-tunnel) module with the
|
|
384
|
+
`agent` option to work with proxies:
|
|
385
|
+
|
|
386
|
+
```js
|
|
387
|
+
const rock = require('rock-req')
|
|
388
|
+
const tunnel = require('tunnel')
|
|
389
|
+
|
|
390
|
+
const opts = {
|
|
391
|
+
url: 'http://example.com',
|
|
392
|
+
agent: tunnel.httpOverHttp({
|
|
393
|
+
proxy: {
|
|
394
|
+
host: 'localhost'
|
|
395
|
+
}
|
|
396
|
+
})
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
rock(opts, function (err, res, data) {})
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
### Cookies
|
|
403
|
+
|
|
404
|
+
You can use the [`cookie`](https://github.com/jshttp/cookie) module to include
|
|
405
|
+
cookies in a request:
|
|
406
|
+
|
|
407
|
+
You can extend and create a new instance of rock-req to keep the cookie in header for each request.
|
|
408
|
+
|
|
409
|
+
```js
|
|
410
|
+
const rock = require('rock-req')
|
|
411
|
+
const cookie = require('cookie')
|
|
412
|
+
|
|
413
|
+
const opts = {
|
|
414
|
+
url: 'http://example.com',
|
|
415
|
+
headers: {
|
|
416
|
+
cookie: cookie.serialize('foo', 'bar')
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
rock(opts, function (err, res, data) {})
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
### Form data
|
|
424
|
+
|
|
425
|
+
You can use the [`form-data`](https://github.com/form-data/form-data) module to
|
|
426
|
+
create POST request with form data:
|
|
427
|
+
|
|
428
|
+
```js
|
|
429
|
+
const fs = require('fs')
|
|
430
|
+
const rock = require('rock-req')
|
|
431
|
+
const FormData = require('form-data')
|
|
432
|
+
const form = new FormData()
|
|
433
|
+
|
|
434
|
+
const opts = {
|
|
435
|
+
url: 'http://example.com',
|
|
436
|
+
body: () => {
|
|
437
|
+
form.append('my_file', fs.createReadStream('/foo/bar.jpg'))
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
rock.post(opts, function (err, res, data) {})
|
|
442
|
+
```
|
|
443
|
+
|
|
444
|
+
#### Or, include `application/x-www-form-urlencoded` form data manually:
|
|
445
|
+
|
|
446
|
+
```js
|
|
447
|
+
const rock = require('rock-req')
|
|
448
|
+
|
|
449
|
+
const opts = {
|
|
450
|
+
url: 'http://example.com',
|
|
451
|
+
form: {
|
|
452
|
+
key: 'value'
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
rock.post(opts, function (err, res, data) {})
|
|
456
|
+
```
|
|
457
|
+
|
|
458
|
+
### Specifically disallowing redirects
|
|
459
|
+
|
|
460
|
+
```js
|
|
461
|
+
const rock = require('rock-req')
|
|
462
|
+
|
|
463
|
+
const opts = {
|
|
464
|
+
url: 'http://example.com/will-redirect-elsewhere',
|
|
465
|
+
followRedirects: false
|
|
466
|
+
}
|
|
467
|
+
// res.statusCode will be 301, no error thrown
|
|
468
|
+
rock(opts, function (err, res, data) {})
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
### Basic Auth
|
|
472
|
+
|
|
473
|
+
```js
|
|
474
|
+
const user = 'someuser'
|
|
475
|
+
const pass = 'pa$$word'
|
|
476
|
+
const encodedAuth = Buffer.from(`${user}:${pass}`).toString('base64')
|
|
477
|
+
|
|
478
|
+
rock('http://example.com', {
|
|
479
|
+
headers: {
|
|
480
|
+
authorization: `Basic ${encodedAuth}`
|
|
481
|
+
}
|
|
482
|
+
})
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
### OAuth
|
|
486
|
+
|
|
487
|
+
You can use the [`oauth-1.0a`](https://github.com/ddo/oauth-1.0a) module to create
|
|
488
|
+
a signed OAuth request:
|
|
489
|
+
|
|
490
|
+
```js
|
|
491
|
+
const rock = require('rock-req')
|
|
492
|
+
const crypto = require('crypto')
|
|
493
|
+
const OAuth = require('oauth-1.0a')
|
|
494
|
+
|
|
495
|
+
const oauth = OAuth({
|
|
496
|
+
consumer: {
|
|
497
|
+
key: process.env.CONSUMER_KEY,
|
|
498
|
+
secret: process.env.CONSUMER_SECRET
|
|
499
|
+
},
|
|
500
|
+
signature_method: 'HMAC-SHA1',
|
|
501
|
+
hash_function: (baseString, key) => crypto.createHmac('sha1', key).update(baseString).digest('base64')
|
|
502
|
+
})
|
|
503
|
+
|
|
504
|
+
const token = {
|
|
505
|
+
key: process.env.ACCESS_TOKEN,
|
|
506
|
+
secret: process.env.ACCESS_TOKEN_SECRET
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
const url = 'https://api.twitter.com/1.1/statuses/home_timeline.json'
|
|
510
|
+
|
|
511
|
+
const opts = {
|
|
512
|
+
url: url,
|
|
513
|
+
headers: oauth.toHeader(oauth.authorize({url, method: 'GET'}, token)),
|
|
514
|
+
json: true
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
rock(opts, function (err, res) {})
|
|
518
|
+
```
|
|
519
|
+
|
|
520
|
+
### Throttle requests
|
|
521
|
+
|
|
522
|
+
You can use [limiter](https://github.com/jhurliman/node-rate-limiter) to throttle requests. This is useful when calling an API that is rate limited.
|
|
523
|
+
|
|
524
|
+
```js
|
|
525
|
+
const rock = require('rock-get')
|
|
526
|
+
const RateLimiter = require('limiter').RateLimiter
|
|
527
|
+
const limiter = new RateLimiter(1, 'second')
|
|
528
|
+
|
|
529
|
+
const rock = (opts, cb) => limiter.removeTokens(1, () => rock(opts, cb))
|
|
530
|
+
rock.concat = (opts, cb) => limiter.removeTokens(1, () => rock.concat(opts, cb))
|
|
531
|
+
|
|
532
|
+
var opts = {
|
|
533
|
+
url: 'http://example.com'
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
rock.concat(opts, processResult)
|
|
537
|
+
rock.concat(opts, processResult)
|
|
538
|
+
|
|
539
|
+
function processResult (err, res, data) {
|
|
540
|
+
if (err) throw err
|
|
541
|
+
console.log(data.toString())
|
|
542
|
+
}
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
## Breaking change for people coming from simplet-get
|
|
547
|
+
|
|
548
|
+
Rock-req is a fork of [simple-get](https://github.com/feross/simple-get)
|
|
549
|
+
|
|
550
|
+
- Rock-req concatenates chunks and returns the concatenated buffer/JSON in the third paramater of the callback.
|
|
551
|
+
Why? because 80% of the time, we need to do simple request without streams.
|
|
552
|
+
When you need to pipe the result to another stream, you must use `opts.output` parameter (see "Output Stream" in the doc)
|
|
553
|
+
- All streams must be created in a function. Rock-req return an error if it is not the case
|
|
554
|
+
- before: `body = stream`
|
|
555
|
+
- after `body = () => { const myStream = create(); return myStream; }`
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
## Notes:
|
|
559
|
+
|
|
560
|
+
- [ ] replace deprecated `url.parse` by `new URL` but new URL is slower than url.parse. Let's see if Node 20 LTS is faster
|
|
561
|
+
- [ ] agent keep Alive
|
|
562
|
+
- [ ] add advanced timeout (response timeout)
|
|
563
|
+
- [ ] test prevError
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
# Supporters
|
|
567
|
+
|
|
568
|
+
This packaged in maintained by Carbone:
|
|
569
|
+
|
|
570
|
+
<p>
|
|
571
|
+
<a href="https://carbone.io" alt="Carbone.io - Efficient PDF / DOCX / XLSX / CSV / HTML / XML generator with templates and JSON">
|
|
572
|
+
<img src="https://raw.githubusercontent.com/carboneio/rock-req/master/doc/carbone-logo.svg" alt="Carbone.io logo" height="60"/>
|
|
573
|
+
</a>
|
|
574
|
+
</p>
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
Thank you [Feross Aboukhadijeh](https://github.com/feross), creator of `simple-get`
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
|
package/index.js
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
module.exports = extend()
|
|
2
|
+
|
|
3
|
+
const http = require('http')
|
|
4
|
+
const https = require('https')
|
|
5
|
+
const querystring = require('querystring')
|
|
6
|
+
const url = require('url')
|
|
7
|
+
const zlib = require('zlib')
|
|
8
|
+
const { pipeline, Writable, Readable } = require('stream')
|
|
9
|
+
|
|
10
|
+
const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === 'function'
|
|
11
|
+
const isFnStream = o => o instanceof Function
|
|
12
|
+
function applyDefault (t, d) { for (const a in d) { if (t[a] === undefined) t[a] = d[a] } return t }
|
|
13
|
+
function cloneLowerCase (s) { const n = {}; for (const a in s) { n[a.toLowerCase()] = s[a] } return n }
|
|
14
|
+
|
|
15
|
+
function extend (defaultOptions = {}) {
|
|
16
|
+
let _default = {
|
|
17
|
+
headers: { 'accept-encoding': 'gzip, deflate, br' },
|
|
18
|
+
maxRedirects: 10,
|
|
19
|
+
maxRetry: 0,
|
|
20
|
+
retryDelay: 100, // ms
|
|
21
|
+
retryOnCode: [408, 429, 500, 502, 503, 504, 521, 522, 524],
|
|
22
|
+
retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'],
|
|
23
|
+
beforeRequest: o => o
|
|
24
|
+
}
|
|
25
|
+
defaultOptions.headers = applyDefault(cloneLowerCase(defaultOptions.headers), _default.headers)
|
|
26
|
+
_default = applyDefault(defaultOptions, _default) // inherits of parent options
|
|
27
|
+
|
|
28
|
+
// all options https://nodejs.org/dist/latest-v18.x/docs/api/http.html#http_http_request_url_options_callback
|
|
29
|
+
function rock (opts, directBody, cb) {
|
|
30
|
+
if (typeof opts === 'string') opts = { url: opts }
|
|
31
|
+
if (!cb) { cb = directBody } else { opts.body = directBody }
|
|
32
|
+
opts.headers = applyDefault(cloneLowerCase(opts.headers), _default.headers)
|
|
33
|
+
opts = applyDefault(opts, _default)
|
|
34
|
+
opts.remainingRetry = opts.remainingRetry ?? opts.maxRetry
|
|
35
|
+
opts.remainingRedirects = opts.remainingRedirects ?? opts.maxRedirects
|
|
36
|
+
|
|
37
|
+
if (opts.url) {
|
|
38
|
+
const { hostname, port, protocol, auth, path } = url.parse(opts.url) // eslint-disable-line node/no-deprecated-api
|
|
39
|
+
if (!hostname && !port && !protocol && !auth) opts.path = path // Relative redirect
|
|
40
|
+
else Object.assign(opts, { hostname, port, protocol, auth, path }) // Absolute redirect
|
|
41
|
+
}
|
|
42
|
+
opts = opts.beforeRequest(opts)
|
|
43
|
+
|
|
44
|
+
let body
|
|
45
|
+
if (opts.body) {
|
|
46
|
+
body = opts.json && !isFnStream(opts.body) && !isStream(opts.body) ? JSON.stringify(opts.body) : opts.body
|
|
47
|
+
} else if (opts.form) {
|
|
48
|
+
body = typeof opts.form === 'string' ? opts.form : querystring.stringify(opts.form)
|
|
49
|
+
opts.headers['content-type'] = 'application/x-www-form-urlencoded'
|
|
50
|
+
}
|
|
51
|
+
if (body) {
|
|
52
|
+
if (isStream(body)) return cb(new Error('opts.body must be a function returning a Readable stream. RTFM'))
|
|
53
|
+
if (!opts.method) opts.method = 'POST'
|
|
54
|
+
if (!isFnStream(body)) opts.headers['content-length'] = Buffer.byteLength(body)
|
|
55
|
+
if (opts.json && !opts.form) opts.headers['content-type'] = 'application/json'
|
|
56
|
+
}
|
|
57
|
+
if (opts.output && (isStream(opts.output) || !isFnStream(opts.output))) return cb(new Error('opts.output must be a function returning a Writable stream. RTFM'))
|
|
58
|
+
if (opts.json) opts.headers.accept = 'application/json'
|
|
59
|
+
if (opts.method) opts.method = opts.method.toUpperCase()
|
|
60
|
+
|
|
61
|
+
const originalHost = opts.hostname // hostname before potential redirect
|
|
62
|
+
const protocol = opts.protocol === 'https:' ? https : http // Support http/https urls
|
|
63
|
+
const chunks = []
|
|
64
|
+
let requestAbortedOrEnded = false
|
|
65
|
+
let response = null
|
|
66
|
+
function onRequestEnd (err) {
|
|
67
|
+
if (requestAbortedOrEnded === true) return
|
|
68
|
+
requestAbortedOrEnded = true
|
|
69
|
+
if (opts.retryOnError.indexOf(err?.code) !== -1 && --opts.remainingRetry > 0) {
|
|
70
|
+
opts.prevError = err
|
|
71
|
+
return setTimeout(rock, opts.retryDelay, opts, cb) // retry in 100ms
|
|
72
|
+
}
|
|
73
|
+
if (err) return cb(err)
|
|
74
|
+
let data = Buffer.concat(chunks)
|
|
75
|
+
if (opts.json) {
|
|
76
|
+
try { data = JSON.parse(data.toString()) } catch (e) { return cb(e, response, data) }
|
|
77
|
+
}
|
|
78
|
+
cb(null, response, data)
|
|
79
|
+
}
|
|
80
|
+
const req = protocol.request(opts, res => {
|
|
81
|
+
opts.prevStatusCode = res.statusCode
|
|
82
|
+
// retry and leave
|
|
83
|
+
if (res.statusCode > 400 /* speed up */ && opts.retryOnCode.indexOf(res.statusCode) !== -1 && opts.remainingRetry-- > 0) {
|
|
84
|
+
requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
|
|
85
|
+
res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
|
|
86
|
+
return setTimeout(rock, opts.retryDelay, opts, cb) // retry later
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// or redirect and leave
|
|
90
|
+
if (opts.followRedirects !== false && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
91
|
+
requestAbortedOrEnded = true // discard all new events which could come after for this request to avoid calling the callback
|
|
92
|
+
res.resume() // Discard response, consume data until the end to free up memory. Mandatory!
|
|
93
|
+
opts.url = res.headers.location // Follow 3xx redirects
|
|
94
|
+
delete opts.headers.host // Discard `host` header on redirect (see #32)
|
|
95
|
+
const redirectHost = url.parse(opts.url).hostname // eslint-disable-line node/no-deprecated-api
|
|
96
|
+
// If redirected host is different than original host, drop headers to prevent cookie leak (#73)
|
|
97
|
+
if (redirectHost !== null && redirectHost !== originalHost) {
|
|
98
|
+
delete opts.headers.cookie
|
|
99
|
+
delete opts.headers.authorization
|
|
100
|
+
}
|
|
101
|
+
if (opts.method === 'POST' && [301, 302].includes(res.statusCode)) {
|
|
102
|
+
opts.method = 'GET' // On 301/302 redirect, change POST to GET (see #35)
|
|
103
|
+
delete opts.headers['content-length']; delete opts.headers['content-type']; delete opts.body; delete opts.form
|
|
104
|
+
}
|
|
105
|
+
if (opts.remainingRedirects-- === 0) {
|
|
106
|
+
requestAbortedOrEnded = false
|
|
107
|
+
return onRequestEnd(new Error('too many redirects'))
|
|
108
|
+
}
|
|
109
|
+
return rock(opts, cb)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// or read response and leave at the end
|
|
113
|
+
response = res
|
|
114
|
+
const contentEncoding = opts.method !== 'HEAD' ? (res.headers['content-encoding'] || '').toLowerCase() : ''
|
|
115
|
+
const output = opts.output ? opts.output(opts, res) : new Writable({ write (chunk, enc, wcb) { chunks.push(chunk); wcb() } })
|
|
116
|
+
switch (contentEncoding) {
|
|
117
|
+
case 'br':
|
|
118
|
+
pipeline(res, zlib.createBrotliDecompress(), output, onRequestEnd); break
|
|
119
|
+
case 'gzip':
|
|
120
|
+
case 'deflate':
|
|
121
|
+
pipeline(res, zlib.createUnzip(), output, onRequestEnd); break
|
|
122
|
+
default:
|
|
123
|
+
pipeline(res, output, onRequestEnd); break
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
req.once('timeout', () => {
|
|
127
|
+
const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT'
|
|
128
|
+
req.destroy() // we must destroy manually
|
|
129
|
+
onRequestEnd(_error) // This timeout event can come after the input pipeline is finished (ex. timeout with no body)
|
|
130
|
+
})
|
|
131
|
+
|
|
132
|
+
const _inputStream = isFnStream(body) ? body(opts) : Readable.from([body], { objectMode: false })
|
|
133
|
+
pipeline(_inputStream, req, (e) => {
|
|
134
|
+
if (e) onRequestEnd(e)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
return req
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
;['get', 'post', 'put', 'patch', 'head', 'delete', 'getJSON', 'postJSON', 'putJSON', 'patchJSON', 'headJSON', 'deleteJSON'].forEach(method => {
|
|
141
|
+
const jsonShortcut = /JSON$/.test(method) === true
|
|
142
|
+
const methodShortcut = jsonShortcut === true ? method.toUpperCase().slice(0, -4) : method.toUpperCase()
|
|
143
|
+
rock[method] = (opts, body, cb) => {
|
|
144
|
+
if (typeof opts === 'string') opts = { url: opts }
|
|
145
|
+
opts.method = methodShortcut
|
|
146
|
+
opts.json = jsonShortcut
|
|
147
|
+
return rock(opts, body, cb)
|
|
148
|
+
}
|
|
149
|
+
})
|
|
150
|
+
rock.concat = rock
|
|
151
|
+
rock.defaults = _default
|
|
152
|
+
rock.extend = extend
|
|
153
|
+
|
|
154
|
+
return rock
|
|
155
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "rock-req",
|
|
3
|
+
"description": "Zero dependencies (160 LOC) & rock-solid request library: http/https, reliable retry on failure, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout",
|
|
4
|
+
"version": "5.0.0",
|
|
5
|
+
"author": {
|
|
6
|
+
"name": "David Grelaud & Feross Aboukhadijeh"
|
|
7
|
+
},
|
|
8
|
+
"devDependencies": {
|
|
9
|
+
"self-signed-https": "1.0.5",
|
|
10
|
+
"standard": "*",
|
|
11
|
+
"string-to-stream": "3.0.1",
|
|
12
|
+
"tape": "5.6.3"
|
|
13
|
+
},
|
|
14
|
+
"engines": {
|
|
15
|
+
"node": ">=16"
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"index.js"
|
|
19
|
+
],
|
|
20
|
+
"keywords": [
|
|
21
|
+
"request",
|
|
22
|
+
"http",
|
|
23
|
+
"get request",
|
|
24
|
+
"http.get",
|
|
25
|
+
"redirects",
|
|
26
|
+
"retry",
|
|
27
|
+
"follow redirects",
|
|
28
|
+
"gzip",
|
|
29
|
+
"deflate",
|
|
30
|
+
"https",
|
|
31
|
+
"stream",
|
|
32
|
+
"simple request",
|
|
33
|
+
"light request",
|
|
34
|
+
"kiss request",
|
|
35
|
+
"tiny request",
|
|
36
|
+
"simple get"
|
|
37
|
+
],
|
|
38
|
+
"license": "MIT",
|
|
39
|
+
"main": "index.js",
|
|
40
|
+
"repository": {
|
|
41
|
+
"type": "git",
|
|
42
|
+
"url": "git://github.com/carboneio/rock-req.git"
|
|
43
|
+
},
|
|
44
|
+
"scripts": {
|
|
45
|
+
"test": "tape test/*.js"
|
|
46
|
+
}
|
|
47
|
+
}
|