urllib 2.38.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -264
- package/package.json +62 -59
- package/src/HttpAgent.ts +72 -0
- package/src/HttpClient.ts +514 -0
- package/src/Request.ts +118 -0
- package/src/Response.ts +41 -0
- package/src/cjs/HttpAgent.d.ts +16 -0
- package/src/cjs/HttpAgent.js +62 -0
- package/src/cjs/HttpAgent.js.map +1 -0
- package/src/cjs/HttpClient.d.ts +39 -0
- package/src/cjs/HttpClient.js +466 -0
- package/src/cjs/HttpClient.js.map +1 -0
- package/src/cjs/Request.d.ts +114 -0
- package/src/cjs/Request.js +3 -0
- package/src/cjs/Request.js.map +1 -0
- package/src/cjs/Response.d.ts +36 -0
- package/src/cjs/Response.js +3 -0
- package/src/cjs/Response.js.map +1 -0
- package/src/cjs/index.d.ts +7 -0
- package/src/cjs/index.js +18 -0
- package/src/cjs/index.js.map +1 -0
- package/src/cjs/package.json +3 -0
- package/src/cjs/utils.d.ts +3 -0
- package/src/cjs/utils.js +56 -0
- package/src/cjs/utils.js.map +1 -0
- package/src/esm/HttpAgent.d.ts +16 -0
- package/src/esm/HttpAgent.js +58 -0
- package/src/esm/HttpAgent.js.map +1 -0
- package/src/esm/HttpClient.d.ts +39 -0
- package/src/esm/HttpClient.js +462 -0
- package/src/esm/HttpClient.js.map +1 -0
- package/src/esm/Request.d.ts +114 -0
- package/src/esm/Request.js +2 -0
- package/src/esm/Request.js.map +1 -0
- package/src/esm/Response.d.ts +36 -0
- package/src/esm/Response.js +2 -0
- package/src/esm/Response.js.map +1 -0
- package/src/esm/index.d.ts +7 -0
- package/src/esm/index.js +13 -0
- package/src/esm/index.js.map +1 -0
- package/src/esm/package.json +3 -0
- package/src/esm/utils.d.ts +3 -0
- package/src/esm/utils.js +51 -0
- package/src/esm/utils.js.map +1 -0
- package/src/index.ts +16 -0
- package/src/utils.ts +53 -0
- package/History.md +0 -804
- package/lib/detect_proxy_agent.js +0 -31
- package/lib/get_proxy_from_uri.js +0 -81
- package/lib/httpclient.js +0 -61
- package/lib/httpclient2.js +0 -83
- package/lib/index.d.ts +0 -279
- package/lib/index.js +0 -21
- package/lib/index.test-d.ts +0 -19
- package/lib/urllib.js +0 -1317
package/README.md
CHANGED
@@ -21,63 +21,29 @@ and digest authentication, redirections, cookies, timeout and more.
|
|
21
21
|
## Install
|
22
22
|
|
23
23
|
```bash
|
24
|
-
|
24
|
+
npm install urllib --save
|
25
25
|
```
|
26
26
|
|
27
27
|
## Usage
|
28
28
|
|
29
|
-
###
|
29
|
+
### TypeScript and ESM
|
30
30
|
|
31
31
|
```js
|
32
|
-
|
32
|
+
import { request } from 'urllib';
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
}
|
38
|
-
console.log(res.statusCode);
|
39
|
-
console.log(res.headers);
|
40
|
-
// data is Buffer instance
|
41
|
-
console.log(data.toString());
|
42
|
-
});
|
34
|
+
const { data, res } = await request('http://cnodejs.org/');
|
35
|
+
// result: { data: Buffer, res: Response }
|
36
|
+
console.log('status: %s, body size: %d, headers: %j', res.statusCode, data.length, res.headers);
|
43
37
|
```
|
44
38
|
|
45
|
-
###
|
46
|
-
|
47
|
-
If you've installed [bluebird][bluebird],
|
48
|
-
[bluebird][bluebird] will be used.
|
49
|
-
`urllib` does not install [bluebird][bluebird] for you.
|
50
|
-
|
51
|
-
Otherwise, if you're using a node that has native v8 Promises (v0.11.13+),
|
52
|
-
then that will be used.
|
53
|
-
|
54
|
-
Otherwise, this library will crash the process and exit,
|
55
|
-
so you might as well install [bluebird][bluebird] as a dependency!
|
39
|
+
### CommonJS
|
56
40
|
|
57
41
|
```js
|
58
|
-
|
59
|
-
|
60
|
-
urllib.request('http://nodejs.org').then(function (result) {
|
61
|
-
// result: {data: buffer, res: response object}
|
62
|
-
console.log('status: %s, body size: %d, headers: %j', result.res.statusCode, result.data.length, result.res.headers);
|
63
|
-
}).catch(function (err) {
|
64
|
-
console.error(err);
|
65
|
-
});
|
66
|
-
```
|
67
|
-
|
68
|
-
### co & generator
|
42
|
+
const { request } = require('urllib');
|
69
43
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
var co = require('co');
|
74
|
-
var urllib = require('urllib');
|
75
|
-
|
76
|
-
co(function* () {
|
77
|
-
var result = yield urllib.requestThunk('http://nodejs.org');
|
78
|
-
console.log('status: %s, body size: %d, headers: %j',
|
79
|
-
result.status, result.data.length, result.headers);
|
80
|
-
})();
|
44
|
+
const { data, res } = await request('http://cnodejs.org/');
|
45
|
+
// result: { data: Buffer, res: Response }
|
46
|
+
console.log('status: %s, body size: %d, headers: %j', res.statusCode, data.length, res.headers);
|
81
47
|
```
|
82
48
|
|
83
49
|
## Global `response` event
|
@@ -85,9 +51,10 @@ co(function* () {
|
|
85
51
|
You should create a urllib instance first.
|
86
52
|
|
87
53
|
```js
|
88
|
-
|
54
|
+
import { HttpClient } from 'urllib';
|
89
55
|
|
90
|
-
httpclient
|
56
|
+
const httpclient = new HttpClient();
|
57
|
+
httpclient.on('response', (info) => {
|
91
58
|
error: err,
|
92
59
|
ctx: args.ctx,
|
93
60
|
req: {
|
@@ -98,89 +65,52 @@ httpclient.on('response', function (info) {
|
|
98
65
|
res: res
|
99
66
|
});
|
100
67
|
|
101
|
-
httpclient.request('
|
102
|
-
|
103
|
-
});
|
68
|
+
const { data, res } = await httpclient.request('https://nodejs.org');
|
69
|
+
console.log('status: %s, body size: %d, headers: %j', res.statusCode, data.length, res.headers);
|
104
70
|
```
|
105
71
|
|
106
72
|
## API Doc
|
107
73
|
|
108
|
-
### Method: `
|
74
|
+
### Method: `async request(url[, options])`
|
109
75
|
|
110
76
|
#### Arguments
|
111
77
|
|
112
78
|
- **url** String | Object - The URL to request, either a String or a Object that return by [url.parse](http://nodejs.org/api/url.html#url_url_parse_urlstr_parsequerystring_slashesdenotehost).
|
113
79
|
- ***options*** Object - Optional
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
- ***keepHeaderCase*** Boolean - by default will convert header keys to lowercase
|
128
|
-
- ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Defaults to `exports.TIMEOUT`, both are 5s. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
|
129
|
-
- ***auth*** String - `username:password` used in HTTP Basic Authorization.
|
130
|
-
- ***digestAuth*** String - `username:password` used in HTTP [Digest Authorization](http://en.wikipedia.org/wiki/Digest_access_authentication).
|
131
|
-
- ***agent*** [http.Agent](http://nodejs.org/api/http.html#http_class_http_agent) - HTTP Agent object.
|
80
|
+
- ***method*** String - Request method, defaults to `GET`. Could be `GET`, `POST`, `DELETE` or `PUT`. Alias 'type'.
|
81
|
+
- ***data*** Object - Data to be sent. Will be stringify automatically.
|
82
|
+
- ***content*** String | [Buffer](http://nodejs.org/api/buffer.html) - Manually set the content of payload. If set, `data` will be ignored.
|
83
|
+
- ***stream*** [stream.Readable](http://nodejs.org/api/stream.html#stream_class_stream_readable) - Stream to be pipe to the remote. If set, `data` and `content` will be ignored.
|
84
|
+
- ***writeStream*** [stream.Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
|
85
|
+
- ***files*** {Array<ReadStream|Buffer|String> | Object | ReadStream | Buffer | String - The files will send with `multipart/form-data` format, base on `formstream`. If `method` not set, will use `POST` method by default.
|
86
|
+
- ***contentType*** String - Type of request data. Could be `json` (**Notes**: not use `application/json` here). If it's `json`, will auto set `Content-Type: application/json` header.
|
87
|
+
- ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
|
88
|
+
- **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.
|
89
|
+
- ***headers*** Object - Request headers.
|
90
|
+
- ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Defaults to `exports.TIMEOUT`, both are 5s. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
|
91
|
+
- ***auth*** String - `username:password` used in HTTP Basic Authorization.
|
92
|
+
- ***agent*** [http.Agent](http://nodejs.org/api/http.html#http_class_http_agent) - HTTP Agent object.
|
132
93
|
Set `false` if you does not use agent.
|
133
|
-
|
94
|
+
- ***httpsAgent*** [https.Agent](http://nodejs.org/api/https.html#https_class_https_agent) - HTTPS Agent object.
|
134
95
|
Set `false` if you does not use agent.
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
certificate and CA certs of the server in PFX or PKCS12 format.
|
143
|
-
- ***key*** String | Buffer - A string or Buffer containing the private key of the client in PEM format.
|
144
|
-
**Notes**: This is necessary only if using the client certificate authentication
|
145
|
-
- ***cert*** String | Buffer - A string or Buffer containing the certificate key of the client in PEM format.
|
146
|
-
**Notes**: This is necessary only if using the client certificate authentication
|
147
|
-
- ***passphrase*** String - A string of passphrase for the private key or pfx.
|
148
|
-
- ***ciphers*** String - A string describing the ciphers to use or exclude.
|
149
|
-
- ***secureProtocol*** String - The SSL method to use, e.g. SSLv3_method to force SSL version 3.
|
150
|
-
- ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
|
151
|
-
- ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.
|
152
|
-
- ***formatRedirectUrl*** Function - Format the redirect url by your self. Default is `url.resolve(from, to)`.
|
153
|
-
- ***beforeRequest*** Function - Before request hook, you can change every thing here.
|
154
|
-
- ***streaming*** Boolean - let you get the `res` object when request connected, default `false`. alias `customResponse`
|
155
|
-
- ***gzip*** Boolean - Accept gzip response content and auto decode it, default is `false`.
|
156
|
-
- ***timing*** Boolean - Enable timing or not, default is `false`.
|
157
|
-
- ***enableProxy*** Boolean - Enable proxy request, default is `false`.
|
158
|
-
- ***proxy*** String | Object - proxy agent uri or options, default is `null`.
|
159
|
-
- ***lookup*** Function - Custom DNS lookup function, default is `dns.lookup`. Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
|
160
|
-
- ***checkAddress*** Function: optional, check request address to protect from SSRF and similar attacks. It receive tow arguments(`ip` and `family`) and should return true or false to identified the address is legal or not. It rely on `lookup` and have the same version requirement.
|
161
|
-
- ***trace*** Boolean - Enable capture stack include call site of library entrance, default is `false`.
|
162
|
-
- ***socketPath*** String - optional Unix Domain Socket. (Refer to [Node.js Document](https://nodejs.org/dist/latest-v14.x/docs/api/http.html#http_http_request_options_callback))
|
163
|
-
- ***callback(err, data, res)*** Function - Optional callback.
|
164
|
-
- **err** Error - Would be `null` if no error accured.
|
165
|
-
- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
|
166
|
-
- **res** [http.IncomingMessage](http://nodejs.org/api/http.html#http_http_incomingmessage) - The response.
|
167
|
-
|
168
|
-
#### Returns
|
169
|
-
|
170
|
-
[http.ClientRequest](http://nodejs.org/api/http.html#http_class_http_clientrequest) - The request.
|
171
|
-
|
172
|
-
Calling `.abort()` method of the request stream can cancel the request.
|
96
|
+
- ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
|
97
|
+
- ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.
|
98
|
+
- ***formatRedirectUrl*** Function - Format the redirect url by your self. Default is `url.resolve(from, to)`.
|
99
|
+
- ***beforeRequest*** Function - Before request hook, you can change every thing here.
|
100
|
+
- ***streaming*** Boolean - let you get the `res` object when request connected, default `false`. alias `customResponse`
|
101
|
+
- ***gzip*** Boolean - Accept `gzip, br` response content and auto decode it, default is `false`.
|
102
|
+
- ***timing*** Boolean - Enable timing or not, default is `false`.
|
173
103
|
|
174
104
|
#### Options: `options.data`
|
175
105
|
|
176
106
|
When making a request:
|
177
107
|
|
178
108
|
```js
|
179
|
-
|
109
|
+
await request('https://example.com', {
|
180
110
|
method: 'GET',
|
181
111
|
data: {
|
182
112
|
'a': 'hello',
|
183
|
-
'b': 'world'
|
113
|
+
'b': 'world',
|
184
114
|
}
|
185
115
|
});
|
186
116
|
```
|
@@ -189,26 +119,26 @@ For `GET` request, `data` will be stringify to query string, e.g. `http://exampl
|
|
189
119
|
|
190
120
|
For others like `POST`, `PATCH` or `PUT` request,
|
191
121
|
in defaults, the `data` will be stringify into `application/x-www-form-urlencoded` format
|
192
|
-
if `
|
122
|
+
if `content-type` header is not set.
|
193
123
|
|
194
|
-
If `
|
124
|
+
If `content-type` is `application/json`, the `data` will be `JSON.stringify` to JSON data format.
|
195
125
|
|
196
126
|
#### Options: `options.content`
|
197
127
|
|
198
128
|
`options.content` is useful when you wish to construct the request body by yourself,
|
199
|
-
for example making a `
|
129
|
+
for example making a `content-type: application/json` request.
|
200
130
|
|
201
131
|
Notes that if you want to send a JSON body, you should stringify it yourself:
|
202
132
|
|
203
133
|
```js
|
204
|
-
|
134
|
+
await request('https://example.com', {
|
205
135
|
method: 'POST',
|
206
136
|
headers: {
|
207
137
|
'Content-Type': 'application/json'
|
208
138
|
},
|
209
139
|
content: JSON.stringify({
|
210
140
|
a: 'hello',
|
211
|
-
b: 'world'
|
141
|
+
b: 'world',
|
212
142
|
})
|
213
143
|
});
|
214
144
|
```
|
@@ -217,8 +147,8 @@ It would make a HTTP request like:
|
|
217
147
|
|
218
148
|
```http
|
219
149
|
POST / HTTP/1.1
|
220
|
-
|
221
|
-
|
150
|
+
host: example.com
|
151
|
+
content-type: application/json
|
222
152
|
|
223
153
|
{
|
224
154
|
"a": "hello",
|
@@ -229,14 +159,14 @@ Content-Type: application/json
|
|
229
159
|
This exmaple can use `options.data` with `application/json` content type:
|
230
160
|
|
231
161
|
```js
|
232
|
-
|
162
|
+
await request('https://example.com', {
|
233
163
|
method: 'POST',
|
234
164
|
headers: {
|
235
|
-
'
|
165
|
+
'content-type': 'application/json'
|
236
166
|
},
|
237
167
|
data: {
|
238
168
|
a: 'hello',
|
239
|
-
b: 'world'
|
169
|
+
b: 'world',
|
240
170
|
}
|
241
171
|
});
|
242
172
|
```
|
@@ -246,24 +176,20 @@ urllib.request('http://example.com', {
|
|
246
176
|
Upload a file with a `hello` field.
|
247
177
|
|
248
178
|
```js
|
249
|
-
|
250
|
-
|
251
|
-
var req = urllib.request('http://my.server.com/upload', {
|
179
|
+
await request('https://example.com/upload', {
|
180
|
+
method: 'POST',
|
252
181
|
files: __filename,
|
253
182
|
data: {
|
254
183
|
hello: 'hello urllib',
|
255
184
|
},
|
256
|
-
}, function (err, data, res) {
|
257
|
-
// upload finished
|
258
185
|
});
|
259
186
|
```
|
260
187
|
|
261
188
|
Upload multi files with a `hello` field.
|
262
189
|
|
263
190
|
```js
|
264
|
-
|
265
|
-
|
266
|
-
var req = urllib.request('http://my.server.com/upload', {
|
191
|
+
await request('https://example.com/upload', {
|
192
|
+
method: 'POST',
|
267
193
|
files: [
|
268
194
|
__filename,
|
269
195
|
fs.createReadStream(__filename),
|
@@ -272,22 +198,17 @@ var req = urllib.request('http://my.server.com/upload', {
|
|
272
198
|
data: {
|
273
199
|
hello: 'hello urllib with multi files',
|
274
200
|
},
|
275
|
-
}, function (err, data, res) {
|
276
|
-
// upload finished
|
277
201
|
});
|
278
202
|
```
|
279
203
|
|
280
204
|
Custom file field name with `uploadfile`.
|
281
205
|
|
282
206
|
```js
|
283
|
-
|
284
|
-
|
285
|
-
var req = urllib.request('http://my.server.com/upload', {
|
207
|
+
await request('https://example.com/upload', {
|
208
|
+
method: 'POST',
|
286
209
|
files: {
|
287
210
|
uploadfile: __filename,
|
288
211
|
},
|
289
|
-
}, function (err, data, res) {
|
290
|
-
// upload finished
|
291
212
|
});
|
292
213
|
```
|
293
214
|
|
@@ -296,19 +217,16 @@ var req = urllib.request('http://my.server.com/upload', {
|
|
296
217
|
Uploads a file with [formstream](https://github.com/node-modules/formstream):
|
297
218
|
|
298
219
|
```js
|
299
|
-
|
300
|
-
var formstream = require('formstream');
|
220
|
+
import formstream from 'formstream';
|
301
221
|
|
302
|
-
|
222
|
+
const form = formstream();
|
303
223
|
form.file('file', __filename);
|
304
224
|
form.field('hello', '你好urllib');
|
305
225
|
|
306
|
-
|
226
|
+
await request('https://example.com/upload', {
|
307
227
|
method: 'POST',
|
308
228
|
headers: form.headers(),
|
309
|
-
stream: form
|
310
|
-
}, function (err, data, res) {
|
311
|
-
// upload finished
|
229
|
+
stream: form,
|
312
230
|
});
|
313
231
|
```
|
314
232
|
|
@@ -316,129 +234,25 @@ var req = urllib.request('http://my.server.com/upload', {
|
|
316
234
|
|
317
235
|
Response is normal object, it contains:
|
318
236
|
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
#### Response: `res.aborted`
|
334
|
-
|
335
|
-
If the underlaying connection was terminated before `response.end()` was called,
|
336
|
-
`res.aborted` should be `true`.
|
337
|
-
|
338
|
-
```js
|
339
|
-
require('http').createServer(function (req, res) {
|
340
|
-
req.resume();
|
341
|
-
req.on('end', function () {
|
342
|
-
res.write('foo haha\n');
|
343
|
-
setTimeout(function () {
|
344
|
-
res.write('foo haha 2');
|
345
|
-
setTimeout(function () {
|
346
|
-
res.socket.end();
|
347
|
-
}, 300);
|
348
|
-
}, 200);
|
349
|
-
return;
|
350
|
-
});
|
351
|
-
}).listen(1984);
|
352
|
-
|
353
|
-
urllib.request('http://127.0.0.1:1984/socket.end', function (err, data, res) {
|
354
|
-
data.toString().should.equal('foo haha\nfoo haha 2');
|
355
|
-
should.ok(res.aborted);
|
356
|
-
done();
|
357
|
-
});
|
358
|
-
```
|
359
|
-
|
360
|
-
### HttpClient2
|
361
|
-
|
362
|
-
HttpClient2 is a new instance for future. request method only return a promise, compatible with `async/await` and generator in co.
|
363
|
-
|
364
|
-
#### Options
|
365
|
-
|
366
|
-
options extends from urllib, besides below
|
367
|
-
|
368
|
-
- ***retry*** Number - a retry count, when get an error, it will request again until reach the retry count.
|
369
|
-
- ***retryDelay*** Number - wait a delay(ms) between retries.
|
370
|
-
- ***isRetry*** Function - determine whether retry, a response object as the first argument. it will retry when status >= 500 by default. Request error is not included.
|
371
|
-
|
372
|
-
#### Warning
|
373
|
-
|
374
|
-
It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
|
375
|
-
|
376
|
-
## Proxy
|
377
|
-
|
378
|
-
Support both `http` and `https` protocol.
|
379
|
-
|
380
|
-
**Notice: Only support on Node.js >= 4.0.0**
|
381
|
-
|
382
|
-
### Programming
|
383
|
-
|
384
|
-
```js
|
385
|
-
urllib.request('https://twitter.com/', {
|
386
|
-
enableProxy: true,
|
387
|
-
proxy: 'http://localhost:8008',
|
388
|
-
}, (err, data, res) => {
|
389
|
-
console.log(res.status, res.headers);
|
390
|
-
});
|
391
|
-
```
|
392
|
-
|
393
|
-
### System environment variable
|
394
|
-
|
395
|
-
- http
|
237
|
+
- `status` or `statusCode`: response status code.
|
238
|
+
- `-1` meaning some network error like `ENOTFOUND`
|
239
|
+
- `-2` meaning ConnectionTimeoutError
|
240
|
+
- `headers`: response http headers, default is `{}`
|
241
|
+
- `size`: response size
|
242
|
+
- `aborted`: response was aborted or not
|
243
|
+
- `rt`: total request and response time in ms.
|
244
|
+
- `timing`: timing object if timing enable.
|
245
|
+
- `remoteAddress`: http server ip address
|
246
|
+
- `remotePort`: http server ip port
|
247
|
+
- `socketHandledRequests`: socket already handled request count
|
248
|
+
- `socketHandledResponses`: socket already handled response count
|
249
|
+
|
250
|
+
## Run test with debug log
|
396
251
|
|
397
252
|
```bash
|
398
|
-
|
399
|
-
http_proxy=http://localhost:8008
|
253
|
+
NODE_DEBUG=urllib npm test
|
400
254
|
```
|
401
255
|
|
402
|
-
- https
|
403
|
-
|
404
|
-
```bash
|
405
|
-
HTTP_PROXY=http://localhost:8008
|
406
|
-
http_proxy=http://localhost:8008
|
407
|
-
HTTPS_PROXY=https://localhost:8008
|
408
|
-
https_proxy=https://localhost:8008
|
409
|
-
```
|
410
|
-
|
411
|
-
```bash
|
412
|
-
$ http_proxy=http://localhost:8008 node index.js
|
413
|
-
```
|
414
|
-
|
415
|
-
### Trace
|
416
|
-
If set trace true, error stack will contains full call stack, like
|
417
|
-
```
|
418
|
-
Error: connect ECONNREFUSED 127.0.0.1:11
|
419
|
-
at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1113:14)
|
420
|
-
--------------------
|
421
|
-
at ~/workspace/urllib/lib/urllib.js:150:13
|
422
|
-
at new Promise (<anonymous>)
|
423
|
-
at Object.request (~/workspace/urllib/lib/urllib.js:149:10)
|
424
|
-
at Context.<anonymous> (~/workspace/urllib/test/urllib_promise.test.js:49:19)
|
425
|
-
....
|
426
|
-
```
|
427
|
-
|
428
|
-
When open the trace, urllib may have poor perfomance, please consider carefully.
|
429
|
-
|
430
|
-
## TODO
|
431
|
-
|
432
|
-
* [ ] Support component
|
433
|
-
* [ ] Browser env use Ajax
|
434
|
-
* [√] Support Proxy
|
435
|
-
* [√] Upload file like form upload
|
436
|
-
* [√] Auto redirect handle
|
437
|
-
* [√] https & self-signed certificate
|
438
|
-
* [√] Connection timeout & Response timeout
|
439
|
-
* [√] Support `Accept-Encoding=gzip` by `options.gzip = true`
|
440
|
-
* [√] Support [Digest access authentication](http://en.wikipedia.org/wiki/Digest_access_authentication)
|
441
|
-
|
442
256
|
<!-- GITCONTRIBUTOR_START -->
|
443
257
|
|
444
258
|
## Contributors
|
@@ -459,6 +273,3 @@ This project follows the git-contributor [spec](https://github.com/xudafeng/git-
|
|
459
273
|
## License
|
460
274
|
|
461
275
|
[MIT](LICENSE)
|
462
|
-
|
463
|
-
|
464
|
-
[bluebird]: https://github.com/petkaantonov/bluebird
|
package/package.json
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "
|
4
|
-
"
|
3
|
+
"version": "3.0.0",
|
4
|
+
"publishConfig": {
|
5
|
+
"tag": "latest"
|
6
|
+
},
|
7
|
+
"description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Base undici fetch API.",
|
5
8
|
"keywords": [
|
6
9
|
"urllib",
|
7
10
|
"http",
|
@@ -9,83 +12,83 @@
|
|
9
12
|
"curl",
|
10
13
|
"wget",
|
11
14
|
"request",
|
12
|
-
"https"
|
15
|
+
"https",
|
16
|
+
"undici",
|
17
|
+
"fetch"
|
13
18
|
],
|
14
|
-
"author": "fengmk2 <fengmk2@gmail.com> (https://
|
19
|
+
"author": "fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)",
|
15
20
|
"homepage": "https://github.com/node-modules/urllib",
|
16
|
-
"
|
17
|
-
"
|
21
|
+
"type": "module",
|
22
|
+
"exports": {
|
23
|
+
".": {
|
24
|
+
"import": {
|
25
|
+
"types": "./src/esm/index.d.ts",
|
26
|
+
"default": "./src/esm/index.js"
|
27
|
+
},
|
28
|
+
"require": {
|
29
|
+
"types": "./src/cjs/index.d.ts",
|
30
|
+
"default": "./src/cjs/index.js"
|
31
|
+
}
|
32
|
+
}
|
33
|
+
},
|
34
|
+
"types": "./src/esm/index.d.ts",
|
35
|
+
"main": "./src/cjs/index.js",
|
18
36
|
"files": [
|
19
|
-
"
|
37
|
+
"src"
|
20
38
|
],
|
21
39
|
"repository": {
|
22
40
|
"type": "git",
|
23
41
|
"url": "git://github.com/node-modules/urllib.git"
|
24
42
|
},
|
25
43
|
"scripts": {
|
26
|
-
"
|
27
|
-
"
|
28
|
-
"
|
29
|
-
"
|
30
|
-
"
|
31
|
-
"
|
32
|
-
"
|
33
|
-
"
|
44
|
+
"lint": "eslint src --ext .ts",
|
45
|
+
"build": "npm run build:dist",
|
46
|
+
"build:cjs": "tsc -p ./tsconfig.build.cjs.json",
|
47
|
+
"build:esm": "tsc -p ./tsconfig.build.esm.json && node ./scripts/esm_import_fix.js",
|
48
|
+
"build:dist": "tsc --version && npm run build:cjs && npm run build:esm",
|
49
|
+
"build:cjs:test": "cd test/cjs && rm -rf node_modules && npm link ../.. && node index.js",
|
50
|
+
"build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
|
51
|
+
"build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test",
|
52
|
+
"test": "tsc --version && jest --coverage",
|
53
|
+
"ci": "npm run lint && npm run test && npm run build:test",
|
54
|
+
"contributor": "git-contributor",
|
55
|
+
"prepack": "npm run build && rm -rf src/*.tsbuildinfo"
|
34
56
|
},
|
35
57
|
"dependencies": {
|
36
|
-
"any-promise": "^1.3.0",
|
37
|
-
"content-type": "^1.0.2",
|
38
|
-
"debug": "^2.6.9",
|
39
58
|
"default-user-agent": "^1.0.0",
|
40
59
|
"digest-header": "^0.0.1",
|
41
|
-
"
|
42
|
-
"
|
43
|
-
"
|
44
|
-
"iconv-lite": "^0.4.15",
|
45
|
-
"ip": "^1.1.5",
|
46
|
-
"proxy-agent": "^5.0.0",
|
60
|
+
"form-data-encoder": "^1.7.2",
|
61
|
+
"formdata-node": "^4.3.3",
|
62
|
+
"mime-types": "^2.1.35",
|
47
63
|
"pump": "^3.0.0",
|
48
|
-
"
|
49
|
-
"statuses": "^1.3.1",
|
50
|
-
"utility": "^1.16.1"
|
64
|
+
"undici": "^5.6.0"
|
51
65
|
},
|
52
66
|
"devDependencies": {
|
53
|
-
"@types/
|
54
|
-
"@types/
|
55
|
-
"
|
56
|
-
"
|
57
|
-
"
|
58
|
-
"
|
59
|
-
"busboy": "^
|
60
|
-
"
|
61
|
-
"
|
62
|
-
"
|
63
|
-
"
|
64
|
-
"
|
65
|
-
"
|
66
|
-
"
|
67
|
-
"
|
68
|
-
"
|
69
|
-
"
|
70
|
-
"
|
71
|
-
"
|
72
|
-
"power-assert": "^1.4.2",
|
73
|
-
"semver": "5",
|
74
|
-
"spy": "^1.0.0",
|
75
|
-
"tar": "^4.4.8",
|
76
|
-
"through2": "^2.0.3",
|
77
|
-
"tsd": "^0.18.0",
|
78
|
-
"typescript": "^4.4.4"
|
67
|
+
"@types/busboy": "^1.5.0",
|
68
|
+
"@types/default-user-agent": "^1.0.0",
|
69
|
+
"@types/jest": "28",
|
70
|
+
"@types/mime-types": "^2.1.1",
|
71
|
+
"@types/pump": "^1.1.1",
|
72
|
+
"@types/selfsigned": "^2.0.1",
|
73
|
+
"busboy": "^1.6.0",
|
74
|
+
"coffee": "5",
|
75
|
+
"egg-ci": "2",
|
76
|
+
"eslint": "^8.17.0",
|
77
|
+
"eslint-config-egg": "^12.0.0",
|
78
|
+
"git-contributor": "1",
|
79
|
+
"iconv-lite": "^0.6.3",
|
80
|
+
"jest": "28",
|
81
|
+
"jest-summary-reporter": "^0.0.2",
|
82
|
+
"selfsigned": "^2.0.1",
|
83
|
+
"ts-jest": "28",
|
84
|
+
"tslib": "^2.4.0",
|
85
|
+
"typescript": "4"
|
79
86
|
},
|
80
87
|
"engines": {
|
81
|
-
"node": ">= 0.
|
88
|
+
"node": ">= 14.0.0"
|
82
89
|
},
|
83
90
|
"ci": {
|
84
|
-
"
|
85
|
-
"os": {
|
86
|
-
"github": "linux, windows, macos"
|
87
|
-
},
|
88
|
-
"version": "8, 10, 12, 14, 16"
|
91
|
+
"version": "14, 16, 18"
|
89
92
|
},
|
90
93
|
"license": "MIT"
|
91
94
|
}
|