urllib 3.16.1 → 3.17.1

package/src/esm/Request.d.ts

@@ -67,11 +67,16 @@ export type RequestOptions = {
     headers?: IncomingHttpHeaders;
     /**
      * Request timeout in milliseconds for connecting phase and response receiving phase.
-     * Defaults to exports.
-     * TIMEOUT, both are 5s. You can use timeout: 5000 to tell urllib use same timeout on two phase or set them seperately such as
+     * Defaults is `5000`, both are 5 seconds. You can use timeout: 5000 to tell urllib use same timeout on two phase or set them separately such as
      * timeout: [3000, 5000], which will set connecting timeout to 3s and response 5s.
      */
     timeout?: number | number[];
+    /**
+     * Default is `4000`,  4 seconds - The timeout after which a socket without active requests will time out.
+     * Monitors time between activity on a connected socket.
+     * This value may be overridden by *keep-alive* hints from the server. See [MDN: HTTP - Headers - Keep-Alive directives](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Keep-Alive#directives) for more details.
+     */
+    keepAliveTimeout?: number;
     /**
      * username:password used in HTTP Basic Authorization.
      * Alias to `headers.authorization = xxx`
@@ -89,7 +94,7 @@ export type RequestOptions = {
     formatRedirectUrl?: (a: any, b: any) => void;
     /** Before request hook, you can change every thing here. */
     beforeRequest?: (...args: any[]) => void;
-    /** Accept `gzip, br` response content and auto decode it, default is false. */
+    /** Accept `gzip, br` response content and auto decode it, default is `true`. */
     compressed?: boolean;
     /**
      * @deprecated
@@ -97,11 +102,11 @@ export type RequestOptions = {
      * */
     gzip?: boolean;
     /**
-     * Enable timing or not, default is false.
+     * Enable timing or not, default is `true`.
      * */
     timing?: boolean;
     /**
-     * Auto retry times on 5xx response, default is 0. Don't work on streaming request
+     * Auto retry times on 5xx response, default is `0`. Don't work on streaming request
      * It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
      **/
     retry?: number;
@@ -112,6 +117,11 @@ export type RequestOptions = {
      * It will retry when status >= 500 by default. Request error is not included.
      */
     isRetry?: (response: HttpClientResponse) => boolean;
+    /**
+     * Auto retry times on socket error, default is `1`. Don't work on streaming request
+     * It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
+     **/
+    socketErrorRetry?: number;
     /** Default: `null` */
     opaque?: unknown;
     /**
@@ -132,3 +142,10 @@ export type RequestOptions = {
     /** Default: `64 KiB` */
     highWaterMark?: number;
 };
+export type RequestMeta = {
+    requestId: number;
+    url: string;
+    args: RequestOptions;
+    ctx?: unknown;
+    retries: number;
+};

package/src/esm/Response.d.ts

@@ -13,6 +13,8 @@ export type SocketInfo = {
     bytesRead: number;
     handledRequests: number;
     handledResponses: number;
+    connectedTime?: Date;
+    lastRequestEndTime?: Date;
 };
 /**
  * https://eggjs.org/en/core/httpclient.html#timing-boolean
@@ -37,6 +39,8 @@ export type RawResponseWithMeta = Readable & {
     rt: number;
     keepAliveSocket: boolean;
     requestUrls: string[];
+    retries: number;
+    socketErrorRetries: number;
 };
 export type HttpClientResponse<T = any> = {
     opaque: unknown;

package/src/esm/diagnosticsChannel.js

@@ -15,6 +15,25 @@ let initedDiagnosticsChannel = false;
 // server --> client
 // undici:request:headers => { request, response }
 //   -> undici:request:trailers => { request, trailers }
+function subscribe(name, listener) {
+    if (typeof diagnosticsChannel.subscribe === 'function') {
+        diagnosticsChannel.subscribe(name, listener);
+    }
+    else {
+        // TODO: support Node.js 14, will be removed on the next major version
+        diagnosticsChannel.channel(name).subscribe(listener);
+    }
+}
+function formatSocket(socket) {
+    if (!socket)
+        return socket;
+    return {
+        localAddress: socket[symbols.kSocketLocalAddress],
+        localPort: socket[symbols.kSocketLocalPort],
+        remoteAddress: socket.remoteAddress,
+        remotePort: socket.remotePort,
+    };
+}
 export function initDiagnosticsChannel() {
     // makre sure init global DiagnosticsChannel once
     if (initedDiagnosticsChannel)
@@ -23,7 +42,7 @@ export function initDiagnosticsChannel() {
     let kHandler;
     // This message is published when a new outgoing request is created.
     // Note: a request is only loosely completed to a given socket.
-    diagnosticsChannel.channel('undici:request:create').subscribe((message, name) => {
+    subscribe('undici:request:create', (message, name) => {
         const { request } = message;
         if (!kHandler) {
             const symbols = Object.getOwnPropertySymbols(request);
@@ -46,16 +65,20 @@ export function initDiagnosticsChannel() {
     // diagnosticsChannel.channel('undici:client:beforeConnect')
     // diagnosticsChannel.channel('undici:client:connectError')
     // This message is published after a connection is established.
-    diagnosticsChannel.channel('undici:client:connected').subscribe((message, name) => {
+    subscribe('undici:client:connected', (message, name) => {
         const { socket } = message;
         socket[symbols.kSocketId] = globalId('UndiciSocket');
         socket[symbols.kSocketStartTime] = performance.now();
+        socket[symbols.kSocketConnectedTime] = new Date();
         socket[symbols.kHandledRequests] = 0;
         socket[symbols.kHandledResponses] = 0;
-        debug('[%s] Socket#%d connected', name, socket[symbols.kSocketId]);
+        // copy local address to symbol, avoid them be reset after request error throw
+        socket[symbols.kSocketLocalAddress] = socket.localAddress;
+        socket[symbols.kSocketLocalPort] = socket.localPort;
+        debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
     });
     // This message is published right before the first byte of the request is written to the socket.
-    diagnosticsChannel.channel('undici:client:sendHeaders').subscribe((message, name) => {
+    subscribe('undici:client:sendHeaders', (message, name) => {
         const { request, socket } = message;
         if (!kHandler)
             return;
@@ -65,7 +88,7 @@ export function initDiagnosticsChannel() {
         socket[symbols.kHandledRequests]++;
         // attach socket to opaque
         opaque[symbols.kRequestSocket] = socket;
-        debug('[%s] Request#%d send headers on Socket#%d (handled %d requests)', name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests]);
+        debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)', name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests], formatSocket(socket));
         if (!opaque[symbols.kEnableRequestTiming])
             return;
         opaque[symbols.kRequestTiming].requestHeadersSent = performanceTime(opaque[symbols.kRequestStartTime]);
@@ -76,7 +99,7 @@ export function initDiagnosticsChannel() {
                 performanceTime(opaque[symbols.kRequestStartTime], socket[symbols.kSocketStartTime]);
         }
     });
-    diagnosticsChannel.channel('undici:request:bodySent').subscribe((message, name) => {
+    subscribe('undici:request:bodySent', (message, name) => {
         const { request } = message;
         if (!kHandler)
             return;
@@ -89,7 +112,7 @@ export function initDiagnosticsChannel() {
         opaque[symbols.kRequestTiming].requestSent = performanceTime(opaque[symbols.kRequestStartTime]);
     });
     // This message is published after the response headers have been received, i.e. the response has been completed.
-    diagnosticsChannel.channel('undici:request:headers').subscribe((message, name) => {
+    subscribe('undici:request:headers', (message, name) => {
         const { request, response } = message;
         if (!kHandler)
             return;
@@ -99,13 +122,13 @@ export function initDiagnosticsChannel() {
         // get socket from opaque
         const socket = opaque[symbols.kRequestSocket];
         socket[symbols.kHandledResponses]++;
-        debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses)', name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses]);
+        debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)', name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses], formatSocket(socket));
         if (!opaque[symbols.kEnableRequestTiming])
             return;
         opaque[symbols.kRequestTiming].waiting = performanceTime(opaque[symbols.kRequestStartTime]);
     });
     // This message is published after the response body and trailers have been received, i.e. the response has been completed.
-    diagnosticsChannel.channel('undici:request:trailers').subscribe((message, name) => {
+    subscribe('undici:request:trailers', (message, name) => {
         const { request } = message;
         if (!kHandler)
             return;
@@ -117,5 +140,14 @@ export function initDiagnosticsChannel() {
             return;
         opaque[symbols.kRequestTiming].contentDownload = performanceTime(opaque[symbols.kRequestStartTime]);
     });
-    // diagnosticsChannel.channel('undici:request:error')
+    // This message is published if the request is going to error, but it has not errored yet.
+    // subscribe('undici:request:error', (message, name) => {
+    //   const { request, error } = message as DiagnosticsChannel.RequestErrorMessage;
+    //   const opaque = request[kHandler]?.opts?.opaque;
+    //   if (!opaque || !opaque[symbols.kRequestId]) return;
+    //   const socket = opaque[symbols.kRequestSocket];
+    //   debug('[%s] Request#%d error on Socket#%d (handled %d responses, sock: %o), error: %o',
+    //     name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledResponses],
+    //     formatSocket(socket), error);
+    // });
 }

package/src/esm/index.d.ts

@@ -2,7 +2,7 @@ import { RequestOptions, RequestURL } from './Request';
 export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse<T>>;
 export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse<T>>;
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient';
+export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient';
 export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request';
 export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response';
 declare const _default: {

package/src/esm/index.js

@@ -28,7 +28,7 @@ export async function curl(url, options) {
 }
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
 // HttpClient2 is keep compatible with urlib@2 HttpClient2
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient.js';
+export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
 export default {
     request,
     curl,

package/src/esm/symbols.d.ts

@@ -1,6 +1,10 @@
 declare const _default: {
     kSocketId: symbol;
     kSocketStartTime: symbol;
+    kSocketConnectedTime: symbol;
+    kSocketRequestEndTime: symbol;
+    kSocketLocalAddress: symbol;
+    kSocketLocalPort: symbol;
     kHandledRequests: symbol;
     kHandledResponses: symbol;
     kRequestSocket: symbol;

package/src/esm/symbols.js

@@ -1,6 +1,10 @@
 export default {
     kSocketId: Symbol('socket id'),
     kSocketStartTime: Symbol('socket start time'),
+    kSocketConnectedTime: Symbol('socket connected time'),
+    kSocketRequestEndTime: Symbol('socket request end time'),
+    kSocketLocalAddress: Symbol('socket local address'),
+    kSocketLocalPort: Symbol('socket local port'),
     kHandledRequests: Symbol('handled requests per socket'),
     kHandledResponses: Symbol('handled responses per socket'),
     kRequestSocket: Symbol('request on the socket'),

package/src/index.ts

@@ -36,7 +36,10 @@ export {
   setGlobalDispatcher, getGlobalDispatcher,
 } from 'undici';
 // HttpClient2 is keep compatible with urlib@2 HttpClient2
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient';
+export {
+  HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
+  RequestDiagnosticsMessage, ResponseDiagnosticsMessage,
+} from './HttpClient';
 // RequestOptions2 is keep compatible with urlib@2 RequestOptions2
 export {
   RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod,

package/src/symbols.ts

@@ -1,6 +1,10 @@
 export default {
   kSocketId: Symbol('socket id'),
   kSocketStartTime: Symbol('socket start time'),
+  kSocketConnectedTime: Symbol('socket connected time'),
+  kSocketRequestEndTime: Symbol('socket request end time'),
+  kSocketLocalAddress: Symbol('socket local address'),
+  kSocketLocalPort: Symbol('socket local port'),
   kHandledRequests: Symbol('handled requests per socket'),
   kHandledResponses: Symbol('handled responses per socket'),
   kRequestSocket: Symbol('request on the socket'),