urllib 3.16.1 → 3.17.1

package/README.md

@@ -64,7 +64,8 @@ console.log('status: %s, body size: %d, headers: %j', res.status, data.length, r
   - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
   - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.
   - ***headers*** Object - Request headers.
-  - ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Defaults to `exports.TIMEOUT`, both are 5s. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
+  - ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Default is `5000`. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
+  - **keepAliveTimeout** `number | null` - Default is `4000`, 4 seconds - The timeout after which a socket without active requests will time out. Monitors time between activity on a connected socket. This value may be overridden by *keep-alive* hints from the server. See [MDN: HTTP - Headers - Keep-Alive directives](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Keep-Alive#directives) for more details.
   - ***auth*** String - `username:password` used in HTTP Basic Authorization.
   - ***digestAuth*** String - `username:password` used in HTTP [Digest Authorization](https://en.wikipedia.org/wiki/Digest_access_authentication).
   - ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
@@ -72,8 +73,8 @@ console.log('status: %s, body size: %d, headers: %j', res.status, data.length, r
   - ***formatRedirectUrl*** Function - Format the redirect url by your self. Default is `url.resolve(from, to)`.
   - ***beforeRequest*** Function - Before request hook, you can change every thing here.
   - ***streaming*** Boolean - let you get the `res` object when request  connected, default `false`. alias `customResponse`
-  - ***compressed*** Boolean - Accept `gzip, br` response content and auto decode it, default is `false`.
-  - ***timing*** Boolean - Enable timing or not, default is `false`.
+  - ***compressed*** Boolean - Accept `gzip, br` response content and auto decode it, default is `true`.
+  - ***timing*** Boolean - Enable timing or not, default is `true`.
   - ***socketPath*** String | null - request a unix socket service, default is `null`.
   - ***highWaterMark*** Number - default is `67108864`, 64 KiB.
 
@@ -200,10 +201,7 @@ Response is normal object, it contains:
 - `aborted`: response was aborted or not
 - `rt`: total request and response time in ms.
 - `timing`: timing object if timing enable.
-- `remoteAddress`: http server ip address
-- `remotePort`: http server ip port
-- `socketHandledRequests`: socket already handled request count
-- `socketHandledResponses`: socket already handled response count
+- `socket`: socket info
 
 ## Run test with debug log
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "3.16.1",
+  "version": "3.17.1",
   "publishConfig": {
     "tag": "latest"
   },
@@ -53,6 +53,7 @@
     "build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test && npm run test-tsc",
     "test-tsc": "tsc -p ./test/fixtures/ts/tsconfig.json",
     "test": "npm run lint && vitest run",
+    "test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 180000 keep-alive-header.test.ts",
     "cov": "vitest run --coverage",
     "ci": "npm run lint && npm run cov && npm run build:test",
     "contributor": "git-contributor",
@@ -78,8 +79,9 @@
     "@types/pump": "^1.1.1",
     "@types/selfsigned": "^2.0.1",
     "@types/tar-stream": "^2.2.2",
-    "@vitest/coverage-c8": "^0.29.7",
+    "@vitest/coverage-v8": "^0.32.0",
     "busboy": "^1.6.0",
+    "cross-env": "^7.0.3",
     "eslint": "^8.25.0",
     "eslint-config-egg": "^12.1.0",
     "git-contributor": "^2.0.0",
@@ -88,7 +90,7 @@
     "selfsigned": "^2.0.1",
     "tar-stream": "^2.2.0",
     "typescript": "^5.0.4",
-    "vitest": "^0.31.1"
+    "vitest": "^0.32.0"
   },
   "engines": {
     "node": ">= 14.19.3"

package/src/HttpClient.ts

@@ -1,3 +1,4 @@
+import diagnosticsChannel from 'node:diagnostics_channel';
 import { EventEmitter } from 'node:events';
 import { LookupFunction } from 'node:net';
 import { STATUS_CODES } from 'node:http';
@@ -29,7 +30,7 @@ import pump from 'pump';
 // Compatible with old style formstream
 import FormStream from 'formstream';
 import { HttpAgent, CheckAddressFunction } from './HttpAgent';
-import { RequestURL, RequestOptions, HttpMethod } from './Request';
+import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request';
 import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response';
 import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils';
 import symbols from './symbols';
@@ -137,8 +138,26 @@ function defaultIsRetry(response: HttpClientResponse) {
 
 type RequestContext = {
   retries: number;
+  socketErrorRetries: number;
+  requestStartTime?: number;
 };
 
+const channels = {
+  request: diagnosticsChannel.channel('urllib:request'),
+  response: diagnosticsChannel.channel('urllib:response'),
+};
+
+export type RequestDiagnosticsMessage = {
+  request: RequestMeta;
+};
+
+export type ResponseDiagnosticsMessage = {
+  request: RequestMeta;
+  response: RawResponseWithMeta;
+  error?: Error;
+};
+
+
 export class HttpClient extends EventEmitter {
   #defaultArgs?: RequestOptions;
   #dispatcher?: Dispatcher;
@@ -188,6 +207,8 @@ export class HttpClient extends EventEmitter {
     const headers: IncomingHttpHeaders = {};
     const args = {
       retry: 0,
+      socketErrorRetry: 1,
+      timing: true,
       ...this.#defaultArgs,
       ...options,
       // keep method and headers exists on args for request event handler to easy use
@@ -196,9 +217,13 @@ export class HttpClient extends EventEmitter {
     };
     requestContext = {
       retries: 0,
+      socketErrorRetries: 0,
       ...requestContext,
     };
-    const requestStartTime = performance.now();
+    if (!requestContext.requestStartTime) {
+      requestContext.requestStartTime = performance.now();
+    }
+    const requestStartTime = requestContext.requestStartTime;
 
     // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
     const timing = {
@@ -232,8 +257,8 @@ export class HttpClient extends EventEmitter {
       args,
       ctx: args.ctx,
       retries: requestContext.retries,
-    };
-    const socketInfo = {
+    } as RequestMeta;
+    const socketInfo: SocketInfo = {
       id: 0,
       localAddress: '',
       localPort: 0,
@@ -259,6 +284,8 @@ export class HttpClient extends EventEmitter {
       requestUrls: [],
       timing,
       socket: socketInfo,
+      retries: requestContext.retries,
+      socketErrorRetries: requestContext.socketErrorRetries,
     } as any as RawResponseWithMeta;
 
     let headersTimeout = 5000;
@@ -302,10 +329,19 @@ export class HttpClient extends EventEmitter {
     if (requestContext.retries > 0) {
       headers['x-urllib-retry'] = `${requestContext.retries}/${args.retry}`;
     }
+    if (requestContext.socketErrorRetries > 0) {
+      headers['x-urllib-retry-on-socket-error'] = `${requestContext.socketErrorRetries}/${args.socketErrorRetry}`;
+    }
     if (args.auth && !headers.authorization) {
       headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
     }
 
+    // streaming request should disable socketErrorRetry and retry
+    let isStreamingRequest = false;
+    if (args.dataType === 'stream' || args.writeStream) {
+      isStreamingRequest = true;
+    }
+
     try {
       const requestOptions: IUndiciRequestOption = {
         method,
@@ -334,9 +370,11 @@ export class HttpClient extends EventEmitter {
         if (isReadable(args.stream) && !(args.stream instanceof Readable)) {
           debug('Request#%d convert old style stream to Readable', requestId);
           args.stream = new Readable().wrap(args.stream);
+          isStreamingRequest = true;
         } else if (args.stream instanceof FormStream) {
           debug('Request#%d convert formstream to Readable', requestId);
           args.stream = new Readable().wrap(args.stream);
+          isStreamingRequest = true;
         }
         args.content = args.stream;
       }
@@ -380,6 +418,7 @@ export class HttpClient extends EventEmitter {
           } else if (file instanceof Readable || isReadable(file as any)) {
             const fileName = getFileName(file) || `streamfile${index}`;
             formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
+            isStreamingRequest = true;
           }
         }
 
@@ -403,6 +442,7 @@ export class HttpClient extends EventEmitter {
           } else if (typeof args.content === 'string' && !headers['content-type']) {
             headers['content-type'] = 'text/plain;charset=UTF-8';
           }
+          isStreamingRequest = isReadable(args.content);
         }
       } else if (args.data) {
         const isStringOrBufferOrReadable = typeof args.data === 'string'
@@ -419,6 +459,7 @@ export class HttpClient extends EventEmitter {
         } else {
           if (isStringOrBufferOrReadable) {
             requestOptions.body = args.data;
+            isStreamingRequest = isReadable(args.data);
           } else {
             if (args.contentType === 'json'
               || args.contentType === 'application/json'
@@ -434,10 +475,17 @@ export class HttpClient extends EventEmitter {
           }
         }
       }
+      if (isStreamingRequest) {
+        args.retry = 0;
+        args.socketErrorRetry = 0;
+      }
 
-      debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s',
-        requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout);
+      debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s',
+        requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout, isStreamingRequest);
       requestOptions.headers = headers;
+      channels.request.publish({
+        request: reqMeta,
+      } as RequestDiagnosticsMessage);
       if (this.listenerCount('request') > 0) {
         this.emit('request', reqMeta);
       }
@@ -486,8 +534,6 @@ export class HttpClient extends EventEmitter {
 
       let data: any = null;
       if (args.dataType === 'stream') {
-        // streaming mode will disable retry
-        args.retry = 0;
         // only auto decompress on request args.compressed = true
         if (args.compressed === true && isCompressedContent) {
           // gzip or br
@@ -497,8 +543,6 @@ export class HttpClient extends EventEmitter {
           res = Object.assign(response.body, res);
         }
       } else if (args.writeStream) {
-        // streaming mode will disable retry
-        args.retry = 0;
         if (args.compressed === true && isCompressedContent) {
           const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
           await pipelinePromise(response.body, decoder, args.writeStream);
@@ -556,6 +600,10 @@ export class HttpClient extends EventEmitter {
         }
       }
 
+      channels.response.publish({
+        request: reqMeta,
+        response: res,
+      } as ResponseDiagnosticsMessage);
       if (this.listenerCount('response') > 0) {
         this.emit('response', {
           requestId,
@@ -577,11 +625,22 @@ export class HttpClient extends EventEmitter {
         err = new HttpClientRequestTimeoutError(headersTimeout, { cause: e });
       } else if (err.name === 'BodyTimeoutError') {
         err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
+      } else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
+        // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
+        if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
+          requestContext.socketErrorRetries++;
+          return await this.#requestInternal(url, options, requestContext);
+        }
       }
       err.opaque = orginalOpaque;
       err.status = res.status;
       err.headers = res.headers;
       err.res = res;
+      if (err.socket) {
+        // store rawSocket
+        err._rawSocket = err.socket;
+      }
+      err.socket = socketInfo;
       // make sure requestUrls not empty
       if (res.requestUrls.length === 0) {
         res.requestUrls.push(requestUrl.href);
@@ -589,6 +648,11 @@ export class HttpClient extends EventEmitter {
       res.rt = performanceTime(requestStartTime);
       this.#updateSocketInfo(socketInfo, internalOpaque);
 
+      channels.response.publish({
+        request: reqMeta,
+        response: res,
+        error: err,
+      } as ResponseDiagnosticsMessage);
       if (this.listenerCount('response') > 0) {
         this.emit('response', {
           requestId,
@@ -611,13 +675,16 @@ export class HttpClient extends EventEmitter {
       socketInfo.id = socket[symbols.kSocketId];
       socketInfo.handledRequests = socket[symbols.kHandledRequests];
       socketInfo.handledResponses = socket[symbols.kHandledResponses];
-      socketInfo.localAddress = socket.localAddress;
-      socketInfo.localPort = socket.localPort;
+      socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
+      socketInfo.localPort = socket[symbols.kSocketLocalPort];
       socketInfo.remoteAddress = socket.remoteAddress;
       socketInfo.remotePort = socket.remotePort;
       socketInfo.remoteFamily = socket.remoteFamily;
       socketInfo.bytesRead = socket.bytesRead;
       socketInfo.bytesWritten = socket.bytesWritten;
+      socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
+      socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
+      socket[symbols.kSocketRequestEndTime] = new Date();
     }
   }
 }

package/src/Request.ts

@@ -69,11 +69,16 @@ export type RequestOptions = {
   headers?: IncomingHttpHeaders;
   /**
    * Request timeout in milliseconds for connecting phase and response receiving phase.
-   * Defaults to exports.
-   * TIMEOUT, both are 5s. You can use timeout: 5000 to tell urllib use same timeout on two phase or set them seperately such as
+   * Defaults is `5000`, both are 5 seconds. You can use timeout: 5000 to tell urllib use same timeout on two phase or set them separately such as
    * timeout: [3000, 5000], which will set connecting timeout to 3s and response 5s.
    */
   timeout?: number | number[];
+  /**
+   * Default is `4000`,  4 seconds - The timeout after which a socket without active requests will time out.
+   * Monitors time between activity on a connected socket.
+   * This value may be overridden by *keep-alive* hints from the server. See [MDN: HTTP - Headers - Keep-Alive directives](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Keep-Alive#directives) for more details.
+   */
+  keepAliveTimeout?: number;
   /**
    * username:password used in HTTP Basic Authorization.
    * Alias to `headers.authorization = xxx`
@@ -91,7 +96,7 @@ export type RequestOptions = {
   formatRedirectUrl?: (a: any, b: any) => void;
   /** Before request hook, you can change every thing here. */
   beforeRequest?: (...args: any[]) => void;
-  /** Accept `gzip, br` response content and auto decode it, default is false. */
+  /** Accept `gzip, br` response content and auto decode it, default is `true`. */
   compressed?: boolean;
   /**
    * @deprecated
@@ -99,11 +104,11 @@ export type RequestOptions = {
    * */
   gzip?: boolean;
   /**
-   * Enable timing or not, default is false.
+   * Enable timing or not, default is `true`.
    * */
   timing?: boolean;
   /**
-   * Auto retry times on 5xx response, default is 0. Don't work on streaming request
+   * Auto retry times on 5xx response, default is `0`. Don't work on streaming request
    * It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
    **/
   retry?: number;
@@ -114,6 +119,11 @@ export type RequestOptions = {
    * It will retry when status >= 500 by default. Request error is not included.
    */
   isRetry?: (response: HttpClientResponse) => boolean;
+  /**
+   * Auto retry times on socket error, default is `1`. Don't work on streaming request
+   * It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
+   **/
+  socketErrorRetry?: number;
   /** Default: `null` */
   opaque?: unknown;
   /**
@@ -134,3 +144,11 @@ export type RequestOptions = {
   /** Default: `64 KiB` */
   highWaterMark?: number;
 };
+
+export type RequestMeta = {
+  requestId: number;
+  url: string;
+  args: RequestOptions;
+  ctx?: unknown;
+  retries: number;
+};

package/src/Response.ts

@@ -12,6 +12,8 @@ export type SocketInfo = {
   bytesRead: number;
   handledRequests: number;
   handledResponses: number;
+  connectedTime?: Date;
+  lastRequestEndTime?: Date;
 };
 
 /**
@@ -47,6 +49,8 @@ export type RawResponseWithMeta = Readable & {
   rt: number;
   keepAliveSocket: boolean;
   requestUrls: string[];
+  retries: number;
+  socketErrorRetries: number;
 };
 
 export type HttpClientResponse<T = any> = {

package/src/cjs/HttpClient.d.ts

@@ -4,8 +4,8 @@
 import { EventEmitter } from 'node:events';
 import { LookupFunction } from 'node:net';
 import { CheckAddressFunction } from './HttpAgent';
-import { RequestURL, RequestOptions } from './Request';
-import { HttpClientResponse } from './Response';
+import { RequestURL, RequestOptions, RequestMeta } from './Request';
+import { RawResponseWithMeta, HttpClientResponse } from './Response';
 export type ClientOptions = {
     defaultArgs?: RequestOptions;
     /**
@@ -37,6 +37,14 @@ export type ClientOptions = {
     };
 };
 export declare const HEADER_USER_AGENT: string;
+export type RequestDiagnosticsMessage = {
+    request: RequestMeta;
+};
+export type ResponseDiagnosticsMessage = {
+    request: RequestMeta;
+    response: RawResponseWithMeta;
+    error?: Error;
+};
 export declare class HttpClient extends EventEmitter {
     #private;
     constructor(clientOptions?: ClientOptions);

package/src/cjs/HttpClient.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HttpClient = exports.HEADER_USER_AGENT = void 0;
+const node_diagnostics_channel_1 = __importDefault(require("node:diagnostics_channel"));
 const node_events_1 = require("node:events");
 const node_http_1 = require("node:http");
 const node_util_1 = require("node:util");
@@ -69,7 +70,7 @@ class HttpClientRequestTimeoutError extends Error {
         Error.captureStackTrace(this, this.constructor);
     }
 }
-exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.16.1');
+exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.17.1');
 function getFileName(stream) {
     const filePath = stream.path;
     if (filePath) {
@@ -80,6 +81,10 @@ function getFileName(stream) {
 function defaultIsRetry(response) {
     return response.status >= 500;
 }
+const channels = {
+    request: node_diagnostics_channel_1.default.channel('urllib:request'),
+    response: node_diagnostics_channel_1.default.channel('urllib:response'),
+};
 class HttpClient extends node_events_1.EventEmitter {
     #defaultArgs;
     #dispatcher;
@@ -126,6 +131,8 @@ class HttpClient extends node_events_1.EventEmitter {
         const headers = {};
         const args = {
             retry: 0,
+            socketErrorRetry: 1,
+            timing: true,
             ...this.#defaultArgs,
             ...options,
             // keep method and headers exists on args for request event handler to easy use
@@ -134,9 +141,13 @@ class HttpClient extends node_events_1.EventEmitter {
         };
         requestContext = {
             retries: 0,
+            socketErrorRetries: 0,
             ...requestContext,
         };
-        const requestStartTime = node_perf_hooks_1.performance.now();
+        if (!requestContext.requestStartTime) {
+            requestContext.requestStartTime = node_perf_hooks_1.performance.now();
+        }
+        const requestStartTime = requestContext.requestStartTime;
         // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
         const timing = {
             // socket assigned
@@ -196,6 +207,8 @@ class HttpClient extends node_events_1.EventEmitter {
             requestUrls: [],
             timing,
             socket: socketInfo,
+            retries: requestContext.retries,
+            socketErrorRetries: requestContext.socketErrorRetries,
         };
         let headersTimeout = 5000;
         let bodyTimeout = 5000;
@@ -240,9 +253,17 @@ class HttpClient extends node_events_1.EventEmitter {
         if (requestContext.retries > 0) {
             headers['x-urllib-retry'] = `${requestContext.retries}/${args.retry}`;
         }
+        if (requestContext.socketErrorRetries > 0) {
+            headers['x-urllib-retry-on-socket-error'] = `${requestContext.socketErrorRetries}/${args.socketErrorRetry}`;
+        }
         if (args.auth && !headers.authorization) {
             headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
         }
+        // streaming request should disable socketErrorRetry and retry
+        let isStreamingRequest = false;
+        if (args.dataType === 'stream' || args.writeStream) {
+            isStreamingRequest = true;
+        }
         try {
             const requestOptions = {
                 method,
@@ -270,10 +291,12 @@ class HttpClient extends node_events_1.EventEmitter {
                 if ((0, utils_1.isReadable)(args.stream) && !(args.stream instanceof node_stream_1.Readable)) {
                     debug('Request#%d convert old style stream to Readable', requestId);
                     args.stream = new node_stream_1.Readable().wrap(args.stream);
+                    isStreamingRequest = true;
                 }
                 else if (args.stream instanceof formstream_1.default) {
                     debug('Request#%d convert formstream to Readable', requestId);
                     args.stream = new node_stream_1.Readable().wrap(args.stream);
+                    isStreamingRequest = true;
                 }
                 args.content = args.stream;
             }
@@ -321,6 +344,7 @@ class HttpClient extends node_events_1.EventEmitter {
                     else if (file instanceof node_stream_1.Readable || (0, utils_1.isReadable)(file)) {
                         const fileName = getFileName(file) || `streamfile${index}`;
                         formData.append(field, new BlobFromStream(file, mime_types_1.default.lookup(fileName) || ''), fileName);
+                        isStreamingRequest = true;
                     }
                 }
                 if (undici_1.FormData) {
@@ -346,6 +370,7 @@ class HttpClient extends node_events_1.EventEmitter {
                     else if (typeof args.content === 'string' && !headers['content-type']) {
                         headers['content-type'] = 'text/plain;charset=UTF-8';
                     }
+                    isStreamingRequest = (0, utils_1.isReadable)(args.content);
                 }
             }
             else if (args.data) {
@@ -365,6 +390,7 @@ class HttpClient extends node_events_1.EventEmitter {
                 else {
                     if (isStringOrBufferOrReadable) {
                         requestOptions.body = args.data;
+                        isStreamingRequest = (0, utils_1.isReadable)(args.data);
                     }
                     else {
                         if (args.contentType === 'json'
@@ -382,8 +408,15 @@ class HttpClient extends node_events_1.EventEmitter {
                     }
                 }
             }
-            debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s', requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout);
+            if (isStreamingRequest) {
+                args.retry = 0;
+                args.socketErrorRetry = 0;
+            }
+            debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s', requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout, isStreamingRequest);
             requestOptions.headers = headers;
+            channels.request.publish({
+                request: reqMeta,
+            });
             if (this.listenerCount('request') > 0) {
                 this.emit('request', reqMeta);
             }
@@ -428,8 +461,6 @@ class HttpClient extends node_events_1.EventEmitter {
             }
             let data = null;
             if (args.dataType === 'stream') {
-                // streaming mode will disable retry
-                args.retry = 0;
                 // only auto decompress on request args.compressed = true
                 if (args.compressed === true && isCompressedContent) {
                     // gzip or br
@@ -441,8 +472,6 @@ class HttpClient extends node_events_1.EventEmitter {
                 }
             }
             else if (args.writeStream) {
-                // streaming mode will disable retry
-                args.retry = 0;
                 if (args.compressed === true && isCompressedContent) {
                     const decoder = contentEncoding === 'gzip' ? (0, node_zlib_1.createGunzip)() : (0, node_zlib_1.createBrotliDecompress)();
                     await pipelinePromise(response.body, decoder, args.writeStream);
@@ -502,6 +531,10 @@ class HttpClient extends node_events_1.EventEmitter {
                     return await this.#requestInternal(url, options, requestContext);
                 }
             }
+            channels.response.publish({
+                request: reqMeta,
+                response: res,
+            });
             if (this.listenerCount('response') > 0) {
                 this.emit('response', {
                     requestId,
@@ -525,16 +558,33 @@ class HttpClient extends node_events_1.EventEmitter {
             else if (err.name === 'BodyTimeoutError') {
                 err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
             }
+            else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
+                // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
+                if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
+                    requestContext.socketErrorRetries++;
+                    return await this.#requestInternal(url, options, requestContext);
+                }
+            }
             err.opaque = orginalOpaque;
             err.status = res.status;
             err.headers = res.headers;
             err.res = res;
+            if (err.socket) {
+                // store rawSocket
+                err._rawSocket = err.socket;
+            }
+            err.socket = socketInfo;
             // make sure requestUrls not empty
             if (res.requestUrls.length === 0) {
                 res.requestUrls.push(requestUrl.href);
             }
             res.rt = (0, utils_1.performanceTime)(requestStartTime);
             this.#updateSocketInfo(socketInfo, internalOpaque);
+            channels.response.publish({
+                request: reqMeta,
+                response: res,
+                error: err,
+            });
             if (this.listenerCount('response') > 0) {
                 this.emit('response', {
                     requestId,
@@ -556,13 +606,16 @@ class HttpClient extends node_events_1.EventEmitter {
             socketInfo.id = socket[symbols_1.default.kSocketId];
             socketInfo.handledRequests = socket[symbols_1.default.kHandledRequests];
             socketInfo.handledResponses = socket[symbols_1.default.kHandledResponses];
-            socketInfo.localAddress = socket.localAddress;
-            socketInfo.localPort = socket.localPort;
+            socketInfo.localAddress = socket[symbols_1.default.kSocketLocalAddress];
+            socketInfo.localPort = socket[symbols_1.default.kSocketLocalPort];
             socketInfo.remoteAddress = socket.remoteAddress;
             socketInfo.remotePort = socket.remotePort;
             socketInfo.remoteFamily = socket.remoteFamily;
             socketInfo.bytesRead = socket.bytesRead;
             socketInfo.bytesWritten = socket.bytesWritten;
+            socketInfo.connectedTime = socket[symbols_1.default.kSocketConnectedTime];
+            socketInfo.lastRequestEndTime = socket[symbols_1.default.kSocketRequestEndTime];
+            socket[symbols_1.default.kSocketRequestEndTime] = new Date();
         }
     }
 }