urllib 3.1.2 → 3.2.0

package/History.md

@@ -1,4 +1,19 @@
 
+3.2.0 / 2022-09-26
+==================
+
+**others**
+  * [[`315f4a0`](http://github.com/node-modules/urllib/commit/315f4a0bf69abc2b29247fcf1a606c44412a86f3)] - 📦 NEW: Support timing and socket info (#400) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`414692b`](http://github.com/node-modules/urllib/commit/414692b183fb205acfb163f8aef4f853ce1b97e1)] - 🧪️ TEST: Add diagnostics_channel test cases (#398) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`65abb60`](http://github.com/node-modules/urllib/commit/65abb6074e5098834dd964a2bedaffd64a703ef9)] - 🤖 TEST: Use vitest instead of jest (#399) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`5f0c4a8`](http://github.com/node-modules/urllib/commit/5f0c4a8da84ef9954942bcd78721a5445e0cb095)] - 📖 DOC: Update contributors (fengmk2 <<fengmk2@gmail.com>>)
+
+3.1.3 / 2022-09-09
+==================
+
+**others**
+  * [[`65c5112`](http://github.com/node-modules/urllib/commit/65c5112aaaa8a6b6a9e324b0522fe1635e3792ba)] - 🐛 FIX: Include query parameters in digest uri (#396) (Chris Byrne <<github@adapt.gen.nz>>)
+
 3.1.2 / 2022-08-24
 ==================
 

package/README.md

@@ -251,7 +251,7 @@ Response is normal object, it contains:
 ## Run test with debug log
 
 ```bash
-NODE_DEBUG=urllib npm test
+NODE_DEBUG=urllib:* npm test
 ```
 
 ## Mocking Request
@@ -323,12 +323,12 @@ Fork [undici benchmarks script](https://github.com/fengmk2/undici/blob/urllib-be
 | :---: | :---: | :---: | :---: | :---: | :---: |
 |[<img src="https://avatars.githubusercontent.com/u/14790466?v=4" width="100px;"/><br/><sub><b>greenkeeperio-bot</b></sub>](https://github.com/greenkeeperio-bot)<br/>|[<img src="https://avatars.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|[<img src="https://avatars.githubusercontent.com/u/5381764?v=4" width="100px;"/><br/><sub><b>paambaati</b></sub>](https://github.com/paambaati)<br/>|[<img src="https://avatars.githubusercontent.com/u/1433247?v=4" width="100px;"/><br/><sub><b>denghongcai</b></sub>](https://github.com/denghongcai)<br/>|[<img src="https://avatars.githubusercontent.com/u/2842176?v=4" width="100px;"/><br/><sub><b>XadillaX</b></sub>](https://github.com/XadillaX)<br/>|[<img src="https://avatars.githubusercontent.com/u/1147375?v=4" width="100px;"/><br/><sub><b>alsotang</b></sub>](https://github.com/alsotang)<br/>|
 |[<img src="https://avatars.githubusercontent.com/u/546535?v=4" width="100px;"/><br/><sub><b>leoner</b></sub>](https://github.com/leoner)<br/>|[<img src="https://avatars.githubusercontent.com/u/19908330?v=4" width="100px;"/><br/><sub><b>hyj1991</b></sub>](https://github.com/hyj1991)<br/>|[<img src="https://avatars.githubusercontent.com/u/1747852?v=4" width="100px;"/><br/><sub><b>isayme</b></sub>](https://github.com/isayme)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/252317?v=4" width="100px;"/><br/><sub><b>cyjake</b></sub>](https://github.com/cyjake)<br/>|[<img src="https://avatars.githubusercontent.com/u/5856440?v=4" width="100px;"/><br/><sub><b>whxaxes</b></sub>](https://github.com/whxaxes)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/309219?v=4" width="100px;"/><br/><sub><b>chadxz</b></sub>](https://github.com/chadxz)<br/>|[<img src="https://avatars.githubusercontent.com/u/5139554?v=4" width="100px;"/><br/><sub><b>danielwpz</b></sub>](https://github.com/danielwpz)<br/>|[<img src="https://avatars.githubusercontent.com/u/5127897?v=4" width="100px;"/><br/><sub><b>danielsss</b></sub>](https://github.com/danielsss)<br/>|[<img src="https://avatars.githubusercontent.com/u/3367820?v=4" width="100px;"/><br/><sub><b>Jeff-Tian</b></sub>](https://github.com/Jeff-Tian)<br/>|[<img src="https://avatars.githubusercontent.com/u/32407?v=4" width="100px;"/><br/><sub><b>jedahan</b></sub>](https://github.com/jedahan)<br/>|[<img src="https://avatars.githubusercontent.com/u/17075261?v=4" width="100px;"/><br/><sub><b>nick-ng</b></sub>](https://github.com/nick-ng)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/1706595?v=4" width="100px;"/><br/><sub><b>rishavsharan</b></sub>](https://github.com/rishavsharan)<br/>|[<img src="https://avatars.githubusercontent.com/u/1886161?v=4" width="100px;"/><br/><sub><b>willizm</b></sub>](https://github.com/willizm)<br/>|[<img src="https://avatars.githubusercontent.com/u/7227589?v=4" width="100px;"/><br/><sub><b>davidkhala</b></sub>](https://github.com/davidkhala)<br/>|[<img src="https://avatars.githubusercontent.com/u/535479?v=4" width="100px;"/><br/><sub><b>aleafs</b></sub>](https://github.com/aleafs)<br/>|[<img src="https://avatars.githubusercontent.com/u/3689968?v=4" width="100px;"/><br/><sub><b>Amunu</b></sub>](https://github.com/Amunu)<br/>|[<img src="https://avatars.githubusercontent.com/in/9426?v=4" width="100px;"/><br/><sub><b>azure-pipelines[bot]</b></sub>](https://github.com/apps/azure-pipelines)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/1281323?v=4" width="100px;"/><br/><sub><b>changzhiwin</b></sub>](https://github.com/changzhiwin)<br/>|[<img src="https://avatars.githubusercontent.com/u/929503?v=4" width="100px;"/><br/><sub><b>yuzhigang33</b></sub>](https://github.com/yuzhigang33)<br/>|[<img src="https://avatars.githubusercontent.com/u/981128?v=4" width="100px;"/><br/><sub><b>fishbar</b></sub>](https://github.com/fishbar)<br/>|[<img src="https://avatars.githubusercontent.com/u/1207064?v=4" width="100px;"/><br/><sub><b>gxcsoccer</b></sub>](https://github.com/gxcsoccer)<br/>|[<img src="https://avatars.githubusercontent.com/u/17476119?v=4" width="100px;"/><br/><sub><b>mars-coder</b></sub>](https://github.com/mars-coder)<br/>|[<img src="https://avatars.githubusercontent.com/u/929179?v=4" width="100px;"/><br/><sub><b>rockdai</b></sub>](https://github.com/rockdai)<br/>|
-[<img src="https://avatars.githubusercontent.com/u/2196373?v=4" width="100px;"/><br/><sub><b>dickeylth</b></sub>](https://github.com/dickeylth)<br/>|[<img src="https://avatars.githubusercontent.com/u/13050025?v=4" width="100px;"/><br/><sub><b>aladdin-add</b></sub>](https://github.com/aladdin-add)<br/>
+|[<img src="https://avatars.githubusercontent.com/u/309219?v=4" width="100px;"/><br/><sub><b>chadxz</b></sub>](https://github.com/chadxz)<br/>|[<img src="https://avatars.githubusercontent.com/u/2055702?v=4" width="100px;"/><br/><sub><b>adapt0</b></sub>](https://github.com/adapt0)<br/>|[<img src="https://avatars.githubusercontent.com/u/5139554?v=4" width="100px;"/><br/><sub><b>danielwpz</b></sub>](https://github.com/danielwpz)<br/>|[<img src="https://avatars.githubusercontent.com/u/5127897?v=4" width="100px;"/><br/><sub><b>danielsss</b></sub>](https://github.com/danielsss)<br/>|[<img src="https://avatars.githubusercontent.com/u/3367820?v=4" width="100px;"/><br/><sub><b>Jeff-Tian</b></sub>](https://github.com/Jeff-Tian)<br/>|[<img src="https://avatars.githubusercontent.com/u/32407?v=4" width="100px;"/><br/><sub><b>jedahan</b></sub>](https://github.com/jedahan)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/17075261?v=4" width="100px;"/><br/><sub><b>nick-ng</b></sub>](https://github.com/nick-ng)<br/>|[<img src="https://avatars.githubusercontent.com/u/1706595?v=4" width="100px;"/><br/><sub><b>rishavsharan</b></sub>](https://github.com/rishavsharan)<br/>|[<img src="https://avatars.githubusercontent.com/u/1886161?v=4" width="100px;"/><br/><sub><b>willizm</b></sub>](https://github.com/willizm)<br/>|[<img src="https://avatars.githubusercontent.com/u/7227589?v=4" width="100px;"/><br/><sub><b>davidkhala</b></sub>](https://github.com/davidkhala)<br/>|[<img src="https://avatars.githubusercontent.com/u/535479?v=4" width="100px;"/><br/><sub><b>aleafs</b></sub>](https://github.com/aleafs)<br/>|[<img src="https://avatars.githubusercontent.com/u/3689968?v=4" width="100px;"/><br/><sub><b>Amunu</b></sub>](https://github.com/Amunu)<br/>|
+|[<img src="https://avatars.githubusercontent.com/in/9426?v=4" width="100px;"/><br/><sub><b>azure-pipelines[bot]</b></sub>](https://github.com/apps/azure-pipelines)<br/>|[<img src="https://avatars.githubusercontent.com/u/1281323?v=4" width="100px;"/><br/><sub><b>changzhiwin</b></sub>](https://github.com/changzhiwin)<br/>|[<img src="https://avatars.githubusercontent.com/u/929503?v=4" width="100px;"/><br/><sub><b>yuzhigang33</b></sub>](https://github.com/yuzhigang33)<br/>|[<img src="https://avatars.githubusercontent.com/u/981128?v=4" width="100px;"/><br/><sub><b>fishbar</b></sub>](https://github.com/fishbar)<br/>|[<img src="https://avatars.githubusercontent.com/u/1207064?v=4" width="100px;"/><br/><sub><b>gxcsoccer</b></sub>](https://github.com/gxcsoccer)<br/>|[<img src="https://avatars.githubusercontent.com/u/17476119?v=4" width="100px;"/><br/><sub><b>mars-coder</b></sub>](https://github.com/mars-coder)<br/>|
+[<img src="https://avatars.githubusercontent.com/u/929179?v=4" width="100px;"/><br/><sub><b>rockdai</b></sub>](https://github.com/rockdai)<br/>|[<img src="https://avatars.githubusercontent.com/u/2196373?v=4" width="100px;"/><br/><sub><b>dickeylth</b></sub>](https://github.com/dickeylth)<br/>|[<img src="https://avatars.githubusercontent.com/u/13050025?v=4" width="100px;"/><br/><sub><b>aladdin-add</b></sub>](https://github.com/aladdin-add)<br/>
 
-This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Tue Jul 05 2022 16:17:31 GMT+0800`.
+This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Fri Sep 09 2022 21:45:49 GMT+0800`.
 
 <!-- GITCONTRIBUTOR_END -->
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "3.1.2",
+  "version": "3.2.0",
   "publishConfig": {
     "tag": "latest"
   },
@@ -49,8 +49,9 @@
     "build:cjs:test": "cd test/cjs && rm -rf node_modules && npm link ../.. && node index.js",
     "build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
     "build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test",
-    "test": "tsc --version && jest --coverage",
-    "ci": "npm run lint && npm run test && npm run build:test",
+    "test": "vitest run",
+    "cov": "vitest run --coverage --no-threads",
+    "ci": "npm run lint && npm run cov && npm run build:test",
     "contributor": "git-contributor",
     "prepack": "npm run build && rm -rf src/*.tsbuildinfo"
   },
@@ -67,10 +68,10 @@
   "devDependencies": {
     "@types/busboy": "^1.5.0",
     "@types/default-user-agent": "^1.0.0",
-    "@types/jest": "28",
     "@types/mime-types": "^2.1.1",
     "@types/pump": "^1.1.1",
     "@types/selfsigned": "^2.0.1",
+    "@vitest/coverage-c8": "^0.23.4",
     "busboy": "^1.6.0",
     "coffee": "5",
     "egg-ci": "2",
@@ -78,14 +79,12 @@
     "eslint-config-egg": "^12.0.0",
     "git-contributor": "1",
     "iconv-lite": "^0.6.3",
-    "jest": "28",
-    "jest-summary-reporter": "^0.0.2",
     "selfsigned": "^2.0.1",
-    "ts-jest": "28",
-    "typescript": "4"
+    "typescript": "^4.8.3",
+    "vitest": "^0.23.4"
   },
   "engines": {
-    "node": ">= 14.0.0"
+    "node": ">= 14.17.0"
   },
   "ci": {
     "version": "14, 16, 18"

package/src/HttpClient.ts

@@ -26,8 +26,10 @@ import mime from 'mime-types';
 import pump from 'pump';
 import { HttpAgent, CheckAddressFunction } from './HttpAgent';
 import { RequestURL, RequestOptions, HttpMethod } from './Request';
-import { HttpClientResponseMeta, HttpClientResponse, ReadableWithMeta } from './Response';
-import { parseJSON, sleep, digestAuthHeader } from './utils';
+import { HttpClientResponseMeta, HttpClientResponse, ReadableWithMeta, BaseResponseMeta, SocketInfo } from './Response';
+import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime } from './utils';
+import symbols from './symbols';
+import { initDiagnosticsChannel } from './diagnosticsChannel';
 
 const FormData = FormDataNative ?? FormDataNode;
 // impl isReadable on Node.js 14
@@ -48,10 +50,7 @@ function noop() {
   // noop
 }
 
-const MAX_REQURE_ID_VALUE = Math.pow(2, 31) - 10;
-let globalRequestId = 0;
-
-const debug = debuglog('urllib');
+const debug = debuglog('urllib:HttpClient');
 
 export type ClientOptions = {
   defaultArgs?: RequestOptions;
@@ -127,10 +126,6 @@ function defaultIsRetry(response: HttpClientResponse) {
   return response.status >= 500;
 }
 
-function performanceTime(startTime: number) {
-  return Math.floor((performance.now() - startTime) * 1000) / 1000;
-}
-
 type RequestContext = {
   retries: number;
 };
@@ -149,6 +144,7 @@ export class HttpClient extends EventEmitter {
         connect: clientOptions.connect,
       });
     }
+    initDiagnosticsChannel();
   }
 
   async request(url: RequestURL, options?: RequestOptions) {
@@ -156,11 +152,7 @@ export class HttpClient extends EventEmitter {
   }
 
   async #requestInternal(url: RequestURL, options?: RequestOptions, requestContext?: RequestContext): Promise<HttpClientResponse> {
-    if (globalRequestId >= MAX_REQURE_ID_VALUE) {
-      globalRequestId = 0;
-    }
-    const requestId = ++globalRequestId;
-
+    const requestId = globalId('HttpClientRequest');
     const requestUrl = typeof url === 'string' ? new URL(url) : url;
     const args = {
       retry: 0,
@@ -173,6 +165,32 @@ export class HttpClient extends EventEmitter {
     };
     const requestStartTime = performance.now();
 
+    // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
+    const timing = {
+      // socket assigned
+      queuing: 0,
+      // dns lookup time
+      // dnslookup: 0,
+      // socket connected
+      connected: 0,
+      // request headers sent
+      requestHeadersSent: 0,
+      // request sent, including headers and body
+      requestSent: 0,
+      // Time to first byte (TTFB), the response headers have been received
+      waiting: 0,
+      // the response body and trailers have been received
+      contentDownload: 0,
+    };
+    const orginalOpaque = args.opaque;
+    // using opaque to diagnostics channel, binding request and socket
+    const internalOpaque = {
+      [symbols.kRequestId]: requestId,
+      [symbols.kRequestStartTime]: requestStartTime,
+      [symbols.kEnableRequestTiming]: !!args.timing,
+      [symbols.kRequestTiming]: timing,
+      [symbols.kRequestOrginalOpaque]: orginalOpaque,
+    };
     const reqMeta = {
       requestId,
       url: requestUrl.href,
@@ -180,6 +198,18 @@ export class HttpClient extends EventEmitter {
       ctx: args.ctx,
       retries: requestContext.retries,
     };
+    const socketInfo = {
+      id: 0,
+      localAddress: '',
+      localPort: 0,
+      remoteAddress: '',
+      remotePort: 0,
+      remoteFamily: '',
+      bytesWritten: 0,
+      bytesRead: 0,
+      handledRequests: 0,
+      handledResponses: 0,
+    };
     // keep urllib createCallbackResponse style
     const resHeaders: IncomingHttpHeaders = {};
     const res: HttpClientResponseMeta = {
@@ -191,10 +221,8 @@ export class HttpClient extends EventEmitter {
       rt: 0,
       keepAliveSocket: true,
       requestUrls: [],
-      timing: {
-        waiting: 0,
-        contentDownload: 0,
-      },
+      timing,
+      socket: socketInfo,
     };
 
     let headersTimeout = 5000;
@@ -245,7 +273,6 @@ export class HttpClient extends EventEmitter {
       headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
     }
 
-    let opaque = args.opaque;
     try {
       const requestOptions: UndiciRquestOptions = {
         method,
@@ -253,7 +280,7 @@ export class HttpClient extends EventEmitter {
         maxRedirections: args.maxRedirects ?? 10,
         headersTimeout,
         bodyTimeout,
-        opaque,
+        opaque: internalOpaque,
         dispatcher: this.#dispatcher,
       };
       if (args.followRedirect === false) {
@@ -373,7 +400,7 @@ export class HttpClient extends EventEmitter {
         if (authenticate.startsWith('Digest ')) {
           debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
           requestOptions.headers.authorization = digestAuthHeader(requestOptions.method!,
-            requestUrl.pathname, authenticate, args.digestAuth);
+            `${requestUrl.pathname}${requestUrl.search}`, authenticate, args.digestAuth);
           debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
           if (response.headers['set-cookie']) {
             // FIXME: merge exists cookie header
@@ -383,11 +410,6 @@ export class HttpClient extends EventEmitter {
         }
       }
 
-      opaque = response.opaque;
-      if (args.timing) {
-        res.timing.waiting = performanceTime(requestStartTime);
-      }
-
       const context = response.context as { history: URL[] };
       let lastUrl = '';
       if (context?.history) {
@@ -413,10 +435,12 @@ export class HttpClient extends EventEmitter {
       if (args.dataType === 'stream') {
         // streaming mode will disable retry
         args.retry = 0;
-        const meta = {
+        const meta: BaseResponseMeta = {
           status: res.status,
           statusCode: res.statusCode,
           headers: res.headers,
+          timing,
+          socket: socketInfo,
         };
         if (isCompressedContent) {
           // gzip or br
@@ -458,12 +482,11 @@ export class HttpClient extends EventEmitter {
         }
       }
       res.rt = performanceTime(requestStartTime);
-      if (args.timing) {
-        res.timing.contentDownload = res.rt;
-      }
+      // get real socket info from internalOpaque
+      this.#updateSocketInfo(socketInfo, internalOpaque);
 
       const clientResponse: HttpClientResponse = {
-        opaque,
+        opaque: orginalOpaque,
         data,
         status: res.status,
         statusCode: res.status,
@@ -507,7 +530,7 @@ export class HttpClient extends EventEmitter {
       } else if (err.name === 'BodyTimeoutError') {
         err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
       }
-      err.opaque = opaque;
+      err.opaque = orginalOpaque;
       err.status = res.status;
       err.headers = res.headers;
       err.res = res;
@@ -516,9 +539,7 @@ export class HttpClient extends EventEmitter {
         res.requestUrls.push(requestUrl.href);
       }
       res.rt = performanceTime(requestStartTime);
-      if (args.timing) {
-        res.timing.contentDownload = res.rt;
-      }
+      this.#updateSocketInfo(socketInfo, internalOpaque);
 
       if (this.listenerCount('response') > 0) {
         this.emit('response', {
@@ -532,4 +553,20 @@ export class HttpClient extends EventEmitter {
       throw err;
     }
   }
+
+  #updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any) {
+    const socket = internalOpaque[symbols.kRequestSocket];
+    if (socket) {
+      socketInfo.id = socket[symbols.kSocketId];
+      socketInfo.handledRequests = socket[symbols.kHandledRequests];
+      socketInfo.handledResponses = socket[symbols.kHandledResponses];
+      socketInfo.localAddress = socket.localAddress;
+      socketInfo.localPort = socket.localPort;
+      socketInfo.remoteAddress = socket.remoteAddress;
+      socketInfo.remotePort = socket.remotePort;
+      socketInfo.remoteFamily = socket.remoteFamily;
+      socketInfo.bytesRead = socket.bytesRead;
+      socketInfo.bytesWritten = socket.bytesWritten;
+    }
+  }
 }

package/src/Request.ts

@@ -96,7 +96,6 @@ export type RequestOptions = {
    * */
   gzip?: boolean;
   /**
-   * @deprecated
    * Enable timing or not, default is false.
    * */
   timing?: boolean;

package/src/Response.ts

@@ -1,37 +1,61 @@
 import { Readable } from 'stream';
 import { IncomingHttpHeaders } from 'http';
 
-export type HttpClientResponseMeta = {
+export type SocketInfo = {
+  id: number;
+  localAddress: string;
+  localPort: number;
+  remoteAddress: string;
+  remotePort: number;
+  remoteFamily: string;
+  bytesWritten: number;
+  bytesRead: number;
+  handledRequests: number;
+  handledResponses: number;
+};
+
+/**
+ * https://eggjs.org/en/core/httpclient.html#timing-boolean
+ */
+export type Timing = {
+  // socket assigned
+  queuing: number;
+  // dns lookup time
+  // dnslookup: number;
+  // socket connected
+  connected: number;
+  // request headers sent
+  requestHeadersSent: number;
+  // request sent, including headers and body
+  requestSent: number;
+  // Time to first byte (TTFB), the response headers have been received
+  waiting: number;
+  // the response body and trailers have been received
+  contentDownload: number;
+};
+
+export type BaseResponseMeta = {
   status: number;
   statusCode: number;
   headers: IncomingHttpHeaders;
+  timing: Timing;
+  // SocketInfo
+  socket: SocketInfo;
+};
+
+export type HttpClientResponseMeta = BaseResponseMeta & {
   size: number;
   aborted: boolean;
   rt: number;
   keepAliveSocket: boolean;
   requestUrls: string[];
-  /**
-   * https://eggjs.org/en/core/httpclient.html#timing-boolean
-   */
-  timing: {
-    contentDownload: number;
-    waiting: number;
-  };
-  // remoteAddress: remoteAddress,
-  // remotePort: remotePort,
-  // socketHandledRequests: socketHandledRequests,
-  // socketHandledResponses: socketHandledResponses,
 };
 
-export type ReadableWithMeta = Readable & {
-  status: number;
-  statusCode: number;
-  headers: IncomingHttpHeaders;
-};
+export type ReadableWithMeta = Readable & BaseResponseMeta;
 
 export type HttpClientResponse = {
   opaque: unknown;
-  data: any
+  data: any;
   status: number;
   // alias to status, keep compatibility
   statusCode: number;

package/src/cjs/HttpClient.js

@@ -1,6 +1,6 @@
 "use strict";
 var _a, _b, _c;
-var _BlobFromStream_stream, _BlobFromStream_type, _HttpClient_instances, _HttpClient_defaultArgs, _HttpClient_dispatcher, _HttpClient_requestInternal;
+var _BlobFromStream_stream, _BlobFromStream_type, _HttpClient_instances, _HttpClient_defaultArgs, _HttpClient_dispatcher, _HttpClient_requestInternal, _HttpClient_updateSocketInfo;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HttpClient = exports.HEADER_USER_AGENT = void 0;
 const tslib_1 = require("tslib");
@@ -21,6 +21,8 @@ const mime_types_1 = tslib_1.__importDefault(require("mime-types"));
 const pump_1 = tslib_1.__importDefault(require("pump"));
 const HttpAgent_1 = require("./HttpAgent");
 const utils_1 = require("./utils");
+const symbols_1 = tslib_1.__importDefault(require("./symbols"));
+const diagnosticsChannel_1 = require("./diagnosticsChannel");
 const FormData = undici_1.FormData !== null && undici_1.FormData !== void 0 ? undici_1.FormData : formdata_node_1.FormData;
 // impl isReadable on Node.js 14
 const isReadable = (_a = stream_2.default.isReadable) !== null && _a !== void 0 ? _a : function isReadable(stream) {
@@ -39,9 +41,7 @@ const pipelinePromise = (_c = (_b = stream_2.default.promises) === null || _b ==
 function noop() {
     // noop
 }
-const MAX_REQURE_ID_VALUE = Math.pow(2, 31) - 10;
-let globalRequestId = 0;
-const debug = (0, util_1.debuglog)('urllib');
+const debug = (0, util_1.debuglog)('urllib:HttpClient');
 // https://github.com/octet-stream/form-data
 class BlobFromStream {
     constructor(stream, type) {
@@ -79,9 +79,6 @@ function getFileName(stream) {
 function defaultIsRetry(response) {
     return response.status >= 500;
 }
-function performanceTime(startTime) {
-    return Math.floor((perf_hooks_1.performance.now() - startTime) * 1000) / 1000;
-}
 class HttpClient extends events_1.EventEmitter {
     constructor(clientOptions) {
         super();
@@ -96,6 +93,7 @@ class HttpClient extends events_1.EventEmitter {
                 connect: clientOptions.connect,
             }), "f");
         }
+        (0, diagnosticsChannel_1.initDiagnosticsChannel)();
     }
     async request(url, options) {
         return await tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_requestInternal).call(this, url, options);
@@ -104,10 +102,7 @@ class HttpClient extends events_1.EventEmitter {
 exports.HttpClient = HttpClient;
 _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(), _HttpClient_instances = new WeakSet(), _HttpClient_requestInternal = async function _HttpClient_requestInternal(url, options, requestContext) {
     var _a, _b, _c, _d, _e, _f;
-    if (globalRequestId >= MAX_REQURE_ID_VALUE) {
-        globalRequestId = 0;
-    }
-    const requestId = ++globalRequestId;
+    const requestId = (0, utils_1.globalId)('HttpClientRequest');
     const requestUrl = typeof url === 'string' ? new URL(url) : url;
     const args = {
         retry: 0,
@@ -119,6 +114,32 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
         ...requestContext,
     };
     const requestStartTime = perf_hooks_1.performance.now();
+    // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
+    const timing = {
+        // socket assigned
+        queuing: 0,
+        // dns lookup time
+        // dnslookup: 0,
+        // socket connected
+        connected: 0,
+        // request headers sent
+        requestHeadersSent: 0,
+        // request sent, including headers and body
+        requestSent: 0,
+        // Time to first byte (TTFB), the response headers have been received
+        waiting: 0,
+        // the response body and trailers have been received
+        contentDownload: 0,
+    };
+    const orginalOpaque = args.opaque;
+    // using opaque to diagnostics channel, binding request and socket
+    const internalOpaque = {
+        [symbols_1.default.kRequestId]: requestId,
+        [symbols_1.default.kRequestStartTime]: requestStartTime,
+        [symbols_1.default.kEnableRequestTiming]: !!args.timing,
+        [symbols_1.default.kRequestTiming]: timing,
+        [symbols_1.default.kRequestOrginalOpaque]: orginalOpaque,
+    };
     const reqMeta = {
         requestId,
         url: requestUrl.href,
@@ -126,6 +147,18 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
         ctx: args.ctx,
         retries: requestContext.retries,
     };
+    const socketInfo = {
+        id: 0,
+        localAddress: '',
+        localPort: 0,
+        remoteAddress: '',
+        remotePort: 0,
+        remoteFamily: '',
+        bytesWritten: 0,
+        bytesRead: 0,
+        handledRequests: 0,
+        handledResponses: 0,
+    };
     // keep urllib createCallbackResponse style
     const resHeaders = {};
     const res = {
@@ -137,10 +170,8 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
         rt: 0,
         keepAliveSocket: true,
         requestUrls: [],
-        timing: {
-            waiting: 0,
-            contentDownload: 0,
-        },
+        timing,
+        socket: socketInfo,
     };
     let headersTimeout = 5000;
     let bodyTimeout = 5000;
@@ -190,7 +221,6 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
     if (args.auth && !headers.authorization) {
         headers.authorization = `Basic ${Buffer.from(args.auth).toString('base64')}`;
     }
-    let opaque = args.opaque;
     try {
         const requestOptions = {
             method,
@@ -198,7 +228,7 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
             maxRedirections: (_d = args.maxRedirects) !== null && _d !== void 0 ? _d : 10,
             headersTimeout,
             bodyTimeout,
-            opaque,
+            opaque: internalOpaque,
             dispatcher: tslib_1.__classPrivateFieldGet(this, _HttpClient_dispatcher, "f"),
         };
         if (args.followRedirect === false) {
@@ -323,7 +353,7 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
             const authenticate = response.headers['www-authenticate'];
             if (authenticate.startsWith('Digest ')) {
                 debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
-                requestOptions.headers.authorization = (0, utils_1.digestAuthHeader)(requestOptions.method, requestUrl.pathname, authenticate, args.digestAuth);
+                requestOptions.headers.authorization = (0, utils_1.digestAuthHeader)(requestOptions.method, `${requestUrl.pathname}${requestUrl.search}`, authenticate, args.digestAuth);
                 debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
                 if (response.headers['set-cookie']) {
                     // FIXME: merge exists cookie header
@@ -332,10 +362,6 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
                 response = await (0, undici_1.request)(requestUrl, requestOptions);
             }
         }
-        opaque = response.opaque;
-        if (args.timing) {
-            res.timing.waiting = performanceTime(requestStartTime);
-        }
         const context = response.context;
         let lastUrl = '';
         if (context === null || context === void 0 ? void 0 : context.history) {
@@ -364,6 +390,8 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
                 status: res.status,
                 statusCode: res.statusCode,
                 headers: res.headers,
+                timing,
+                socket: socketInfo,
             };
             if (isCompressedContent) {
                 // gzip or br
@@ -411,12 +439,11 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
                 }
             }
         }
-        res.rt = performanceTime(requestStartTime);
-        if (args.timing) {
-            res.timing.contentDownload = res.rt;
-        }
+        res.rt = (0, utils_1.performanceTime)(requestStartTime);
+        // get real socket info from internalOpaque
+        tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_updateSocketInfo).call(this, socketInfo, internalOpaque);
         const clientResponse = {
-            opaque,
+            opaque: orginalOpaque,
             data,
             status: res.status,
             statusCode: res.status,
@@ -459,7 +486,7 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
         else if (err.name === 'BodyTimeoutError') {
             err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
         }
-        err.opaque = opaque;
+        err.opaque = orginalOpaque;
         err.status = res.status;
         err.headers = res.headers;
         err.res = res;
@@ -467,10 +494,8 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
         if (res.requestUrls.length === 0) {
             res.requestUrls.push(requestUrl.href);
         }
-        res.rt = performanceTime(requestStartTime);
-        if (args.timing) {
-            res.timing.contentDownload = res.rt;
-        }
+        res.rt = (0, utils_1.performanceTime)(requestStartTime);
+        tslib_1.__classPrivateFieldGet(this, _HttpClient_instances, "m", _HttpClient_updateSocketInfo).call(this, socketInfo, internalOpaque);
         if (this.listenerCount('response') > 0) {
             this.emit('response', {
                 requestId,
@@ -482,5 +507,19 @@ _HttpClient_defaultArgs = new WeakMap(), _HttpClient_dispatcher = new WeakMap(),
         }
         throw err;
     }
+}, _HttpClient_updateSocketInfo = function _HttpClient_updateSocketInfo(socketInfo, internalOpaque) {
+    const socket = internalOpaque[symbols_1.default.kRequestSocket];
+    if (socket) {
+        socketInfo.id = socket[symbols_1.default.kSocketId];
+        socketInfo.handledRequests = socket[symbols_1.default.kHandledRequests];
+        socketInfo.handledResponses = socket[symbols_1.default.kHandledResponses];
+        socketInfo.localAddress = socket.localAddress;
+        socketInfo.localPort = socket.localPort;
+        socketInfo.remoteAddress = socket.remoteAddress;
+        socketInfo.remotePort = socket.remotePort;
+        socketInfo.remoteFamily = socket.remoteFamily;
+        socketInfo.bytesRead = socket.bytesRead;
+        socketInfo.bytesWritten = socket.bytesWritten;
+    }
 };
 //# sourceMappingURL=HttpClient.js.map