urllib 4.6.4 → 4.6.6

package/src/diagnosticsChannel.ts

@@ -89,7 +89,9 @@ export function initDiagnosticsChannel() {
     const opaque = getRequestOpaque(request, kHandler);
     // ignore non HttpClient Request
     if (!opaque || !opaque[symbols.kRequestId]) return;
-    debug('[%s] Request#%d %s %s, path: %s, headers: %o',
+
+    Reflect.set(request, symbols.kRequestInternalOpaque, opaque);
+    debug('[%s] Request#%d %s %s, path: %s, headers: %j',
       name, opaque[symbols.kRequestId], request.method, request.origin, request.path, request.headers);
     if (!opaque[symbols.kEnableRequestTiming]) return;
     opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
@@ -114,10 +116,10 @@ export function initDiagnosticsChannel() {
       sock[symbols.kSocketConnectProtocol] = connectParams.protocol;
       sock[symbols.kSocketConnectHost] = connectParams.host;
       sock[symbols.kSocketConnectPort] = connectParams.port;
-      debug('[%s] Socket#%d connectError, connectParams: %o, error: %s, (sock: %o)',
+      debug('[%s] Socket#%d connectError, connectParams: %j, error: %s, (sock: %j)',
         name, sock[symbols.kSocketId], connectParams, (error as Error).message, formatSocket(sock));
     } else {
-      debug('[%s] connectError, connectParams: %o, error: %o',
+      debug('[%s] connectError, connectParams: %j, error: %o',
         name, connectParams, error);
     }
   });
@@ -136,13 +138,13 @@ export function initDiagnosticsChannel() {
     socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
     socket[symbols.kSocketConnectHost] = connectParams.host;
     socket[symbols.kSocketConnectPort] = connectParams.port;
-    debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
+    debug('[%s] Socket#%d connected (sock: %j)', name, socket[symbols.kSocketId], formatSocket(socket));
   });
 
   // This message is published right before the first byte of the request is written to the socket.
   subscribe('undici:client:sendHeaders', (message, name) => {
     const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage & { socket: SocketExtend };
-    const opaque = getRequestOpaque(request, kHandler);
+    const opaque = Reflect.get(request, symbols.kRequestInternalOpaque);
     if (!opaque || !opaque[symbols.kRequestId]) {
       debug('[%s] opaque not found', name);
       return;
@@ -151,7 +153,7 @@ export function initDiagnosticsChannel() {
     (socket[symbols.kHandledRequests] as number)++;
     // attach socket to opaque
     opaque[symbols.kRequestSocket] = socket;
-    debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)',
+    debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %j)',
       name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests],
       formatSocket(socket));
 
@@ -167,7 +169,7 @@ export function initDiagnosticsChannel() {
 
   subscribe('undici:request:bodySent', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestBodySentMessage;
-    const opaque = getRequestOpaque(request, kHandler);
+    const opaque = Reflect.get(request, symbols.kRequestInternalOpaque);
     if (!opaque || !opaque[symbols.kRequestId]) {
       debug('[%s] opaque not found', name);
       return;
@@ -181,7 +183,7 @@ export function initDiagnosticsChannel() {
   // This message is published after the response headers have been received, i.e. the response has been completed.
   subscribe('undici:request:headers', (message, name) => {
     const { request, response } = message as DiagnosticsChannel.RequestHeadersMessage;
-    const opaque = getRequestOpaque(request, kHandler);
+    const opaque = Reflect.get(request, symbols.kRequestInternalOpaque);
     if (!opaque || !opaque[symbols.kRequestId]) {
       debug('[%s] opaque not found', name);
       return;
@@ -191,7 +193,7 @@ export function initDiagnosticsChannel() {
     const socket = opaque[symbols.kRequestSocket];
     if (socket) {
       socket[symbols.kHandledResponses]++;
-      debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)',
+      debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %j)',
         name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses],
         formatSocket(socket));
     } else {
@@ -206,7 +208,7 @@ export function initDiagnosticsChannel() {
   // This message is published after the response body and trailers have been received, i.e. the response has been completed.
   subscribe('undici:request:trailers', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestTrailersMessage;
-    const opaque = getRequestOpaque(request, kHandler);
+    const opaque = Reflect.get(request, symbols.kRequestInternalOpaque);
     if (!opaque || !opaque[symbols.kRequestId]) {
       debug('[%s] opaque not found', name);
       return;

package/src/fetch.ts

@@ -1,4 +1,5 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
+import { debuglog } from 'node:util';
 import {
   fetch as UndiciFetch,
   RequestInfo,
@@ -41,6 +42,8 @@ import { RawResponseWithMeta, SocketInfo } from './Response.js';
 import { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 import { BaseAgent, BaseAgentOptions } from './BaseAgent.js';
 
+const debug = debuglog('urllib:fetch');
+
 export interface UrllibRequestInit extends RequestInit {
   // default is true
   timing?: boolean;
@@ -137,7 +140,7 @@ export class FetchFactory {
       // socket assigned
       queuing: 0,
       // dns lookup time
-      // dnslookup: 0,
+      dnslookup: 0,
       // socket connected
       connected: 0,
       // request headers sent
@@ -218,8 +221,9 @@ export class FetchFactory {
         res = await UndiciFetch(input, init);
       });
     } catch (e: any) {
-      updateSocketInfo(socketInfo, internalOpaque /* , rawError */);
+      updateSocketInfo(socketInfo, internalOpaque, e);
       urllibResponse.rt = performanceTime(requestStartTime);
+      debug('Request#%d throw error: %s', requestId, e);
       channels.fetchResponse.publish({
         fetch: fetchMeta,
         error: e,
@@ -234,7 +238,7 @@ export class FetchFactory {
 
     // get undici internal response
     const state = getResponseState(res!);
-    updateSocketInfo(socketInfo, internalOpaque /* , rawError */);
+    updateSocketInfo(socketInfo, internalOpaque);
 
     urllibResponse.headers = convertHeader(res!.headers);
     urllibResponse.status = urllibResponse.statusCode = res!.status;
@@ -243,7 +247,8 @@ export class FetchFactory {
       urllibResponse.size = parseInt(urllibResponse.headers['content-length']);
     }
     urllibResponse.rt = performanceTime(requestStartTime);
-
+    debug('Request#%d got response, status: %s, headers: %j, timing: %j, socket: %j',
+      requestId, urllibResponse.status, urllibResponse.headers, timing, urllibResponse.socket);
     channels.fetchResponse.publish({
       fetch: fetchMeta,
       timingInfo: state.timingInfo,

package/src/index.ts

@@ -7,16 +7,62 @@ import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
 import { RequestOptions, RequestURL } from './Request.js';
 
 let httpClient: HttpClient;
+let allowH2HttpClient: HttpClient;
+let allowUnauthorizedHttpClient: HttpClient;
+let allowH2AndUnauthorizedHttpClient: HttpClient;
 const domainSocketHttpClients = new LRU(50);
 
-export function getDefaultHttpClient(): HttpClient {
+export function getDefaultHttpClient(rejectUnauthorized?: boolean, allowH2?: boolean): HttpClient {
+  if (rejectUnauthorized === false) {
+    if (allowH2) {
+      if (!allowH2AndUnauthorizedHttpClient) {
+        allowH2AndUnauthorizedHttpClient = new HttpClient({
+          allowH2,
+          connect: {
+            rejectUnauthorized,
+          },
+        });
+      }
+      return allowH2AndUnauthorizedHttpClient;
+    }
+
+    if (!allowUnauthorizedHttpClient) {
+      allowUnauthorizedHttpClient = new HttpClient({
+        connect: {
+          rejectUnauthorized,
+        },
+      });
+    }
+    return allowUnauthorizedHttpClient;
+  }
+
+  if (allowH2) {
+    if (!allowH2HttpClient) {
+      allowH2HttpClient = new HttpClient({
+        allowH2,
+      });
+    }
+    return allowH2HttpClient;
+  }
+
   if (!httpClient) {
     httpClient = new HttpClient();
   }
   return httpClient;
 }
 
-export async function request<T = any>(url: RequestURL, options?: RequestOptions) {
+interface UrllibRequestOptions extends RequestOptions {
+  /**
+   * If `true`, the server certificate is verified against the list of supplied CAs.
+   * An 'error' event is emitted if verification fails.
+   * Default: `true`
+   */
+  rejectUnauthorized?: boolean;
+  /** Allow to use HTTP2 first. Default is `false` */
+  allowH2?: boolean;
+}
+
+export async function request<T = any>(url: RequestURL, options?: UrllibRequestOptions) {
   if (options?.socketPath) {
     let domainSocketHttpclient = domainSocketHttpClients.get<HttpClient>(options.socketPath);
     if (!domainSocketHttpclient) {
@@ -28,7 +74,7 @@ export async function request<T = any>(url: RequestURL, options?: RequestOptions
     return await domainSocketHttpclient.request<T>(url, options);
   }
 
-  return await getDefaultHttpClient().request<T>(url, options);
+  return await getDefaultHttpClient(options?.rejectUnauthorized, options?.allowH2).request<T>(url, options);
 }
 
 // export curl method is keep compatible with urllib.curl()
@@ -36,7 +82,7 @@ export async function request<T = any>(url: RequestURL, options?: RequestOptions
 // import * as urllib from 'urllib';
 // urllib.curl(url);
 // ```
-export async function curl<T = any>(url: RequestURL, options?: RequestOptions) {
+export async function curl<T = any>(url: RequestURL, options?: UrllibRequestOptions) {
   return await request<T>(url, options);
 }
 

package/src/symbols.ts

@@ -17,5 +17,6 @@ export default {
   kEnableRequestTiming: Symbol('enable request timing or not'),
   kRequestTiming: Symbol('request timing'),
   kRequestOriginalOpaque: Symbol('request original opaque'),
+  kRequestInternalOpaque: Symbol('request internal opaque'),
   kErrorSocket: Symbol('socket of error'),
 };