urllib 3.26.0 → 4.1.0

package/src/HttpClient.ts

@@ -11,34 +11,33 @@ import {
 } from 'node:zlib';
 import { Blob } from 'node:buffer';
 import { Readable, pipeline } from 'node:stream';
-import stream from 'node:stream';
+import { pipeline as pipelinePromise } from 'node:stream/promises';
 import { basename } from 'node:path';
 import { createReadStream } from 'node:fs';
 import { format as urlFormat } from 'node:url';
 import { performance } from 'node:perf_hooks';
 import querystring from 'node:querystring';
+import { setTimeout as sleep } from 'node:timers/promises';
 import {
-  FormData as FormDataNative,
+  FormData,
   request as undiciRequest,
   Dispatcher,
   Agent,
   getGlobalDispatcher,
   Pool,
 } from 'undici';
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
 import undiciSymbols from 'undici/lib/core/symbols.js';
-import { FormData as FormDataNode } from 'formdata-node';
-import { FormDataEncoder } from 'form-data-encoder';
-import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
 import qs from 'qs';
-import pump from 'pump';
 // Compatible with old style formstream
 import FormStream from 'formstream';
 import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
 import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
 import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
-import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
+import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
 import symbols from './symbols.js';
 import { initDiagnosticsChannel } from './diagnosticsChannel.js';
 import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
@@ -49,24 +48,12 @@ type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
 type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;
 
 const PROTO_RE = /^https?:\/\//i;
-const FormData = FormDataNative ?? FormDataNode;
-// impl promise pipeline on Node.js 14
-const pipelinePromise = stream.promises?.pipeline ?? function pipeline(...args: any[]) {
-  return new Promise<void>((resolve, reject) => {
-    pump(...args, (err?: Error) => {
-      if (err) return reject(err);
-      resolve();
-    });
-  });
-};
 
 function noop() {
   // noop
 }
 
 const debug = debuglog('urllib:HttpClient');
-// Node.js 14 or 16
-const isNode14Or16 = /v1[46]\./.test(process.version);
 
 export type ClientOptions = {
   defaultArgs?: RequestOptions;
@@ -127,7 +114,10 @@ class BlobFromStream {
   }
 }
 
-export const HEADER_USER_AGENT = createUserAgent('node-urllib', 'VERSION');
+export const VERSION = 'VERSION';
+// 'node-urllib/4.0.0 Node.js/18.19.0 (darwin; x64)'
+export const HEADER_USER_AGENT =
+  `node-urllib/${VERSION} Node.js/${process.version.substring(1)} (${process.platform}; ${process.arch})`;
 
 function getFileName(stream: Readable) {
   const filePath: string = (stream as any).path;
@@ -216,13 +206,13 @@ export class HttpClient extends EventEmitter {
   getDispatcherPoolStats() {
     const agent = this.getDispatcher();
     // origin => Pool Instance
-    const clients: Map<string, WeakRef<Pool>> | undefined = agent[undiciSymbols.kClients];
+    const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
     const poolStatsMap: Record<string, PoolStat> = {};
     if (!clients) {
       return poolStatsMap;
     }
     for (const [ key, ref ] of clients) {
-      const pool = ref.deref();
+      const pool = typeof ref.deref === 'function' ? ref.deref() : ref as unknown as Pool;
       const stats = pool?.stats;
       if (!stats) continue;
       poolStatsMap[key] = {
@@ -460,9 +450,11 @@ export class HttpClient extends EventEmitter {
         } else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
           uploadFiles.push([ 'file', args.files ]);
         } else if (typeof args.files === 'object') {
-          for (const field in args.files) {
+          const files = args.files as Record<string, string | Readable | Buffer>;
+          for (const field in files) {
             // set custom fileName
-            uploadFiles.push([ field, args.files[field], field ]);
+            const file = files[field];
+            uploadFiles.push([ field, file, field ]);
           }
         }
         // set normal fields first
@@ -487,18 +479,7 @@ export class HttpClient extends EventEmitter {
             isStreamingRequest = true;
           }
         }
-
-        if (FormDataNative) {
-          requestOptions.body = formData;
-        } else {
-          // Node.js 14 does not support spec-compliant FormData
-          // https://github.com/octet-stream/form-data#usage
-          const encoder = new FormDataEncoder(formData as any);
-          Object.assign(headers, encoder.headers);
-          // fix "Content-Length":"NaN"
-          delete headers['Content-Length'];
-          requestOptions.body = Readable.from(encoder);
-        }
+        requestOptions.body = formData;
       } else if (args.content) {
         if (!isGETOrHEAD) {
           // handle content
@@ -516,7 +497,7 @@ export class HttpClient extends EventEmitter {
           || isReadable(args.data);
         if (isGETOrHEAD) {
           if (!isStringOrBufferOrReadable) {
-            let query;
+            let query: string;
             if (args.nestedQuerystring) {
               query = qs.stringify(args.data);
             } else {
@@ -617,9 +598,6 @@ export class HttpClient extends EventEmitter {
           res = Object.assign(response.body, res);
         }
       } else if (args.writeStream) {
-        if (isNode14Or16 && args.writeStream.destroyed) {
-          throw new Error('writeStream is destroyed');
-        }
         if (args.compressed === true && isCompressedContent) {
           const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
           await pipelinePromise(response.body, decoder, args.writeStream);

package/src/diagnosticsChannel.ts

@@ -20,15 +20,14 @@ let initedDiagnosticsChannel = false;
 //   -> undici:request:trailers => { request, trailers }
 
 function subscribe(name: string, listener: (message: unknown, channelName: string | symbol) => void) {
-  if (typeof diagnosticsChannel.subscribe === 'function') {
-    diagnosticsChannel.subscribe(name, listener);
-  } else {
-    // TODO: support Node.js 14, will be removed on the next major version
-    diagnosticsChannel.channel(name).subscribe(listener);
-  }
+  diagnosticsChannel.subscribe(name, listener);
 }
 
-function formatSocket(socket: Socket) {
+type SocketExtend = Socket & {
+  [key: symbol]: string | number | Date | undefined;
+};
+
+function formatSocket(socket: SocketExtend) {
   if (!socket) return socket;
   return {
     localAddress: socket[symbols.kSocketLocalAddress],
@@ -41,8 +40,7 @@ function formatSocket(socket: Socket) {
 }
 
 // make sure error contains socket info
-const kDestroy = Symbol('kDestroy');
-Socket.prototype[kDestroy] = Socket.prototype.destroy;
+const destroySocket = Socket.prototype.destroy;
 Socket.prototype.destroy = function(err?: any) {
   if (err) {
     Object.defineProperty(err, symbols.kErrorSocket, {
@@ -51,12 +49,12 @@ Socket.prototype.destroy = function(err?: any) {
       value: this,
     });
   }
-  return this[kDestroy](err);
+  return destroySocket.call(this, err);
 };
 
 function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol) {
   if (!kHandler) return;
-  const handler = request[kHandler];
+  const handler = Reflect.get(request, kHandler);
   // maxRedirects = 0 will get [Symbol(handler)]: RequestHandler {
   // responseHeaders: null,
   // opaque: {
@@ -70,7 +68,7 @@ function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol
 }
 
 export function initDiagnosticsChannel() {
-  // makre sure init global DiagnosticsChannel once
+  // make sure init global DiagnosticsChannel once
   if (initedDiagnosticsChannel) return;
   initedDiagnosticsChannel = true;
 
@@ -97,29 +95,27 @@ export function initDiagnosticsChannel() {
     opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
   });
 
-  // diagnosticsChannel.channel('undici:client:beforeConnect')
-
   subscribe('undici:client:connectError', (message, name) => {
-    const { error, connectParams } = message as DiagnosticsChannel.ClientConnectErrorMessage & { error: any };
-    let { socket } = message as DiagnosticsChannel.ClientConnectErrorMessage;
-    if (!socket && error[symbols.kErrorSocket]) {
-      socket = error[symbols.kErrorSocket];
+    const { error, connectParams, socket } = message as DiagnosticsChannel.ClientConnectErrorMessage & { error: any, socket: SocketExtend };
+    let sock = socket;
+    if (!sock && error[symbols.kErrorSocket]) {
+      sock = error[symbols.kErrorSocket];
     }
-    if (socket) {
-      socket[symbols.kSocketId] = globalId('UndiciSocket');
-      socket[symbols.kSocketConnectErrorTime] = new Date();
-      socket[symbols.kHandledRequests] = 0;
-      socket[symbols.kHandledResponses] = 0;
+    if (sock) {
+      sock[symbols.kSocketId] = globalId('UndiciSocket');
+      sock[symbols.kSocketConnectErrorTime] = new Date();
+      sock[symbols.kHandledRequests] = 0;
+      sock[symbols.kHandledResponses] = 0;
       // copy local address to symbol, avoid them be reset after request error throw
-      if (socket.localAddress) {
-        socket[symbols.kSocketLocalAddress] = socket.localAddress;
-        socket[symbols.kSocketLocalPort] = socket.localPort;
+      if (sock.localAddress) {
+        sock[symbols.kSocketLocalAddress] = sock.localAddress;
+        sock[symbols.kSocketLocalPort] = sock.localPort;
       }
-      socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
-      socket[symbols.kSocketConnectHost] = connectParams.host;
-      socket[symbols.kSocketConnectPort] = connectParams.port;
+      sock[symbols.kSocketConnectProtocol] = connectParams.protocol;
+      sock[symbols.kSocketConnectHost] = connectParams.host;
+      sock[symbols.kSocketConnectPort] = connectParams.port;
       debug('[%s] Socket#%d connectError, connectParams: %o, error: %s, (sock: %o)',
-        name, socket[symbols.kSocketId], connectParams, (error as Error).message, formatSocket(socket));
+        name, sock[symbols.kSocketId], connectParams, (error as Error).message, formatSocket(sock));
     } else {
       debug('[%s] connectError, connectParams: %o, error: %o',
         name, connectParams, error);
@@ -128,7 +124,7 @@ export function initDiagnosticsChannel() {
 
   // This message is published after a connection is established.
   subscribe('undici:client:connected', (message, name) => {
-    const { socket, connectParams } = message as DiagnosticsChannel.ClientConnectedMessage;
+    const { socket, connectParams } = message as DiagnosticsChannel.ClientConnectedMessage & { socket: SocketExtend };
     socket[symbols.kSocketId] = globalId('UndiciSocket');
     socket[symbols.kSocketStartTime] = performance.now();
     socket[symbols.kSocketConnectedTime] = new Date();
@@ -145,11 +141,11 @@ export function initDiagnosticsChannel() {
 
   // This message is published right before the first byte of the request is written to the socket.
   subscribe('undici:client:sendHeaders', (message, name) => {
-    const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage;
+    const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage & { socket: SocketExtend };
     const opaque = getRequestOpaque(request, kHandler);
     if (!opaque || !opaque[symbols.kRequestId]) return;
 
-    socket[symbols.kHandledRequests]++;
+    (socket[symbols.kHandledRequests] as number)++;
     // attach socket to opaque
     opaque[symbols.kRequestSocket] = socket;
     debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)',
@@ -158,11 +154,11 @@ export function initDiagnosticsChannel() {
 
     if (!opaque[symbols.kEnableRequestTiming]) return;
     opaque[symbols.kRequestTiming].requestHeadersSent = performanceTime(opaque[symbols.kRequestStartTime]);
-    // first socket need to caculate the connected time
+    // first socket need to calculate the connected time
     if (socket[symbols.kHandledRequests] === 1) {
       // kSocketStartTime - kRequestStartTime = connected time
       opaque[symbols.kRequestTiming].connected =
-        performanceTime(opaque[symbols.kRequestStartTime], socket[symbols.kSocketStartTime]);
+        performanceTime(opaque[symbols.kRequestStartTime], socket[symbols.kSocketStartTime] as number);
     }
   });
 

package/src/index.ts

@@ -1,4 +1,4 @@
-import LRU from 'ylru';
+import { LRU } from 'ylru';
 import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
 import { RequestOptions, RequestURL } from './Request.js';
 

package/src/utils.ts

@@ -3,7 +3,7 @@ import { Readable } from 'node:stream';
 import { performance } from 'node:perf_hooks';
 import type { FixJSONCtlChars } from './Request.js';
 
-const JSONCtlCharsMap = {
+const JSONCtlCharsMap: Record<string, string> = {
   '"': '\\"', // \u0022
   '\\': '\\\\', // \u005c
   '\b': '\\b', // \u0008
@@ -49,12 +49,6 @@ export function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars) {
   return data;
 }
 
-export function sleep(ms: number) {
-  return new Promise<void>(resolve => {
-    setTimeout(resolve, ms);
-  });
-}
-
 function md5(s: string) {
   const sum = createHash('md5');
   sum.update(s, 'utf8');