urllib 3.25.0 → 4.0.0

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "urllib",
-  "version": "3.25.0",
+  "version": "4.0.0",
   "publishConfig": {
     "tag": "latest"
   },
-  "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Base undici fetch API.",
+  "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, timeout and more. Base undici API.",
   "keywords": [
     "urllib",
     "http",
@@ -39,42 +39,35 @@
     "test": "npm run lint && vitest run",
     "test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 180000 keep-alive-header.test.ts",
     "cov": "vitest run --coverage",
-    "preci": "node scripts/pre_test.js",
-    "ci": "npm run lint && npm run cov && node scripts/build_test.js",
-    "contributor": "git-contributor",
+    "ci": "npm run lint && npm run cov && npm run prepublishOnly && attw --pack",
     "clean": "rm -rf dist",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "default-user-agent": "^1.0.0",
-    "digest-header": "^1.0.0",
-    "form-data-encoder": "^1.7.2",
-    "formdata-node": "^4.3.3",
-    "formstream": "^1.1.1",
+    "formstream": "^1.5.1",
     "mime-types": "^2.1.35",
-    "pump": "^3.0.0",
-    "qs": "^6.11.2",
-    "type-fest": "^4.3.1",
-    "undici": "^5.28.2",
-    "ylru": "^1.3.2"
+    "qs": "^6.12.1",
+    "type-fest": "^4.20.1",
+    "undici": "^6.19.2",
+    "ylru": "^2.0.0"
   },
   "devDependencies": {
+    "@arethetypeswrong/cli": "^0.15.3",
+    "@eggjs/tsconfig": "^1.3.3",
     "@tsconfig/node18": "^18.2.1",
     "@tsconfig/strictest": "^2.0.2",
     "@types/busboy": "^1.5.0",
-    "@types/default-user-agent": "^1.0.0",
     "@types/mime-types": "^2.1.1",
     "@types/node": "^20.2.1",
-    "@types/pump": "^1.1.1",
+    "@types/proxy": "^1.0.4",
     "@types/qs": "^6.9.7",
     "@types/selfsigned": "^2.0.1",
     "@types/tar-stream": "^2.2.2",
-    "@vitest/coverage-v8": "^1.3.1",
+    "@vitest/coverage-v8": "^1.6.0",
     "busboy": "^1.6.0",
     "cross-env": "^7.0.3",
-    "eslint": "^8.25.0",
-    "eslint-config-egg": "^12.1.0",
-    "git-contributor": "^2.0.0",
+    "eslint": "8",
+    "eslint-config-egg": "13",
     "iconv-lite": "^0.6.3",
     "proxy": "^1.0.2",
     "selfsigned": "^2.0.1",
@@ -82,10 +75,10 @@
     "tshy": "^1.0.0",
     "tshy-after": "^1.0.0",
     "typescript": "^5.0.4",
-    "vitest": "^1.3.1"
+    "vitest": "^1.6.0"
   },
   "engines": {
-    "node": ">= 14.19.3"
+    "node": ">= 18.19.0"
   },
   "license": "MIT",
   "type": "module",
@@ -98,10 +91,12 @@
   "exports": {
     ".": {
       "import": {
+        "source": "./src/index.ts",
         "types": "./dist/esm/index.d.ts",
         "default": "./dist/esm/index.js"
       },
       "require": {
+        "source": "./src/index.ts",
         "types": "./dist/commonjs/index.d.ts",
         "default": "./dist/commonjs/index.js"
       }

package/src/HttpClient.ts

@@ -11,34 +11,33 @@ import {
 } from 'node:zlib';
 import { Blob } from 'node:buffer';
 import { Readable, pipeline } from 'node:stream';
-import stream from 'node:stream';
+import { pipeline as pipelinePromise } from 'node:stream/promises';
 import { basename } from 'node:path';
 import { createReadStream } from 'node:fs';
 import { format as urlFormat } from 'node:url';
 import { performance } from 'node:perf_hooks';
 import querystring from 'node:querystring';
+import { setTimeout as sleep } from 'node:timers/promises';
 import {
-  FormData as FormDataNative,
+  FormData,
   request as undiciRequest,
   Dispatcher,
   Agent,
   getGlobalDispatcher,
   Pool,
 } from 'undici';
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
 import undiciSymbols from 'undici/lib/core/symbols.js';
-import { FormData as FormDataNode } from 'formdata-node';
-import { FormDataEncoder } from 'form-data-encoder';
-import createUserAgent from 'default-user-agent';
 import mime from 'mime-types';
 import qs from 'qs';
-import pump from 'pump';
 // Compatible with old style formstream
 import FormStream from 'formstream';
 import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
 import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
 import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
-import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
+import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
 import symbols from './symbols.js';
 import { initDiagnosticsChannel } from './diagnosticsChannel.js';
 import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
@@ -49,24 +48,12 @@ type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
 type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;
 
 const PROTO_RE = /^https?:\/\//i;
-const FormData = FormDataNative ?? FormDataNode;
-// impl promise pipeline on Node.js 14
-const pipelinePromise = stream.promises?.pipeline ?? function pipeline(...args: any[]) {
-  return new Promise<void>((resolve, reject) => {
-    pump(...args, (err?: Error) => {
-      if (err) return reject(err);
-      resolve();
-    });
-  });
-};
 
 function noop() {
   // noop
 }
 
 const debug = debuglog('urllib:HttpClient');
-// Node.js 14 or 16
-const isNode14Or16 = /v1[46]\./.test(process.version);
 
 export type ClientOptions = {
   defaultArgs?: RequestOptions;
@@ -125,7 +112,10 @@ class BlobFromStream {
   }
 }
 
-export const HEADER_USER_AGENT = createUserAgent('node-urllib', 'VERSION');
+export const VERSION = 'VERSION';
+// 'node-urllib/4.0.0 Node.js/18.19.0 (darwin; x64)'
+export const HEADER_USER_AGENT =
+  `node-urllib/${VERSION} Node.js/${process.version.substring(1)} (${process.platform}; ${process.arch})`;
 
 function getFileName(stream: Readable) {
   const filePath: string = (stream as any).path;
@@ -207,13 +197,13 @@ export class HttpClient extends EventEmitter {
   getDispatcherPoolStats() {
     const agent = this.getDispatcher();
     // origin => Pool Instance
-    const clients: Map<string, WeakRef<Pool>> | undefined = agent[undiciSymbols.kClients];
+    const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
     const poolStatsMap: Record<string, PoolStat> = {};
     if (!clients) {
       return poolStatsMap;
     }
     for (const [ key, ref ] of clients) {
-      const pool = ref.deref();
+      const pool = typeof ref.deref === 'function' ? ref.deref() : ref as unknown as Pool;
       const stats = pool?.stats;
       if (!stats) continue;
       poolStatsMap[key] = {
@@ -451,9 +441,11 @@ export class HttpClient extends EventEmitter {
         } else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
           uploadFiles.push([ 'file', args.files ]);
         } else if (typeof args.files === 'object') {
-          for (const field in args.files) {
+          const files = args.files as Record<string, string | Readable | Buffer>;
+          for (const field in files) {
             // set custom fileName
-            uploadFiles.push([ field, args.files[field], field ]);
+            const file = files[field];
+            uploadFiles.push([ field, file, field ]);
           }
         }
         // set normal fields first
@@ -478,18 +470,7 @@ export class HttpClient extends EventEmitter {
             isStreamingRequest = true;
           }
         }
-
-        if (FormDataNative) {
-          requestOptions.body = formData;
-        } else {
-          // Node.js 14 does not support spec-compliant FormData
-          // https://github.com/octet-stream/form-data#usage
-          const encoder = new FormDataEncoder(formData as any);
-          Object.assign(headers, encoder.headers);
-          // fix "Content-Length":"NaN"
-          delete headers['Content-Length'];
-          requestOptions.body = Readable.from(encoder);
-        }
+        requestOptions.body = formData;
       } else if (args.content) {
         if (!isGETOrHEAD) {
           // handle content
@@ -507,7 +488,7 @@ export class HttpClient extends EventEmitter {
           || isReadable(args.data);
         if (isGETOrHEAD) {
           if (!isStringOrBufferOrReadable) {
-            let query;
+            let query: string;
             if (args.nestedQuerystring) {
               query = qs.stringify(args.data);
             } else {
@@ -608,9 +589,6 @@ export class HttpClient extends EventEmitter {
           res = Object.assign(response.body, res);
         }
       } else if (args.writeStream) {
-        if (isNode14Or16 && args.writeStream.destroyed) {
-          throw new Error('writeStream is destroyed');
-        }
         if (args.compressed === true && isCompressedContent) {
           const decoder = contentEncoding === 'gzip' ? createGunzip() : createBrotliDecompress();
           await pipelinePromise(response.body, decoder, args.writeStream);

package/src/diagnosticsChannel.ts

@@ -20,15 +20,14 @@ let initedDiagnosticsChannel = false;
 //   -> undici:request:trailers => { request, trailers }
 
 function subscribe(name: string, listener: (message: unknown, channelName: string | symbol) => void) {
-  if (typeof diagnosticsChannel.subscribe === 'function') {
-    diagnosticsChannel.subscribe(name, listener);
-  } else {
-    // TODO: support Node.js 14, will be removed on the next major version
-    diagnosticsChannel.channel(name).subscribe(listener);
-  }
+  diagnosticsChannel.subscribe(name, listener);
 }
 
-function formatSocket(socket: Socket) {
+type SocketExtend = Socket & {
+  [key: symbol]: string | number | Date | undefined;
+};
+
+function formatSocket(socket: SocketExtend) {
   if (!socket) return socket;
   return {
     localAddress: socket[symbols.kSocketLocalAddress],
@@ -41,8 +40,7 @@ function formatSocket(socket: Socket) {
 }
 
 // make sure error contains socket info
-const kDestroy = Symbol('kDestroy');
-Socket.prototype[kDestroy] = Socket.prototype.destroy;
+const destroySocket = Socket.prototype.destroy;
 Socket.prototype.destroy = function(err?: any) {
   if (err) {
     Object.defineProperty(err, symbols.kErrorSocket, {
@@ -51,12 +49,12 @@ Socket.prototype.destroy = function(err?: any) {
       value: this,
     });
   }
-  return this[kDestroy](err);
+  return destroySocket.call(this, err);
 };
 
 function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol) {
   if (!kHandler) return;
-  const handler = request[kHandler];
+  const handler = Reflect.get(request, kHandler);
   // maxRedirects = 0 will get [Symbol(handler)]: RequestHandler {
   // responseHeaders: null,
   // opaque: {
@@ -70,7 +68,7 @@ function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol
 }
 
 export function initDiagnosticsChannel() {
-  // makre sure init global DiagnosticsChannel once
+  // make sure init global DiagnosticsChannel once
   if (initedDiagnosticsChannel) return;
   initedDiagnosticsChannel = true;
 
@@ -97,29 +95,27 @@ export function initDiagnosticsChannel() {
     opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
   });
 
-  // diagnosticsChannel.channel('undici:client:beforeConnect')
-
   subscribe('undici:client:connectError', (message, name) => {
-    const { error, connectParams } = message as DiagnosticsChannel.ClientConnectErrorMessage & { error: any };
-    let { socket } = message as DiagnosticsChannel.ClientConnectErrorMessage;
-    if (!socket && error[symbols.kErrorSocket]) {
-      socket = error[symbols.kErrorSocket];
+    const { error, connectParams, socket } = message as DiagnosticsChannel.ClientConnectErrorMessage & { error: any, socket: SocketExtend };
+    let sock = socket;
+    if (!sock && error[symbols.kErrorSocket]) {
+      sock = error[symbols.kErrorSocket];
     }
-    if (socket) {
-      socket[symbols.kSocketId] = globalId('UndiciSocket');
-      socket[symbols.kSocketConnectErrorTime] = new Date();
-      socket[symbols.kHandledRequests] = 0;
-      socket[symbols.kHandledResponses] = 0;
+    if (sock) {
+      sock[symbols.kSocketId] = globalId('UndiciSocket');
+      sock[symbols.kSocketConnectErrorTime] = new Date();
+      sock[symbols.kHandledRequests] = 0;
+      sock[symbols.kHandledResponses] = 0;
       // copy local address to symbol, avoid them be reset after request error throw
-      if (socket.localAddress) {
-        socket[symbols.kSocketLocalAddress] = socket.localAddress;
-        socket[symbols.kSocketLocalPort] = socket.localPort;
+      if (sock.localAddress) {
+        sock[symbols.kSocketLocalAddress] = sock.localAddress;
+        sock[symbols.kSocketLocalPort] = sock.localPort;
       }
-      socket[symbols.kSocketConnectProtocol] = connectParams.protocol;
-      socket[symbols.kSocketConnectHost] = connectParams.host;
-      socket[symbols.kSocketConnectPort] = connectParams.port;
+      sock[symbols.kSocketConnectProtocol] = connectParams.protocol;
+      sock[symbols.kSocketConnectHost] = connectParams.host;
+      sock[symbols.kSocketConnectPort] = connectParams.port;
       debug('[%s] Socket#%d connectError, connectParams: %o, error: %s, (sock: %o)',
-        name, socket[symbols.kSocketId], connectParams, (error as Error).message, formatSocket(socket));
+        name, sock[symbols.kSocketId], connectParams, (error as Error).message, formatSocket(sock));
     } else {
       debug('[%s] connectError, connectParams: %o, error: %o',
         name, connectParams, error);
@@ -128,7 +124,7 @@ export function initDiagnosticsChannel() {
 
   // This message is published after a connection is established.
   subscribe('undici:client:connected', (message, name) => {
-    const { socket, connectParams } = message as DiagnosticsChannel.ClientConnectedMessage;
+    const { socket, connectParams } = message as DiagnosticsChannel.ClientConnectedMessage & { socket: SocketExtend };
     socket[symbols.kSocketId] = globalId('UndiciSocket');
     socket[symbols.kSocketStartTime] = performance.now();
     socket[symbols.kSocketConnectedTime] = new Date();
@@ -145,11 +141,11 @@ export function initDiagnosticsChannel() {
 
   // This message is published right before the first byte of the request is written to the socket.
   subscribe('undici:client:sendHeaders', (message, name) => {
-    const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage;
+    const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage & { socket: SocketExtend };
     const opaque = getRequestOpaque(request, kHandler);
     if (!opaque || !opaque[symbols.kRequestId]) return;
 
-    socket[symbols.kHandledRequests]++;
+    (socket[symbols.kHandledRequests] as number)++;
     // attach socket to opaque
     opaque[symbols.kRequestSocket] = socket;
     debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)',
@@ -158,11 +154,11 @@ export function initDiagnosticsChannel() {
 
     if (!opaque[symbols.kEnableRequestTiming]) return;
     opaque[symbols.kRequestTiming].requestHeadersSent = performanceTime(opaque[symbols.kRequestStartTime]);
-    // first socket need to caculate the connected time
+    // first socket need to calculate the connected time
     if (socket[symbols.kHandledRequests] === 1) {
       // kSocketStartTime - kRequestStartTime = connected time
       opaque[symbols.kRequestTiming].connected =
-        performanceTime(opaque[symbols.kRequestStartTime], socket[symbols.kSocketStartTime]);
+        performanceTime(opaque[symbols.kRequestStartTime], socket[symbols.kSocketStartTime] as number);
     }
   });
 
@@ -184,10 +180,15 @@ export function initDiagnosticsChannel() {
 
     // get socket from opaque
     const socket = opaque[symbols.kRequestSocket];
-    socket[symbols.kHandledResponses]++;
-    debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)',
-      name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses],
-      formatSocket(socket));
+    if (socket) {
+      socket[symbols.kHandledResponses]++;
+      debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)',
+        name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses],
+        formatSocket(socket));
+    } else {
+      debug('[%s] Request#%d get %s response headers on Unknown Socket',
+        name, opaque[symbols.kRequestId], response.statusCode);
+    }
 
     if (!opaque[symbols.kEnableRequestTiming]) return;
     opaque[symbols.kRequestTiming].waiting = performanceTime(opaque[symbols.kRequestStartTime]);
@@ -197,7 +198,9 @@ export function initDiagnosticsChannel() {
   subscribe('undici:request:trailers', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestTrailersMessage;
     const opaque = getRequestOpaque(request, kHandler);
-    if (!opaque || !opaque[symbols.kRequestId]) return;
+    if (!opaque || !opaque[symbols.kRequestId]) {
+      return;
+    }
 
     debug('[%s] Request#%d get response body and trailers', name, opaque[symbols.kRequestId]);
 

package/src/index.ts

@@ -1,4 +1,4 @@
-import LRU from 'ylru';
+import { LRU } from 'ylru';
 import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
 import { RequestOptions, RequestURL } from './Request.js';
 

package/src/utils.ts

@@ -3,7 +3,7 @@ import { Readable } from 'node:stream';
 import { performance } from 'node:perf_hooks';
 import type { FixJSONCtlChars } from './Request.js';
 
-const JSONCtlCharsMap = {
+const JSONCtlCharsMap: Record<string, string> = {
   '"': '\\"', // \u0022
   '\\': '\\\\', // \u005c
   '\b': '\\b', // \u0008
@@ -49,12 +49,6 @@ export function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars) {
   return data;
 }
 
-export function sleep(ms: number) {
-  return new Promise<void>(resolve => {
-    setTimeout(resolve, ms);
-  });
-}
-
 function md5(s: string) {
   const sum = createHash('md5');
   sum.update(s, 'utf8');