urllib 4.8.2 → 4.9.1

package/src/fetch.ts

@@ -1,46 +1,34 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { debuglog } from 'node:util';
-import {
-  fetch as UndiciFetch,
-  RequestInfo,
-  RequestInit,
-  Request,
-  Response,
-  Agent,
-  getGlobalDispatcher,
-  Pool,
-  Dispatcher,
-} from 'undici';
+
+import { fetch as UndiciFetch, Request, Response, Agent, getGlobalDispatcher, Pool, Dispatcher } from 'undici';
+import type { RequestInfo, RequestInit } from 'undici';
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import undiciSymbols from 'undici/lib/core/symbols.js';
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import { getResponseState } from 'undici/lib/web/fetch/response.js';
-import {
-  channels,
+
+import { BaseAgent } from './BaseAgent.js';
+import type { BaseAgentOptions } from './BaseAgent.js';
+import { initDiagnosticsChannel } from './diagnosticsChannel.js';
+import type { FetchOpaque } from './FetchOpaqueInterceptor.js';
+import { HttpAgent } from './HttpAgent.js';
+import type { HttpAgentOptions } from './HttpAgent.js';
+import { channels } from './HttpClient.js';
+import type {
   ClientOptions,
   PoolStat,
   RequestDiagnosticsMessage,
   ResponseDiagnosticsMessage,
   UndiciTimingInfo,
 } from './HttpClient.js';
-import {
-  HttpAgent,
-  HttpAgentOptions,
-} from './HttpAgent.js';
-import { initDiagnosticsChannel } from './diagnosticsChannel.js';
-import { convertHeader, globalId, performanceTime, updateSocketInfo } from './utils.js';
+import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
+import type { FetchMeta, HttpMethod, RequestMeta } from './Request.js';
+import type { RawResponseWithMeta, SocketInfo } from './Response.js';
 import symbols from './symbols.js';
-import {
-  FetchMeta,
-  HttpMethod,
-  RequestMeta,
-} from './Request.js';
-import { FetchOpaque } from './FetchOpaqueInterceptor.js';
-import { RawResponseWithMeta, SocketInfo } from './Response.js';
-import { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
-import { BaseAgent, BaseAgentOptions } from './BaseAgent.js';
+import { convertHeader, globalId, performanceTime, updateSocketInfo } from './utils.js';
 
 const debug = debuglog('urllib/fetch');
 
@@ -68,7 +56,7 @@ export class FetchFactory {
 
   static #instance = new FetchFactory();
 
-  setClientOptions(clientOptions: ClientOptions) {
+  setClientOptions(clientOptions: ClientOptions): void {
     let dispatcherOption: BaseAgentOptions = {
       opaqueLocalStorage: this.#opaqueLocalStorage,
     };
@@ -101,15 +89,15 @@ export class FetchFactory {
     initDiagnosticsChannel();
   }
 
-  getDispatcher() {
+  getDispatcher(): Dispatcher {
     return this.#dispatcher ?? getGlobalDispatcher();
   }
 
-  setDispatcher(dispatcher: Agent) {
+  setDispatcher(dispatcher: Agent): void {
     this.#dispatcher = dispatcher;
   }
 
-  getDispatcherPoolStats() {
+  getDispatcherPoolStats(): Record<string, PoolStat> {
     const agent = this.getDispatcher();
     // origin => Pool Instance
     const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
@@ -117,8 +105,8 @@ export class FetchFactory {
     if (!clients) {
       return poolStatsMap;
     }
-    for (const [ key, ref ] of clients) {
-      const pool = (typeof ref.deref === 'function' ? ref.deref() : ref) as unknown as (Pool & { dispatcher: Pool });
+    for (const [key, ref] of clients) {
+      const pool = (typeof ref.deref === 'function' ? ref.deref() : ref) as unknown as Pool & { dispatcher: Pool };
       // NOTE: pool become to { dispatcher: Pool } in undici@v7
       const stats = pool?.stats ?? pool?.dispatcher?.stats;
       if (!stats) continue;
@@ -135,11 +123,11 @@ export class FetchFactory {
     return poolStatsMap;
   }
 
-  static setClientOptions(clientOptions: ClientOptions) {
+  static setClientOptions(clientOptions: ClientOptions): void {
     FetchFactory.#instance.setClientOptions(clientOptions);
   }
 
-  static getDispatcherPoolStats() {
+  static getDispatcherPoolStats(): Record<string, PoolStat> {
     return FetchFactory.#instance.getDispatcherPoolStats();
   }
 
@@ -225,9 +213,7 @@ export class FetchFactory {
       aborted: false,
       rt: 0,
       keepAliveSocket: true,
-      requestUrls: [
-        request.url,
-      ],
+      requestUrls: [request.url],
       timing,
       socket: socketInfo,
       retries: 0,
@@ -267,8 +253,14 @@ export class FetchFactory {
       urllibResponse.size = parseInt(urllibResponse.headers['content-length']);
     }
     urllibResponse.rt = performanceTime(requestStartTime);
-    debug('Request#%d got response, status: %s, headers: %j, timing: %j, socket: %j',
-      requestId, urllibResponse.status, urllibResponse.headers, timing, urllibResponse.socket);
+    debug(
+      'Request#%d got response, status: %s, headers: %j, timing: %j, socket: %j',
+      requestId,
+      urllibResponse.status,
+      urllibResponse.headers,
+      timing,
+      urllibResponse.socket,
+    );
     channels.fetchResponse.publish({
       fetch: fetchMeta,
       timingInfo: state.timingInfo,
@@ -284,11 +276,11 @@ export class FetchFactory {
     return res!;
   }
 
-  static getDispatcher() {
+  static getDispatcher(): Dispatcher {
     return FetchFactory.#instance.getDispatcher();
   }
 
-  static setDispatcher(dispatcher: Agent) {
+  static setDispatcher(dispatcher: Agent): void {
     FetchFactory.#instance.setDispatcher(dispatcher);
   }
 
@@ -297,4 +289,4 @@ export class FetchFactory {
   }
 }
 
-export const fetch = FetchFactory.fetch;
+export const fetch: (input: RequestInfo, init?: UrllibRequestInit) => Promise<Response> = FetchFactory.fetch;

package/src/formstream.d.ts

@@ -1,3 +1,7 @@
 declare module 'formstream' {
-  export default class FormStream {}
+  export default class FormStream {
+    headers(): Record<string, string>;
+    file(name: string, path: string): void;
+    field(name: string, value: string): void;
+  }
 }

package/src/index.ts

@@ -1,11 +1,12 @@
 import { LRU } from 'ylru';
-import { patchForNode16 } from './utils.js';
 
+import { patchForNode16 } from './utils.js';
 
 patchForNode16();
 
 import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
-import { RequestOptions, RequestURL } from './Request.js';
+import type { RequestOptions, RequestURL } from './Request.js';
+import type { HttpClientResponse } from './Response.js';
 
 let httpClient: HttpClient;
 let allowH2HttpClient: HttpClient;
@@ -63,7 +64,10 @@ interface UrllibRequestOptions extends RequestOptions {
   allowH2?: boolean;
 }
 
-export async function request<T = any>(url: RequestURL, options?: UrllibRequestOptions) {
+export async function request<T = any>(
+  url: RequestURL,
+  options?: UrllibRequestOptions,
+): Promise<HttpClientResponse<T>> {
   if (options?.socketPath) {
     let domainSocketHttpclient = domainSocketHttpClients.get<HttpClient>(options.socketPath);
     if (!domainSocketHttpclient) {
@@ -83,42 +87,52 @@ export async function request<T = any>(url: RequestURL, options?: UrllibRequestO
 // import * as urllib from 'urllib';
 // urllib.curl(url);
 // ```
-export async function curl<T = any>(url: RequestURL, options?: UrllibRequestOptions) {
+export async function curl<T = any>(url: RequestURL, options?: UrllibRequestOptions): Promise<HttpClientResponse<T>> {
   return await request<T>(url, options);
 }
 
 export {
-  MockAgent, ProxyAgent, Agent, Dispatcher,
-  setGlobalDispatcher, getGlobalDispatcher,
-  Request, RequestInfo, RequestInit,
-  Response, BodyInit, ResponseInit,
-  Headers, FormData,
+  MockAgent,
+  ProxyAgent,
+  Agent,
+  Dispatcher,
+  setGlobalDispatcher,
+  getGlobalDispatcher,
+  Request,
+  Response,
+  Headers,
+  FormData,
 } from 'undici';
+export type { RequestInfo, RequestInit, BodyInit, ResponseInit } from 'undici';
 // HttpClient2 is keep compatible with urllib@2 HttpClient2
-export {
-  HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
-  RequestDiagnosticsMessage, ResponseDiagnosticsMessage, ClientOptions,
-} from './HttpClient.js';
+export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient.js';
+export type { RequestDiagnosticsMessage, ResponseDiagnosticsMessage, ClientOptions } from './HttpClient.js';
 // RequestOptions2 is keep compatible with urllib@2 RequestOptions2
-export {
-  RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod,
-  FixJSONCtlCharsHandler, FixJSONCtlChars,
+export type {
+  RequestOptions,
+  RequestOptions as RequestOptions2,
+  RequestURL,
+  HttpMethod,
+  FixJSONCtlCharsHandler,
+  FixJSONCtlChars,
 } from './Request.js';
 
-export { CheckAddressFunction } from './HttpAgent.js';
+export type { CheckAddressFunction } from './HttpAgent.js';
 
-export {
-  SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse,
-} from './Response.js';
-export {
-  IncomingHttpHeaders,
-} from './IncomingHttpHeaders.js';
+export type { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response.js';
+export type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 export * from './HttpClientError.js';
 export { FetchFactory, fetch } from './fetch.js';
 export { FormData as WebFormData } from './FormData.js';
 
-export default {
+const urllib: {
+  request: typeof request;
+  curl: typeof curl;
+  USER_AGENT: string;
+} = {
   request,
   curl,
   USER_AGENT: HEADER_USER_AGENT,
 };
+
+export default urllib;

package/src/symbols.ts

@@ -1,4 +1,25 @@
-export default {
+const symbols: {
+  kSocketId: symbol;
+  kSocketStartTime: symbol;
+  kSocketConnectedTime: symbol;
+  kSocketConnectErrorTime: symbol;
+  kSocketRequestEndTime: symbol;
+  kSocketLocalAddress: symbol;
+  kSocketLocalPort: symbol;
+  kSocketConnectHost: symbol;
+  kSocketConnectPort: symbol;
+  kSocketConnectProtocol: symbol;
+  kHandledRequests: symbol;
+  kHandledResponses: symbol;
+  kRequestSocket: symbol;
+  kRequestId: symbol;
+  kRequestStartTime: symbol;
+  kEnableRequestTiming: symbol;
+  kRequestTiming: symbol;
+  kRequestOriginalOpaque: symbol;
+  kRequestInternalOpaque: symbol;
+  kErrorSocket: symbol;
+} = {
   kSocketId: Symbol('socket id'),
   kSocketStartTime: Symbol('socket start time'),
   kSocketConnectedTime: Symbol('socket connected time'),
@@ -20,3 +41,5 @@ export default {
   kRequestInternalOpaque: Symbol('request internal opaque'),
   kErrorSocket: Symbol('socket of error'),
 };
+
+export default symbols;

package/src/utils.ts

@@ -1,13 +1,14 @@
+import { Blob, File } from 'node:buffer';
 import { randomBytes, createHash } from 'node:crypto';
-import { Readable } from 'node:stream';
 import { performance } from 'node:perf_hooks';
+import { Readable } from 'node:stream';
 import { ReadableStream, TransformStream } from 'node:stream/web';
-import { Blob, File } from 'node:buffer';
 import { toUSVString } from 'node:util';
+
+import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 import type { FixJSONCtlChars } from './Request.js';
-import { SocketInfo } from './Response.js';
+import type { SocketInfo } from './Response.js';
 import symbols from './symbols.js';
-import { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
 
 const JSONCtlCharsMap: Record<string, string> = {
   '"': '\\"', // \u0022
@@ -29,7 +30,7 @@ function replaceJSONCtlChars(value: string) {
   return value.replace(JSONCtlCharsRE, replaceOneChar);
 }
 
-export function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars) {
+export function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars): unknown {
   if (typeof fixJSONCtlChars === 'function') {
     data = fixJSONCtlChars(data);
   } else if (fixJSONCtlChars) {
@@ -45,8 +46,12 @@ export function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars) {
     }
     if (data.length > 1024) {
       // show 0~512 ... -512~end data
-      err.message += ' (data json format: ' +
-        JSON.stringify(data.slice(0, 512)) + ' ...skip... ' + JSON.stringify(data.slice(data.length - 512)) + ')';
+      err.message +=
+        ' (data json format: ' +
+        JSON.stringify(data.slice(0, 512)) +
+        ' ...skip... ' +
+        JSON.stringify(data.slice(data.length - 512)) +
+        ')';
     } else {
       err.message += ' (data json format: ' + JSON.stringify(data) + ')';
     }
@@ -55,7 +60,7 @@ export function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars) {
   return data;
 }
 
-function md5(s: string) {
+function md5(s: string): string {
   const sum = createHash('md5');
   sum.update(s, 'utf8');
   return sum.digest('hex');
@@ -65,7 +70,7 @@ const AUTH_KEY_VALUE_RE = /(\w{1,100})=["']?([^'"]+)["']?/;
 let NC = 0;
 const NC_PAD = '00000000';
 
-export function digestAuthHeader(method: string, uri: string, wwwAuthenticate: string, userpass: string) {
+export function digestAuthHeader(method: string, uri: string, wwwAuthenticate: string, userpass: string): string {
   // WWW-Authenticate: Digest realm="testrealm@host.com",
   //                       qop="auth,auth-int",
   //                       nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093",
@@ -134,31 +139,33 @@ export function digestAuthHeader(method: string, uri: string, wwwAuthenticate: s
 const MAX_ID_VALUE = Math.pow(2, 31) - 10;
 const globalIds: Record<string, number> = {};
 
-export function globalId(category: string) {
+export function globalId(category: string): number {
   if (!globalIds[category] || globalIds[category] >= MAX_ID_VALUE) {
     globalIds[category] = 0;
   }
   return ++globalIds[category];
 }
 
-export function performanceTime(startTime: number, now?: number) {
+export function performanceTime(startTime: number, now?: number): number {
   return Math.floor(((now ?? performance.now()) - startTime) * 1000) / 1000;
 }
 
-export function isReadable(stream: any) {
+export function isReadable(stream: any): boolean {
   if (typeof Readable.isReadable === 'function') return Readable.isReadable(stream);
   // patch from node
   // https://github.com/nodejs/node/blob/1287530385137dda1d44975063217ccf90759475/lib/internal/streams/utils.js#L119
   // simple way https://github.com/sindresorhus/is-stream/blob/main/index.js
-  return stream !== null
-    && typeof stream === 'object'
-    && typeof stream.pipe === 'function'
-    && stream.readable !== false
-    && typeof stream._read === 'function'
-    && typeof stream._readableState === 'object';
+  return (
+    stream !== null &&
+    typeof stream === 'object' &&
+    typeof stream.pipe === 'function' &&
+    stream.readable !== false &&
+    typeof stream._read === 'function' &&
+    typeof stream._readableState === 'object'
+  );
 }
 
-export function updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
+export function updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any): void {
   const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
 
   if (socket) {
@@ -197,10 +204,10 @@ export function updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, er
 
 export function convertHeader(headers: Headers): IncomingHttpHeaders {
   const res: IncomingHttpHeaders = {};
-  for (const [ key, value ] of headers.entries()) {
+  for (const [key, value] of headers.entries()) {
     if (res[key]) {
       if (!Array.isArray(res[key])) {
-        res[key] = [ res[key] ];
+        res[key] = [res[key]];
       }
       res[key].push(value);
     } else {
@@ -211,7 +218,7 @@ export function convertHeader(headers: Headers): IncomingHttpHeaders {
 }
 
 // support require from Node.js 16
-export function patchForNode16() {
+export function patchForNode16(): void {
   if (typeof global.ReadableStream === 'undefined') {
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
@@ -244,14 +251,15 @@ export function patchForNode16() {
   if (String.prototype.toWellFormed === undefined) {
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
+    // eslint-disable-next-line no-extend-native
     Object.defineProperty(String.prototype, 'toWellFormed', {
-      value: function() {
+      value: function () {
         return toUSVString(this);
       },
       enumerable: false,
       configurable: true,
       writable: true,
-    })
+    });
   }
 
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -259,8 +267,9 @@ export function patchForNode16() {
   if (String.prototype.isWellFormed === undefined) {
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
+    // eslint-disable-next-line no-extend-native
     Object.defineProperty(String.prototype, 'isWellFormed', {
-      value: function() {
+      value: function () {
         // eslint-disable-next-line @typescript-eslint/ban-ts-comment
         // @ts-ignore
         return toUSVString(this) === this;
@@ -270,7 +279,6 @@ export function patchForNode16() {
       writable: true,
     });
   }
-
 }
 
 // https://github.com/jimmywarting/node-domexception/blob/main/index.js