urllib 2.37.3 → 3.0.0-alpha.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "urllib",
-  "version": "2.37.3",
-  "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
+  "version": "3.0.0-alpha.0",
+  "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Base undici fetch API.",
   "keywords": [
     "urllib",
     "http",
@@ -9,81 +9,83 @@
     "curl",
     "wget",
     "request",
-    "https"
+    "https",
+    "undici",
+    "fetch"
   ],
-  "author": "fengmk2 <fengmk2@gmail.com> (https://fengmk2.com)",
+  "author": "fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)",
   "homepage": "https://github.com/node-modules/urllib",
-  "main": "lib/index.js",
-  "types": "lib/index.d.ts",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./src/esm/index.d.ts",
+        "default": "./src/esm/index.js"
+      },
+      "require": {
+        "types": "./src/cjs/index.d.ts",
+        "default": "./src/cjs/index.js"
+      }
+    }
+  },
+  "types": "./src/esm/index.d.ts",
+  "main": "./src/cjs/index.js",
   "files": [
-    "lib"
+    "src"
   ],
   "repository": {
     "type": "git",
     "url": "git://github.com/node-modules/urllib.git"
   },
   "scripts": {
-    "test-local": "mocha -t 30000 -r intelli-espower-loader test/*.test.js",
-    "test": "npm run lint && npm run test-local",
-    "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- -t 30000 -r intelli-espower-loader test/*.test.js",
-    "ci": "npm run lint && npm run test-cov",
-    "lint": "jshint .",
-    "autod": "autod -w --prefix '^' -t test -e examples",
-    "contributor": "git-contributor"
+    "lint": "eslint src --ext .ts",
+    "build": "npm run build:dist",
+    "build:cjs": "tsc -p ./tsconfig.build.cjs.json",
+    "build:esm": "tsc -p ./tsconfig.build.esm.json && node ./scripts/esm_import_fix.js",
+    "build:dist": "tsc --version && npm run build:cjs && npm run build:esm",
+    "build:cjs:test": "cd test/cjs && rm -rf node_modules && npm link ../.. && node index.js",
+    "build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
+    "build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test",
+    "test": "tsc --version && jest --coverage",
+    "ci": "npm run lint && npm run test && npm run build:test",
+    "contributor": "git-contributor",
+    "prepack": "npm run build && rm -rf src/*.tsbuildinfo"
   },
   "dependencies": {
-    "any-promise": "^1.3.0",
     "content-type": "^1.0.2",
-    "debug": "^2.6.9",
     "default-user-agent": "^1.0.0",
     "digest-header": "^0.0.1",
-    "ee-first": "~1.1.1",
-    "formstream": "^1.1.0",
     "humanize-ms": "^1.2.0",
     "iconv-lite": "^0.4.15",
     "ip": "^1.1.5",
-    "proxy-agent": "^4.0.1",
-    "pump": "^3.0.0",
-    "qs": "^6.4.0",
-    "statuses": "^1.3.1",
-    "utility": "^1.16.1"
+    "mime-types": "^2.1.35",
+    "tslib": "^2.4.0",
+    "undici": "^5.4.0"
   },
   "devDependencies": {
-    "@types/mocha": "^5.2.5",
-    "@types/node": "^10.12.18",
-    "agentkeepalive": "^4.0.0",
-    "autod": "*",
-    "benchmark": "^2.1.4",
-    "bluebird": "*",
-    "busboy": "^0.2.14",
-    "co": "*",
-    "coffee": "1",
-    "egg-ci": "^1.15.0",
-    "git-contributor": "^1.0.10",
-    "http-proxy": "^1.16.2",
-    "intelli-espower-loader": "^1.0.1",
-    "istanbul": "*",
-    "jshint": "*",
-    "mkdirp": "^0.5.1",
-    "mocha": "3",
-    "muk": "^0.5.3",
-    "pedding": "^1.1.0",
-    "power-assert": "^1.4.2",
-    "semver": "5",
-    "spy": "^1.0.0",
-    "tar": "^4.4.8",
-    "through2": "^2.0.3",
-    "typescript": "^3.2.2"
+    "@types/busboy": "^1.5.0",
+    "@types/default-user-agent": "^1.0.0",
+    "@types/jest": "28",
+    "@types/mime-types": "^2.1.1",
+    "busboy": "^1.6.0",
+    "coffee": "5",
+    "egg-ci": "2",
+    "eslint": "^8.17.0",
+    "eslint-config-egg": "^12.0.0",
+    "git-contributor": "1",
+    "jest": "28",
+    "jest-summary-reporter": "^0.0.2",
+    "ts-jest": "28",
+    "typescript": "4"
   },
   "engines": {
-    "node": ">= 0.10.0"
+    "node": ">= 16.0.0"
   },
   "ci": {
-    "type": "github",
-    "os": {
-      "github": "linux, windows, macos"
-    },
-    "version": "8, 10, 12, 14"
+    "version": "16, 18"
+  },
+  "publishConfig": {
+    "tag": "next"
   },
   "license": "MIT"
 }

package/src/HttpClient.ts

@@ -0,0 +1,274 @@
+import { EventEmitter } from 'events';
+import { debuglog } from 'util';
+import { Readable, isReadable } from 'stream';
+import { Blob } from 'buffer';
+import { createReadStream } from 'fs';
+import { basename } from 'path';
+import {
+  fetch, RequestInit, Headers, FormData,
+} from 'undici';
+import createUserAgent from 'default-user-agent';
+import mime from 'mime-types';
+import { RequestURL, RequestOptions } from './Request';
+
+const debug = debuglog('urllib');
+
+export type ClientOptions = {
+  defaultArgs?: RequestOptions;
+};
+
+// https://github.com/octet-stream/form-data
+class BlobFromStream {
+  #stream;
+  #type;
+  constructor(stream: Readable, type: string) {
+    this.#stream = stream;
+    this.#type = type;
+  }
+
+  stream() {
+    return this.#stream;
+  }
+
+  get type(): string {
+    return this.#type;
+  }
+
+  get [Symbol.toStringTag]() {
+    return 'Blob';
+  }
+}
+
+class HttpClientRequestTimeoutError extends Error {
+  constructor(timeout: number, options: ErrorOptions) {
+    const message = `Request timeout for ${timeout} ms`;
+    super(message, options);
+    this.name = this.constructor.name;
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.0.0');
+
+function getFileName(stream: Readable) {
+  const filePath: string = (stream as any).path;
+  if (filePath) {
+    return basename(filePath);
+  }
+  return '';
+}
+
+export class HttpClient extends EventEmitter {
+  defaultArgs?: RequestOptions;
+
+  constructor(clientOptions?: ClientOptions) {
+    super();
+    this.defaultArgs = clientOptions?.defaultArgs;
+  }
+
+  async request(url: RequestURL, options?: RequestOptions) {
+    const requestUrl = typeof url === 'string' ? new URL(url) : url;
+    const args = {
+      ...this.defaultArgs,
+      ...options,
+      emitter: this,
+    };
+    const requestStartTime = Date.now();
+    // keep urllib createCallbackResponse style
+    const resHeaders: Record<string, string> = {};
+    const res = {
+      status: -1,
+      statusCode: -1,
+      statusMessage: '',
+      headers: resHeaders,
+      size: 0,
+      aborted: false,
+      rt: 0,
+      keepAliveSocket: true,
+      requestUrls: [ url.toString() ],
+      timing: {
+        contentDownload: 0,
+      },
+      // remoteAddress: remoteAddress,
+      // remotePort: remotePort,
+      // socketHandledRequests: socketHandledRequests,
+      // socketHandledResponses: socketHandledResponses,
+    };
+
+    let requestTimeout = 5000;
+    if (args.timeout) {
+      if (Array.isArray(args.timeout)) {
+        requestTimeout = args.timeout[args.timeout.length - 1] ?? requestTimeout;
+      } else {
+        requestTimeout = args.timeout;
+      }
+    }
+
+    const requestTimeoutController = new AbortController();
+    const requestTimerId = setTimeout(() => requestTimeoutController.abort(), requestTimeout);
+    const method = (args.method ?? 'GET').toUpperCase();
+
+    try {
+      const headers = new Headers(args.headers ?? {});
+      // don't set user-agent
+      const disableUserAgent = args.headers &&
+        (args.headers['User-Agent'] === null || args.headers['user-agent'] === null);
+      if (!disableUserAgent && !headers.has('user-agent')) {
+        // need to set user-agent
+        headers.set('user-agent', HEADER_USER_AGENT);
+      }
+      if (args.dataType === 'json' && !headers.has('accept')) {
+        headers.set('accept', 'application/json');
+      }
+
+      const requestOptions: RequestInit = {
+        method,
+        keepalive: true,
+        signal: requestTimeoutController.signal,
+      };
+      if (args.followRedirect === false) {
+        requestOptions.redirect = 'manual';
+      }
+
+      const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
+
+      if (args.files) {
+        if (isGETOrHEAD) {
+          requestOptions.method = 'POST';
+        }
+        const formData = new FormData();
+        const uploadFiles: [string, string | Readable | Buffer][] = [];
+        if (Array.isArray(args.files)) {
+          for (const [ index, file ] of args.files.entries()) {
+            const field = index === 0 ? 'file' : `file${index}`;
+            uploadFiles.push([ field, file ]);
+          }
+        } else if (args.files instanceof Readable || isReadable(args.files as any)) {
+          uploadFiles.push([ 'file', args.files as Readable ]);
+        } else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
+          uploadFiles.push([ 'file', args.files ]);
+        } else if (typeof args.files === 'object') {
+          for (const field in args.files) {
+            uploadFiles.push([ field, args.files[field] ]);
+          }
+        }
+        // set normal fields first
+        if (args.data) {
+          for (const field in args.data) {
+            formData.append(field, args.data[field]);
+          }
+        }
+        for (const [ index, [ field, file ]] of uploadFiles.entries()) {
+          if (typeof file === 'string') {
+            // FIXME: support non-ascii filename
+            // const fileName = encodeURIComponent(basename(file));
+            // formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
+            const fileName = basename(file);
+            const fileReader = createReadStream(file);
+            formData.append(field, new BlobFromStream(fileReader, mime.lookup(fileName) || ''), fileName);
+          } else if (Buffer.isBuffer(file)) {
+            formData.append(field, new Blob([ file ]), `bufferfile${index}`);
+          } else if (file instanceof Readable || isReadable(file as any)) {
+            const fileName = getFileName(file) || `streamfile${index}`;
+            formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
+          }
+        }
+        requestOptions.body = formData;
+      } else if (args.content) {
+        if (!isGETOrHEAD) {
+          if (isReadable(args.content as Readable)) {
+            // disable keepalive
+            requestOptions.keepalive = false;
+          }
+          // handle content
+          requestOptions.body = args.content;
+          if (args.contentType) {
+            headers.set('content-type', args.contentType);
+          }
+        }
+      } else if (args.data) {
+        const isStringOrBufferOrReadable = typeof args.data === 'string'
+          || Buffer.isBuffer(args.data)
+          || isReadable(args.data);
+        if (isGETOrHEAD) {
+          if (!isStringOrBufferOrReadable) {
+            for (const field in args.data) {
+              requestUrl.searchParams.append(field, args.data[field]);
+            }
+          }
+        } else {
+          if (isStringOrBufferOrReadable) {
+            if (isReadable(args.data as Readable)) {
+              // disable keepalive
+              requestOptions.keepalive = false;
+            }
+            requestOptions.body = args.data;
+          } else {
+            if (args.contentType === 'json'
+              || args.contentType === 'application/json'
+              || headers.get('content-type')?.startsWith('application/json')) {
+              requestOptions.body = JSON.stringify(args.data);
+              if (!headers.has('content-type')) {
+                headers.set('content-type', 'application/json');
+              }
+            } else {
+              requestOptions.body = new URLSearchParams(args.data);
+            }
+          }
+        }
+      }
+
+      debug('%s %s, headers: %j, timeout: %s', requestOptions.method, url, headers, requestTimeout);
+      requestOptions.headers = headers;
+
+      const response = await fetch(requestUrl, requestOptions);
+      for (const [ name, value ] of response.headers) {
+        res.headers[name] = value;
+      }
+      res.status = res.statusCode = response.status;
+      res.statusMessage = response.statusText;
+      if (response.redirected) {
+        res.requestUrls.push(response.url);
+      }
+      if (res.headers['content-length']) {
+        res.size = parseInt(res.headers['content-length']);
+      }
+
+      let data: any;
+      if (args.streaming || args.dataType === 'stream') {
+        data = response.body;
+      } else if (args.dataType === 'text') {
+        data = await response.text();
+      } else if (args.dataType === 'json') {
+        if (requestOptions.method === 'HEAD') {
+          data = {};
+        } else {
+          data = await response.json();
+        }
+      } else {
+        // buffer
+        data = Buffer.from(await response.arrayBuffer());
+      }
+      res.rt = res.timing.contentDownload = Date.now() - requestStartTime;
+
+      return {
+        status: res.status,
+        data,
+        headers: res.headers,
+        url: response.url,
+        redirected: response.redirected,
+        res,
+      };
+    } catch (e: any) {
+      let err = e;
+      if (requestTimeoutController.signal.aborted) {
+        err = new HttpClientRequestTimeoutError(requestTimeout, { cause: e });
+      }
+      err.res = res;
+      // console.error(err);
+      throw err;
+    } finally {
+      clearTimeout(requestTimerId);
+    }
+  }
+}

package/src/Request.ts

@@ -0,0 +1,116 @@
+import { Readable, Writable } from 'stream';
+import { LookupFunction } from 'net';
+
+export type HttpMethod = 'GET' | 'POST' | 'DELETE' | 'PUT' | 'HEAD' | 'OPTIONS' | 'PATCH' | 'TRACE' | 'CONNECT';
+
+export type RequestURL = string | URL;
+
+export type RequestOptions = {
+  /** Request method, defaults to GET. Could be GET, POST, DELETE or PUT. Alias 'type'. */
+  method?: HttpMethod | Lowercase<HttpMethod>;
+  /** Data to be sent. Will be stringify automatically. */
+  data?: any;
+  /** Force convert data to query string. */
+  dataAsQueryString?: boolean;
+  /** Manually set the content of payload. If set, data will be ignored. */
+  content?: string | Buffer | Readable;
+  /** Stream to be pipe to the remote. If set, data and content will be ignored. */
+  stream?: Readable;
+  /**
+   * A writable stream to be piped by the response stream.
+   * Responding data will be write to this stream and callback
+   * will be called with data set null after finished writing.
+   */
+  writeStream?: Writable;
+  /** consume the writeStream, invoke the callback after writeStream close. */
+  consumeWriteStream?: boolean;
+  /**
+    * The files will send with multipart/form-data format, base on formstream.
+    * If method not set, will use POST method by default.
+    */
+  files?: Array<Readable | Buffer | string> | Record<string, Readable | Buffer | string> | Readable | Buffer | string;
+  /** Type of request data, could be 'json'. If it's 'json', will auto set Content-Type: 'application/json' header. */
+  contentType?: string;
+  /**
+   * Type of response data. Could be text or json.
+   * If it's text, the callbacked data would be a String.
+   * If it's json, the data of callback would be a parsed JSON Object
+   * and will auto set Accept: 'application/json' header.
+   * Default is buffer.
+   */
+  dataType?: 'text' | 'json' | 'buffer' | 'stream';
+  /**
+   * Let you get the res object when request connected, default false.
+   * If set to true, `data` will be response readable stream.
+   * Equal to `dataType = 'stream'`
+   */
+  streaming?: boolean;
+  /** Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is false. */
+  fixJSONCtlChars?: boolean;
+  /** Request headers. */
+  headers?: Record<string, string>;
+  /**
+   * Request timeout in milliseconds for connecting phase and response receiving phase.
+   * Defaults to exports.
+   * TIMEOUT, both are 5s. You can use timeout: 5000 to tell urllib use same timeout on two phase or set them seperately such as
+   * timeout: [3000, 5000], which will set connecting timeout to 3s and response 5s.
+   */
+  timeout?: number | number[];
+  /** username:password used in HTTP Basic Authorization. */
+  auth?: string;
+  /** username:password used in HTTP Digest Authorization. */
+  digestAuth?: string;
+  /**
+   * An array of strings or Buffers of trusted certificates.
+   * If this is omitted several well known "root" CAs will be used, like VeriSign.
+   * These are used to authorize connections.
+   * Notes: This is necessary only if the server uses the self - signed certificate
+   */
+  ca?: string | Buffer | string[] | Buffer[];
+  /**
+   * If true, the server certificate is verified against the list of supplied CAs.
+   * An 'error' event is emitted if verification fails.Default: true.
+   */
+  rejectUnauthorized?: boolean;
+  /** A string or Buffer containing the private key, certificate and CA certs of the server in PFX or PKCS12 format. */
+  pfx?: string | Buffer;
+  /**
+   * A string or Buffer containing the private key of the client in PEM format.
+   * Notes: This is necessary only if using the client certificate authentication
+   */
+  key?: string | Buffer;
+  /**
+   * A string or Buffer containing the certificate key of the client in PEM format.
+   * Notes: This is necessary only if using the client certificate authentication
+   */
+  cert?: string | Buffer;
+  /** A string of passphrase for the private key or pfx. */
+  passphrase?: string;
+  /** A string describing the ciphers to use or exclude. */
+  ciphers?: string;
+  /** The SSL method to use, e.g.SSLv3_method to force SSL version 3. */
+  secureProtocol?: string;
+  /** follow HTTP 3xx responses as redirects. defaults to false. */
+  followRedirect?: boolean;
+  /** The maximum number of redirects to follow, defaults to 10. */
+  maxRedirects?: number;
+  /** Format the redirect url by your self. Default is url.resolve(from, to). */
+  formatRedirectUrl?: (a: any, b: any) => void;
+  /** Before request hook, you can change every thing here. */
+  beforeRequest?: (...args: any[]) => void;
+  /** Accept gzip response content and auto decode it, default is false. */
+  gzip?: boolean;
+  /** Enable timing or not, default is false. */
+  timing?: boolean;
+  /**
+   * Custom DNS lookup function, default is dns.lookup.
+   * Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
+   */
+  lookup?: LookupFunction;
+  /**
+   * check request address to protect from SSRF and similar attacks.
+   * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
+   * It rely on lookup and have the same version requirement.
+   */
+  checkAddress?: (ip: string, family: number | string) => boolean;
+};

package/src/Response.ts

@@ -0,0 +1,20 @@
+import { OutgoingHttpHeaders, IncomingMessage } from 'http';
+
+export interface HttpClientResponse<T> {
+  data: T;
+  status: number;
+  headers: OutgoingHttpHeaders;
+  res: IncomingMessage & {
+    /**
+     * https://eggjs.org/en/core/httpclient.html#timing-boolean
+     */
+    timing?: {
+      queuing: number;
+      dnslookup: number;
+      connected: number;
+      requestSent: number;
+      waiting: number;
+      contentDownload: number;
+    }
+  };
+}

package/src/cjs/HttpClient.d.ts

@@ -0,0 +1,31 @@
+/// <reference types="node" />
+import { EventEmitter } from 'events';
+import { RequestURL, RequestOptions } from './Request';
+export declare type ClientOptions = {
+    defaultArgs?: RequestOptions;
+};
+export declare class HttpClient extends EventEmitter {
+    defaultArgs?: RequestOptions;
+    constructor(clientOptions?: ClientOptions);
+    request(url: RequestURL, options?: RequestOptions): Promise<{
+        status: number;
+        data: any;
+        headers: Record<string, string>;
+        url: string;
+        redirected: boolean;
+        res: {
+            status: number;
+            statusCode: number;
+            statusMessage: string;
+            headers: Record<string, string>;
+            size: number;
+            aborted: boolean;
+            rt: number;
+            keepAliveSocket: boolean;
+            requestUrls: string[];
+            timing: {
+                contentDownload: number;
+            };
+        };
+    }>;
+}