urllib 2.38.1 → 3.0.0

package/lib/detect_proxy_agent.js

@@ -1,31 +0,0 @@
-'use strict';
-
-var debug = require('debug')('urllib:detect_proxy_agent');
-var getProxyFromURI = require('./get_proxy_from_uri');
-
-var proxyAgents = {};
-
-function detectProxyAgent(uri, args) {
-  if (!args.enableProxy && !process.env.URLLIB_ENABLE_PROXY) {
-    return null;
-  }
-  var proxy = args.proxy || process.env.URLLIB_PROXY;
-  if (!proxy) {
-    proxy = getProxyFromURI(uri);
-    if (!proxy) {
-      return null;
-    }
-  }
-
-  var proxyAgent = proxyAgents[proxy];
-  if (!proxyAgent) {
-    debug('create new proxy %s', proxy);
-    // lazy require, only support node >= 4
-    proxyAgent = proxyAgents[proxy] = new (require('proxy-agent'))(proxy);
-  }
-  debug('get proxy: %s', proxy);
-  return proxyAgent;
-}
-
-module.exports = detectProxyAgent;
-module.exports.proxyAgents = proxyAgents;

package/lib/get_proxy_from_uri.js

@@ -1,81 +0,0 @@
-// copy from https://github.com/request/request/blob/90cf8c743bb9fd6a4cb683a56fb7844c6b316866/lib/getProxyFromURI.js
-
-'use strict'
-
-function formatHostname(hostname) {
-  // canonicalize the hostname, so that 'oogle.com' won't match 'google.com'
-  return hostname.replace(/^\.*/, '.').toLowerCase()
-}
-
-function parseNoProxyZone(zone) {
-  zone = zone.trim().toLowerCase()
-
-  var zoneParts = zone.split(':', 2)
-    , zoneHost = formatHostname(zoneParts[0])
-    , zonePort = zoneParts[1]
-    , hasPort = zone.indexOf(':') > -1
-
-  return {hostname: zoneHost, port: zonePort, hasPort: hasPort}
-}
-
-function uriInNoProxy(uri, noProxy) {
-  var port = uri.port || (uri.protocol === 'https:' ? '443' : '80')
-    , hostname = formatHostname(uri.hostname)
-    , noProxyList = noProxy.split(',')
-
-  // iterate through the noProxyList until it finds a match.
-  return noProxyList.map(parseNoProxyZone).some(function(noProxyZone) {
-    var isMatchedAt = hostname.indexOf(noProxyZone.hostname)
-      , hostnameMatched = (
-          isMatchedAt > -1 &&
-          (isMatchedAt === hostname.length - noProxyZone.hostname.length)
-        )
-
-    if (noProxyZone.hasPort) {
-      return (port === noProxyZone.port) && hostnameMatched
-    }
-
-    return hostnameMatched
-  })
-}
-
-function getProxyFromURI(uri) {
-  // Decide the proper request proxy to use based on the request URI object and the
-  // environmental variables (NO_PROXY, HTTP_PROXY, etc.)
-  // respect NO_PROXY environment variables (see: http://lynx.isc.org/current/breakout/lynx_help/keystrokes/environments.html)
-
-  var noProxy = process.env.NO_PROXY || process.env.no_proxy || ''
-
-  // if the noProxy is a wildcard then return null
-
-  if (noProxy === '*') {
-    return null
-  }
-
-  // if the noProxy is not empty and the uri is found return null
-
-  if (noProxy !== '' && uriInNoProxy(uri, noProxy)) {
-    return null
-  }
-
-  // Check for HTTP or HTTPS Proxy in environment Else default to null
-
-  if (uri.protocol === 'http:') {
-    return process.env.HTTP_PROXY ||
-           process.env.http_proxy || null
-  }
-
-  if (uri.protocol === 'https:') {
-    return process.env.HTTPS_PROXY ||
-           process.env.https_proxy ||
-           process.env.HTTP_PROXY  ||
-           process.env.http_proxy  || null
-  }
-
-  // if none of that works, return null
-  // (What uri protocol are you using then?)
-
-  return null
-}
-
-module.exports = getProxyFromURI

package/lib/httpclient.js

@@ -1,61 +0,0 @@
-'use strict';
-
-var EventEmitter = require('events').EventEmitter;
-var util = require('util');
-var utility = require('utility');
-var urllib = require('./urllib');
-
-module.exports = HttpClient;
-
-function HttpClient(options) {
-  EventEmitter.call(this);
-  options = options || {};
-
-  if (options.agent !== undefined) {
-    this.agent = options.agent;
-    this.hasCustomAgent = true;
-  } else {
-    this.agent = urllib.agent;
-    this.hasCustomAgent = false;
-  }
-
-  if (options.httpsAgent !== undefined) {
-    this.httpsAgent = options.httpsAgent;
-    this.hasCustomHttpsAgent = true;
-  } else {
-    this.httpsAgent = urllib.httpsAgent;
-    this.hasCustomHttpsAgent = false;
-  }
-  this.defaultArgs = options.defaultArgs;
-}
-util.inherits(HttpClient, EventEmitter);
-
-HttpClient.prototype.request = HttpClient.prototype.curl = function (url, args, callback) {
-  if (typeof args === 'function') {
-    callback = args;
-    args = null;
-  }
-  args = args || {};
-  if (this.defaultArgs) {
-    args = utility.assign({}, [ this.defaultArgs, args ]);
-  }
-  args.emitter = this;
-  args.agent = getAgent(args.agent, this.agent);
-  args.httpsAgent = getAgent(args.httpsAgent, this.httpsAgent);
-  return urllib.request(url, args, callback);
-};
-
-HttpClient.prototype.requestThunk = function (url, args) {
-  args = args || {};
-  if (this.defaultArgs) {
-    args = utility.assign({}, [ this.defaultArgs, args ]);
-  }
-  args.emitter = this;
-  args.agent = getAgent(args.agent, this.agent);
-  args.httpsAgent = getAgent(args.httpsAgent, this.httpsAgent);
-  return urllib.requestThunk(url, args);
-};
-
-function getAgent(agent, defaultAgent) {
-  return agent === undefined ? defaultAgent : agent;
-}

package/lib/httpclient2.js

@@ -1,83 +0,0 @@
-'use strict';
-
-var util = require('util');
-var debug = require('debug')('urllib');
-var ms = require('humanize-ms');
-var HttpClient = require('./httpclient');
-
-var _Promise;
-
-module.exports = HttpClient2;
-
-function HttpClient2(options) {
-  HttpClient.call(this, options);
-}
-
-util.inherits(HttpClient2, HttpClient);
-
-HttpClient2.prototype.request = HttpClient2.prototype.curl = function request(url, args) {
-  var self = this;
-  args = args || {};
-  args.retry = args.retry || 0;
-  if (args.retryDelay) {
-    args.retryDelay = ms(args.retryDelay);
-  }
-  args.isRetry = args.isRetry || function(res) {
-    return res.status >= 500;
-  };
-  return HttpClient.prototype.request.call(self, url, args)
-  .then(function(res) {
-    if (args.retry > 0 && typeof args.isRetry === 'function' && args.isRetry(res)) {
-      args.retry--;
-      debug('retry request %s, remain %s', url, args.retry);
-      if (args.retryDelay) {
-        debug('retry after %sms', args.retryDelay);
-        return sleep(args.retryDelay).then(function() { return self.request(url, args); });
-      }
-      return self.request(url, args);
-    }
-    return res;
-  })
-  .catch(function(err) {
-    if (args.retry > 0) {
-      args.retry--;
-      debug('retry request %s, remain %s, err %s', url, args.retry, err);
-      if (args.retryDelay) {
-        debug('retry after %sms', args.retryDelay);
-        return sleep(args.retryDelay).then(function() { return self.request(url, args); });
-      }
-      return self.request(url, args);
-    }
-    throw err;
-  });
-};
-
-HttpClient2.prototype.requestThunk = function requestThunk(url, args) {
-  var self = this;
-  return function(callback) {
-    self.request(url, args)
-    .then(function(res) {
-      var cb = callback;
-      // make sure cb(null, res) throw error won't emit catch callback below
-      callback = null;
-      cb(null, res);
-    })
-    .catch(function(err) {
-      if (!callback) {
-        // TODO: how to handle this error?
-        return;
-      }
-      callback(err);
-    });
-  };
-};
-
-function sleep(ms) {
-  if (!_Promise) {
-    _Promise = require('any-promise');
-  }
-
-  return new _Promise(function(resolve) {
-    setTimeout(resolve, ms);
-  });
-}

package/lib/index.d.ts

@@ -1,279 +0,0 @@
-import * as https from 'https';
-import * as http from 'http';
-import { IncomingHttpHeaders, OutgoingHttpHeaders } from 'http';
-import * as url from 'url';
-import { Readable, Writable } from 'stream';
-import { EventEmitter } from 'events';
-import { LookupFunction } from 'net';
-
-export { IncomingHttpHeaders, OutgoingHttpHeaders };
-export type HttpMethod = "GET" | "POST" | "DELETE" | "PUT" | "HEAD" | "OPTIONS" | "PATCH" | "TRACE" | "CONNECT";
-
-export as namespace urllib;
-export interface RequestOptions {
-  /** Request method, defaults to GET. Could be GET, POST, DELETE or PUT. Alias 'type'. */
-  method?: HttpMethod;
-  /** Alias method  */
-  type?: HttpMethod;
-  /** Data to be sent. Will be stringify automatically. */
-  data?: any;
-  /** Force convert data to query string. */
-  dataAsQueryString?: boolean;
-  /** Manually set the content of payload. If set, data will be ignored. */
-  content?: string | Buffer;
-  /** Stream to be pipe to the remote.If set, data and content will be ignored. */
-  stream?: Readable;
-  /**
-   * A writable stream to be piped by the response stream.
-   * Responding data will be write to this stream and callback
-   * will be called with data set null after finished writing.
-   */
-  writeStream?: Writable;
-  /** consume the writeStream, invoke the callback after writeStream close. */
-  consumeWriteStream?: boolean;
-  /** 
-    * The files will send with multipart/form-data format, base on formstream.
-    * If method not set, will use POST method by default.
-    */
-  files?: Array<Readable | Buffer | string> | object | Readable | Buffer | string;
-  /** Type of request data.Could be json.If it's json, will auto set Content-Type: application/json header. */
-  contentType?: string;
-  /**
-   * urllib default use querystring to stringify form data which don't support nested object,
-   * will use qs instead of querystring to support nested object by set this option to true.
-   */
-  nestedQuerystring?: boolean;
-  /**
-   * Type of response data. Could be text or json.
-   * If it's text, the callbacked data would be a String.
-   * If it's json, the data of callback would be a parsed JSON Object
-   * and will auto set Accept: application/json header. Default callbacked data would be a Buffer.
-   */
-  dataType?: string;
-  /** Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is false. */
-  fixJSONCtlChars?: boolean;
-  /** Request headers. */
-  headers?: IncomingHttpHeaders;
-  /** by default will convert header keys to lowercase */
-  keepHeaderCase?: boolean;
-  /**
-   * Request timeout in milliseconds for connecting phase and response receiving phase.
-   * Defaults to exports.
-   * TIMEOUT, both are 5s.You can use timeout: 5000 to tell urllib use same timeout on two phase or set them seperately such as
-   * timeout: [3000, 5000], which will set connecting timeout to 3s and response 5s.
-   */
-  timeout?: number | number[];
-  /** username:password used in HTTP Basic Authorization. */
-  auth?: string;
-  /** username:password used in HTTP Digest Authorization. */
-  digestAuth?: string;
-  /** HTTP Agent object.Set false if you does not use agent. */
-  agent?: http.Agent;
-  /** HTTPS Agent object. Set false if you does not use agent. */
-  httpsAgent?: https.Agent;
-  /**
-   * An array of strings or Buffers of trusted certificates.
-   * If this is omitted several well known "root" CAs will be used, like VeriSign.
-   * These are used to authorize connections.
-   * Notes: This is necessary only if the server uses the self - signed certificate
-   */
-  ca?: string | Buffer | string[] | Buffer[];
-  /**
-   * If true, the server certificate is verified against the list of supplied CAs.
-   * An 'error' event is emitted if verification fails.Default: true.
-   */
-  rejectUnauthorized?: boolean;
-  /** A string or Buffer containing the private key, certificate and CA certs of the server in PFX or PKCS12 format. */
-  pfx?: string | Buffer;
-  /**
-   * A string or Buffer containing the private key of the client in PEM format.
-   * Notes: This is necessary only if using the client certificate authentication
-   */
-  key?: string | Buffer;
-  /**
-   * A string or Buffer containing the certificate key of the client in PEM format.
-   * Notes: This is necessary only if using the client certificate authentication
-   */
-  cert?: string | Buffer;
-  /** A string of passphrase for the private key or pfx. */
-  passphrase?: string;
-  /** A string describing the ciphers to use or exclude. */
-  ciphers?: string;
-  /** The SSL method to use, e.g.SSLv3_method to force SSL version 3. */
-  secureProtocol?: string;
-  /** follow HTTP 3xx responses as redirects. defaults to false. */
-  followRedirect?: boolean;
-  /** The maximum number of redirects to follow, defaults to 10. */
-  maxRedirects?: number;
-  /** Format the redirect url by your self. Default is url.resolve(from, to). */
-  formatRedirectUrl?: (a: any, b: any) => void;
-  /** Before request hook, you can change every thing here. */
-  beforeRequest?: (...args: any[]) => void;
-  /** let you get the res object when request connected, default false. alias customResponse */
-  streaming?: boolean;
-  /** Accept gzip response content and auto decode it, default is false. */
-  gzip?: boolean;
-  /** Enable timing or not, default is false. */
-  timing?: boolean;
-  /** Enable proxy request, default is false. */
-  enableProxy?: boolean;
-  /** proxy agent uri or options, default is null. */
-  proxy?: string | { [key: string]: any };
-  /**
-   * Custom DNS lookup function, default is dns.lookup.
-   * Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
-   */
-  lookup?: LookupFunction;
-  /**
-   * check request address to protect from SSRF and similar attacks.
-   * It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
-   * It rely on lookup and have the same version requirement.
-   */
-  checkAddress?: (ip: string, family: number | string) => boolean;
-  /**
-   * UNIX domain socket path. (Windows is not supported)
-   */
-  socketPath?: string;
-}
-
-export interface HttpClientResponse<T> {
-  data: T;
-  status: number;
-  headers:  OutgoingHttpHeaders;
-  res: http.IncomingMessage & {
-    /**
-     * https://eggjs.org/en/core/httpclient.html#timing-boolean
-     */
-    timing?: {
-      queuing: number;
-      dnslookup: number;
-      connected: number;
-      requestSent: number;
-      waiting: number;
-      contentDownload: number;
-    }
-  };
-}
-
-
-/**
- * @param data Outgoing message
- * @param res http response
- */
-export type Callback<T> = (err: Error, data: T, res: http.IncomingMessage) => void;
-
-/**
- * Handle all http request, both http and https support well.
- *
- * @example
- * // GET http://httptest.cnodejs.net
- * urllib.request('http://httptest.cnodejs.net/test/get', function(err, data, res) {});
- * // POST http://httptest.cnodejs.net
- * var args = { type: 'post', data: { foo: 'bar' } };
- * urllib.request('http://httptest.cnodejs.net/test/post', args, function(err, data, res) {});
- *
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function request<T = any>(url: string | url.URL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
-/**
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function request<T = any>(url: string | url.URL, callback: Callback<T>): void;
-/**
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function request<T = any>(url: string | url.URL, options: RequestOptions, callback: Callback<T>): void;
-
-/**
- * Handle request with a callback.
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function requestWithCallback<T = any>(url: string | url.URL, callback: Callback<T>): void;
-/**
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function requestWithCallback<T = any>(url: string | url.URL, options: RequestOptions, callback: Callback<T>): void;
-
-/**
- * yield urllib.requestThunk(url, args)
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function requestThunk<T = any>(url: string | url.URL, options?: RequestOptions): (callback: Callback<T>) => void;
-
-/**
- * alias to request.
- * Handle all http request, both http and https support well.
- *
- * @example
- * // GET http://httptest.cnodejs.net
- * urllib.request('http://httptest.cnodejs.net/test/get', function(err, data, res) {});
- * // POST http://httptest.cnodejs.net
- * var args = { type: 'post', data: { foo: 'bar' } };
- * urllib.request('http://httptest.cnodejs.net/test/post', args, function(err, data, res) {});
- *
- * @param url The URL to request, either a String or a Object that return by url.parse.
- * @param options Optional, @see RequestOptions.
- */
-export function curl<T = any>(url: string | url.URL, options?: RequestOptions): Promise<HttpClientResponse<T>>;
-/**
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function curl<T = any>(url: string | url.URL, callback: Callback<T>): void;
-/**
- * @param url The URL to request, either a String or a Object that return by url.parse.
- */
-export function curl<T = any>(url: string | url.URL, options: RequestOptions, callback: Callback<T>): void;
-/**
- * The default request timeout(in milliseconds).
- */
-export const TIMEOUT: number;
-/**
- * The default request & response timeout(in milliseconds).
- */
-export const TIMEOUTS: [number, number];
-
-/**
- * Request user agent.
- */
-export const USER_AGENT: string;
-
-/**
- * Request http agent.
- */
-export const agent: http.Agent;
-
-/**
- * Request https agent.
- */
-export const httpsAgent: https.Agent;
-
-export class HttpClient<O extends RequestOptions = RequestOptions> extends EventEmitter {
-  request<T = any>(url: string | url.URL): Promise<HttpClientResponse<T>>;
-  request<T = any>(url: string | url.URL, options: O): Promise<HttpClientResponse<T>>;
-  request<T = any>(url: string | url.URL, callback: Callback<T>): void;
-  request<T = any>(url: string | url.URL, options: O, callback: Callback<T>): void;
-
-  curl<T = any>(url: string | url.URL, options: O): Promise<HttpClientResponse<T>>;
-  curl<T = any>(url: string | url.URL): Promise<HttpClientResponse<T>>;
-  curl<T = any>(url: string | url.URL, callback: Callback<T>): void;
-  curl<T = any>(url: string | url.URL, options: O, callback: Callback<T>): void;
-
-  requestThunk(url: string | url.URL, options?: O): (callback: (...args: any[]) => void) => void;
-}
-
-export interface RequestOptions2 extends RequestOptions {
-  retry?: number;
-  retryDelay?: number;
-  isRetry?: (res: HttpClientResponse<any>) => boolean;
-}
-
-/**
- * request method only return a promise,
- * compatible with async/await and generator in co.
- */
-export class HttpClient2 extends HttpClient<RequestOptions2> {}
-
-/**
- * Create a HttpClient instance.
- */
-export function create(options?: RequestOptions): HttpClient;

package/lib/index.js

@@ -1,21 +0,0 @@
-'use strict';
-
-var urllib = require('./urllib');
-
-exports.USER_AGENT = urllib.USER_AGENT;
-exports.TIMEOUT = urllib.TIMEOUT;
-exports.TIMEOUTS = urllib.TIMEOUTS;
-exports.agent = urllib.agent;
-exports.httpsAgent = urllib.httpsAgent;
-
-exports.curl = urllib.curl;
-exports.request = urllib.request;
-exports.requestWithCallback = urllib.requestWithCallback;
-exports.requestThunk = urllib.requestThunk;
-
-exports.HttpClient = require('./httpclient');
-exports.HttpClient2 = require('./httpclient2');
-
-exports.create = function (options) {
-  return new exports.HttpClient(options);
-};

package/lib/index.test-d.ts

@@ -1,19 +0,0 @@
-import { expectType } from 'tsd';
-import { curl } from '..';
-
-// curl
-expectType<Buffer>((await curl<Buffer>('http://a.com')).data);
-// RequestOptions
-expectType<Buffer>((await curl<Buffer>('http://a.com', {})).data);
-expectType<string>((await curl<string>('http://a.com', {
-  method: 'HEAD',
-})).data);
-
-// HttpClientResponse
-const res = await curl<Buffer>('http://a.com');
-expectType<number | undefined>(res.res.timing?.queuing);
-expectType<number | undefined>(res.res.timing?.dnslookup);
-expectType<number | undefined>(res.res.timing?.connected);
-expectType<number | undefined>(res.res.timing?.requestSent);
-expectType<number | undefined>(res.res.timing?.waiting);
-expectType<number | undefined>(res.res.timing?.contentDownload);