tunnelfetch 1.0.0

package/src/pool.js

@@ -0,0 +1,256 @@
+// Connection pool for HTTP/1.1 keep-alive.
+//
+// Scope, stated up front because it is unusual: on the target runtime a socket cannot outlive the
+// request that created it — I/O objects do not cross request contexts — so this pool is
+// per-Client and deliberately not a global. The win it exists for is the crawl-shaped workload:
+// one invocation fetching thirty pages from one host pays one TLS handshake instead of thirty,
+// and a userland handshake is the single most expensive thing this package does.
+//
+// The rule that makes reuse safe, and the one that is worth getting paranoid about:
+//
+//   A connection returns to the pool ONLY after its response body has been read to the exact end
+//   the framing declared. Not "the caller stopped reading", not "it looked finished" — the body's
+//   own completion signal. Handing back a socket with unread bytes on it means the next request
+//   reads the tail of the previous response and attributes it to the wrong request, which is the
+//   worst class of bug an HTTP client can have: it corrupts data silently, under load, and only
+//   sometimes.
+//
+// The second rule follows from the first: a body framed by connection close has no determinate
+// end, so such a connection is never eligible, whatever else is true of it.
+
+import { TunnelFetchError, codes } from './errors.js';
+
+/**
+ * Everything that decides whether two requests may share a socket. Mirrors what openConnection
+ * consumed to build the connection, because anything that influenced the connection must
+ * influence the key.
+ * @typedef {object} PoolKeyInput
+ * @property {string} scheme the URL protocol, colon included ('http:' | 'https:')
+ * @property {string} hostname
+ * @property {number} port
+ * @property {import('./proxy/index.js').ProxyConfig | null | undefined} proxy
+ * @property {import('./trust/index.js').TrustConfig | null | undefined} trust
+ * @property {import('./tls/connect.js').TlsOptions | null | undefined} tls
+ */
+
+/**
+ * The trust configuration is part of the key. Two requests to the same origin under different
+ * verification policies must not share a connection — the peer was validated under one policy and
+ * silently reusing it satisfies the other policy without ever having checked it.
+ *
+ * @param {PoolKeyInput} input
+ * @returns {string}
+ */
+export function poolKey({ scheme, hostname, port, proxy, trust, tls }) {
+  const proxyPart = proxy
+    ? `${proxy.protocol}://${proxy.username ? `${proxy.username}@` : ''}${proxy.hostname}:${proxy.port}`
+    : 'direct';
+  const trustPart = trustFingerprint(trust);
+  const tlsPart = tls && Object.keys(tls).length ? JSON.stringify(sortedEntries(tls)) : '-';
+  return `${scheme}//${hostname}:${port}|${proxyPart}|${trustPart}|${tlsPart}`;
+}
+
+function trustFingerprint(trust) {
+  const mode = trust?.mode ?? 'system';
+  if (mode === 'system') return 'system';
+  if (mode === 'none') return 'none';
+  if (mode === 'custom') {
+    // Two different callbacks are two different policies and we cannot compare functions, so a
+    // custom policy never shares a connection. Correct, and cheap: custom trust is rare.
+    return `custom:${customCounter(trust.verify)}`;
+  }
+  if (mode === 'pinned') return `pinned:${[...(trust.pins ?? [])].sort().join(',')}`;
+  if (mode === 'anchors') return `anchors:${(trust.anchors ?? []).length}:${anchorDigest(trust.anchors)}`;
+  return `unknown:${mode}`;
+}
+
+const customIds = new WeakMap();
+let customSeq = 0;
+function customCounter(fn) {
+  if (typeof fn !== 'function') return 'invalid';
+  let id = customIds.get(fn);
+  if (id === undefined) {
+    id = ++customSeq;
+    customIds.set(fn, id);
+  }
+  return id;
+}
+
+const anchorDigests = new WeakMap();
+
+/**
+ * Order-sensitive digest of anchor material.
+ *
+ * Every byte is hashed. An earlier version sampled every 97th byte to save work and two anchor
+ * sets of equal length that differed only in the middle produced the same key — which would have
+ * let a connection validated against one anchor set serve a request that asked for a different
+ * one. Cheap-and-approximate is the wrong trade when the output decides whether two security
+ * policies are the same policy. The result is memoised per array, so the cost is paid once.
+ */
+function anchorDigest(anchors = []) {
+  const cached = typeof anchors === 'object' ? anchorDigests.get(anchors) : undefined;
+  if (cached !== undefined) return cached;
+
+  let h = 0x811c9dc5;
+  const mix = (byte) => {
+    h = Math.imul(h ^ byte, 0x01000193) >>> 0;
+  };
+  for (const a of anchors) {
+    const isText = typeof a === 'string';
+    const len = isText ? a.length : (a?.byteLength ?? 0);
+    // Length is folded in separately so concatenation cannot forge an equal digest.
+    mix(len & 0xff);
+    mix((len >>> 8) & 0xff);
+    mix((len >>> 16) & 0xff);
+    if (isText) for (let i = 0; i < len; i++) mix(a.charCodeAt(i) & 0xff);
+    else for (let i = 0; i < len; i++) mix(a[i]);
+    mix(0xff); // separator, so ['ab','c'] and ['a','bc'] differ
+  }
+  const out = h.toString(16);
+  if (typeof anchors === 'object') anchorDigests.set(anchors, out);
+  return out;
+}
+
+const sortedEntries = (o) => Object.entries(o).sort(([a], [b]) => (a < b ? -1 : 1));
+
+/**
+ * What the pool stores: the connection object openConnection resolves to. The pool itself only
+ * ever calls `close?.()`, but naming the real type keeps take() useful to a caller.
+ * @typedef {import('./transport.js').Connection} PooledConnection
+ */
+
+/**
+ * @typedef {object} PoolOptions
+ * @property {number} [maxPerKey] idle connections kept per key, default 6
+ * @property {number} [maxTotal] idle connections kept across all keys, default 24
+ */
+
+/**
+ * Running counters, never reset. `discarded` includes connections refused at release time;
+ * `evicted` counts victims pushed out by a newer release under a full pool.
+ * @typedef {object} PoolStats
+ * @property {number} hits
+ * @property {number} misses
+ * @property {number} released
+ * @property {number} discarded
+ * @property {number} evicted
+ */
+
+export class ConnectionPool {
+  /**
+   * @param {PoolOptions} [opts]
+   */
+  constructor({ maxPerKey = 6, maxTotal = 24 } = {}) {
+    /** @type {Map<string, Array<{conn: PooledConnection}>>} */
+    this._idle = new Map();
+    this._total = 0;
+    this._maxPerKey = maxPerKey;
+    this._maxTotal = maxTotal;
+    this._closed = false;
+    /** @type {PoolStats} */
+    this.stats = { hits: 0, misses: 0, released: 0, discarded: 0, evicted: 0 };
+  }
+
+  get idleCount() {
+    return this._total;
+  }
+
+  /**
+   * Take an idle connection for `key`, or null. Most-recently-used first: it is likeliest live.
+   * @param {string} key
+   * @returns {PooledConnection | null}
+   */
+  take(key) {
+    this._assertOpen();
+    const list = this._idle.get(key);
+    if (!list || list.length === 0) {
+      this.stats.misses++;
+      return null;
+    }
+    const entry = list.pop();
+    if (list.length === 0) this._idle.delete(key);
+    this._total--;
+    this.stats.hits++;
+    return entry.conn;
+  }
+
+  /**
+   * Offer a connection back. Callers must have proven the body reached its declared end; this
+   * method cannot verify that and deliberately does not pretend to — `eligible` is the caller's
+   * assertion, and the one place it is computed is the HTTP framing layer.
+   * Idle entries carry no age, deliberately. Ageing them out would only narrow the window in
+   * which a peer reaps a socket we still believe in, never close it — the peer can hang up at any
+   * instant, including the one after the check. What actually makes reuse safe is the recovery in
+   * sendAndReceive(): a reused connection that ends without producing one response byte is proof
+   * the request was never seen, and it is re-sent on a fresh connection. An age field would look
+   * like a second line of defence while being neither necessary nor sufficient.
+   *
+   * @param {string} key
+   * @param {PooledConnection} conn
+   * @param {boolean} eligible
+   * @returns {boolean} whether the connection was retained
+   */
+  release(key, conn, eligible) {
+    if (this._closed || !eligible) {
+      this.stats.discarded++;
+      void discard(conn);
+      return false;
+    }
+    const list = this._idle.get(key) ?? [];
+    if (list.length >= this._maxPerKey || this._total >= this._maxTotal) {
+      // Drop the oldest rather than refusing the newest: the newest is the one just proven alive.
+      const victim = list.shift() ?? null;
+      if (victim) {
+        this._total--;
+        this.stats.evicted++;
+        void discard(victim.conn);
+      } else {
+        this.stats.discarded++;
+        void discard(conn);
+        return false;
+      }
+    }
+    list.push({ conn });
+    this._idle.set(key, list);
+    this._total++;
+    this.stats.released++;
+    return true;
+  }
+
+  /**
+   * Close and forget one connection that must not be reused.
+   * @param {PooledConnection} conn
+   * @returns {Promise<void>}
+   */
+  discard(conn) {
+    this.stats.discarded++;
+    return discard(conn);
+  }
+
+  /**
+   * Close everything. A Client that is done must call this or sockets leak for the isolate.
+   * @returns {Promise<void>}
+   */
+  async closeAll() {
+    this._closed = true;
+    const all = [];
+    for (const list of this._idle.values()) for (const e of list) all.push(e.conn);
+    this._idle.clear();
+    this._total = 0;
+    await Promise.all(all.map(discard));
+  }
+
+  _assertOpen() {
+    if (this._closed) {
+      throw new TunnelFetchError(codes.POOL_CLOSED, 'the connection pool has been closed');
+    }
+  }
+}
+
+async function discard(conn) {
+  try {
+    await conn?.close?.();
+  } catch {
+    /* a connection being discarded is already suspect; its close failing changes nothing */
+  }
+}

package/src/proxy/direct.js

@@ -0,0 +1,62 @@
+// No proxy: the socket is the tunnel.
+//
+// Worth stating plainly, because it shapes what this package is for: on the target runtime a
+// direct connection cannot reach a large fraction of the public web at all — the platform refuses
+// outbound TCP to its own address ranges, and a great many sites sit behind them. Direct mode is
+// therefore useful for hosts that are demonstrably not behind the platform, and for exercising the
+// TLS stack in tests; for everything else the proxy is not an optimisation, it is the only route.
+
+import { ProxyError, codes } from '../errors.js';
+
+/**
+ * @typedef {object} DirectOptions
+ * @property {{ hostname: string, port: number }} target
+ * @property {import('./index.js').ConnectFn} connect injected socket factory
+ * @property {AbortSignal} [signal]
+ */
+
+/**
+ * Dial the target itself. Resolves with the raw socket duplex; a refused or failed dial throws
+ * ProxyError (PROXY_UNREACHABLE) quoting the runtime's own message, which is the best
+ * diagnostic a caller will get.
+ *
+ * @param {DirectOptions} args
+ * @returns {Promise<import('./index.js').Duplex>}
+ */
+export async function openDirect({ target, connect, signal }) {
+  signal?.throwIfAborted?.();
+  let socket;
+  try {
+    socket = connect({ hostname: target.hostname, port: target.port }, {
+      secureTransport: 'off',
+      allowHalfOpen: false,
+    });
+  } catch (cause) {
+    throw new ProxyError(
+      codes.PROXY_UNREACHABLE,
+      `could not open a socket to ${target.hostname}:${target.port}: ${cause?.message ?? cause}`,
+      { target: `${target.hostname}:${target.port}` },
+    );
+  }
+  if (socket.opened) {
+    try {
+      await socket.opened;
+    } catch (cause) {
+      // A failed dial still leaves a socket object behind; not closing it leaks one per attempt,
+      // which for a crawl retrying against an unreachable host adds up inside a single isolate.
+      try {
+        await socket.close?.();
+      } catch {
+        /* it never opened; close failing tells us nothing new */
+      }
+      // The runtime's refusal messages are the most useful diagnostic a caller will get here, so
+      // they are quoted rather than replaced.
+      throw new ProxyError(
+        codes.PROXY_UNREACHABLE,
+        `connection to ${target.hostname}:${target.port} failed: ${cause?.message ?? cause}`,
+        { target: `${target.hostname}:${target.port}` },
+      );
+    }
+  }
+  return socket;
+}

package/src/proxy/http-connect.js

@@ -0,0 +1,206 @@
+// HTTP CONNECT tunnelling (RFC 9110 s9.3.6; wire syntax RFC 9112).
+//
+// Two details here are not theory — they were measured against a live proxy while building this:
+//
+//   * The reply may be `HTTP/1.0 200 OK` even though we sent an HTTP/1.1 request. Matching on the
+//     version or the reason phrase would break against real deployments; only the status code is
+//     meaningful, and any 2xx opens the tunnel.
+//   * The reply header block can be as short as 19 bytes and the server may put tunnel payload in
+//     the SAME read as the terminating CRLFCRLF. Those bytes belong to the peer, not to us. Losing
+//     them silently truncates the first TLS record, which surfaces much later as an inexplicable
+//     handshake failure, so the reader that consumed the header block is handed on intact rather
+//     than being discarded.
+
+import { ProxyError, LimitError, codes } from '../errors.js';
+import { ByteReader, ByteWriter, latin1, utf8 } from '../util/bytes.js';
+
+const CRLFCRLF = utf8('\r\n\r\n');
+const MAX_REPLY_HEADER = 32 * 1024;
+
+/**
+ * The tunnel a proxy module hands back: the byte duplex plus the underlying socket, kept so a
+ * caller that must tear down the transport can reach past the wrapping streams.
+ * @typedef {import('./index.js').Duplex & { socket: import('./index.js').Duplex }} ProxyTunnel
+ */
+
+/**
+ * @typedef {object} HttpConnectOptions
+ * @property {import('./index.js').ProxyConfig} proxy protocol 'http' or 'https'
+ * @property {{ hostname: string, port: number }} target
+ * @property {import('./index.js').ConnectFn} connect injected socket factory
+ * @property {AbortSignal} [signal]
+ * @property {{ maxProxyReplyBytes?: number }} [limits] CONNECT reply head cap, default 32768
+ */
+
+/** RFC 7617: credentials are UTF-8 before base64, and btoa only accepts code units below 256. */
+function basicCredentials(username, password) {
+  const raw = utf8(`${username}:${password ?? ''}`);
+  return btoa(latin1(raw));
+}
+
+/** IPv6 literals must be bracketed in an authority, or the port cannot be told from the address. */
+function authority(hostname, port) {
+  return hostname.includes(':') ? `[${hostname}]:${port}` : `${hostname}:${port}`;
+}
+
+/**
+ * Establish a CONNECT tunnel through an http/https proxy. Resolves with the tunnel duplex;
+ * every refusal (non-2xx, 407 with or without credentials, malformed reply) throws a
+ * ProxyError naming what the proxy answered.
+ *
+ * @param {HttpConnectOptions} args
+ * @returns {Promise<ProxyTunnel>}
+ */
+export async function openHttpConnect({ proxy, target, connect, signal, limits = {} }) {
+  signal?.throwIfAborted?.();
+  const overTls = proxy.protocol === 'https';
+  const where = `${proxy.hostname}:${proxy.port}`;
+
+  let socket;
+  try {
+    socket = connect(
+      { hostname: proxy.hostname, port: proxy.port },
+      // For an https proxy the runtime does TLS to the proxy itself. That is the one hop where
+      // the platform's certificate check asks the right question: the identity it verifies is the
+      // hostname handed to connect(), which here IS the proxy. (It follows that an https proxy
+      // addressed by bare IP will fail that check, since public certificates carry no IP SAN.)
+      { secureTransport: overTls ? 'on' : 'starttls', allowHalfOpen: false },
+    );
+  } catch (cause) {
+    throw new ProxyError(
+      codes.PROXY_UNREACHABLE,
+      `could not open a socket to proxy ${where}: ${cause?.message ?? cause}`,
+      { proxy: where },
+    );
+  }
+
+  let reader;
+  let writer;
+  try {
+    if (socket.opened) await socket.opened;
+
+    writer = new ByteWriter(socket.writable);
+    await writer.write(utf8(buildConnectRequest(proxy, target)));
+    writer.releaseLock();
+
+    reader = new ByteReader(socket.readable);
+    const max = limits.maxProxyReplyBytes ?? MAX_REPLY_HEADER;
+    let block;
+    try {
+      block = await reader.readUntil(CRLFCRLF, max, 'proxy CONNECT reply');
+    } catch (cause) {
+      if (cause instanceof LimitError) throw cause;
+      throw new ProxyError(
+        codes.PROXY_PROTOCOL,
+        `proxy ${where} closed or misbehaved before completing its CONNECT reply: ${cause?.message ?? cause}`,
+        { proxy: where },
+      );
+    }
+    const reply = parseReply(latin1(block), where);
+
+    if (reply.status < 200 || reply.status > 299) {
+      throw replyError(reply, proxy, target, where);
+    }
+    // Anything the proxy sent after the blank line is already tunnel payload. It stays buffered in
+    // `reader`, which becomes the tunnel's read side, so it cannot be dropped.
+    return tunnelFrom(socket, reader);
+  } catch (err) {
+    try {
+      writer?.releaseLock();
+      await reader?.cancel(err);
+      await socket.close?.();
+    } catch {
+      /* the socket may already be unusable; the original error is what matters */
+    }
+    throw err;
+  }
+}
+
+function buildConnectRequest(proxy, target) {
+  const host = authority(target.hostname, target.port);
+  const lines = [`CONNECT ${host} HTTP/1.1`, `Host: ${host}`];
+  if (proxy.username) {
+    lines.push(`Proxy-Authorization: Basic ${basicCredentials(proxy.username, proxy.password)}`);
+  }
+  // Some proxies still key off the pre-standard hop header; sending it is harmless and avoids a
+  // class of proxy that closes the tunnel after one request without it.
+  lines.push('Proxy-Connection: keep-alive');
+  return `${lines.join('\r\n')}\r\n\r\n`;
+}
+
+function parseReply(text, where) {
+  const [statusLine, ...headerLines] = text.split('\r\n');
+  const m = /^HTTP\/(\d)\.(\d) (\d{3})(?: (.*))?$/.exec(statusLine ?? '');
+  if (!m) {
+    throw new ProxyError(
+      codes.PROXY_PROTOCOL,
+      `proxy ${where} did not answer CONNECT with an HTTP status line (got ${JSON.stringify(
+        (statusLine ?? '').slice(0, 80),
+      )})`,
+      { proxy: where, statusLine },
+    );
+  }
+  const headers = new Map();
+  for (const line of headerLines) {
+    if (line === '') break;
+    const i = line.indexOf(':');
+    if (i <= 0) continue; // a malformed header in an otherwise valid reply is not worth failing on
+    headers.set(line.slice(0, i).toLowerCase().trim(), line.slice(i + 1).trim());
+  }
+  return {
+    httpVersion: `${m[1]}.${m[2]}`,
+    status: Number(m[3]),
+    reason: m[4] ?? '',
+    headers,
+    statusLine,
+  };
+}
+
+function replyError(reply, proxy, target, where) {
+  const to = authority(target.hostname, target.port);
+  if (reply.status === 407) {
+    const challenge = reply.headers.get('proxy-authenticate') ?? '(none sent)';
+    // Naming the scheme the proxy asked for is the whole value of this error: "auth failed" does
+    // not tell anyone whether they typed the password wrong or the proxy wants Digest/NTLM.
+    return proxy.username
+      ? new ProxyError(
+          codes.PROXY_AUTH_FAILED,
+          `proxy ${where} rejected the supplied credentials for user "${proxy.username}" ` +
+            `(407 ${reply.reason}); it offers: ${challenge}`,
+          { proxy: where, challenge, status: 407 },
+        )
+      : new ProxyError(
+          codes.PROXY_AUTH_REQUIRED,
+          `proxy ${where} requires authentication (407 ${reply.reason}); it offers: ${challenge}`,
+          { proxy: where, challenge, status: 407 },
+        );
+  }
+  return new ProxyError(
+    codes.PROXY_CONNECT_REFUSED,
+    `proxy ${where} refused CONNECT to ${to}: ${reply.status} ${reply.reason}`.trimEnd(),
+    { proxy: where, target: to, status: reply.status, reason: reply.reason },
+  );
+}
+
+/**
+ * Wrap the still-buffered reader as the tunnel's readable so bytes that arrived alongside the
+ * CONNECT reply are delivered first, in order.
+ */
+function tunnelFrom(socket, reader) {
+  return {
+    readable: new ReadableStream({
+      async pull(controller) {
+        const chunk = await reader.readSome();
+        if (chunk === null) controller.close();
+        else controller.enqueue(chunk);
+      },
+      cancel(reason) {
+        return reader.cancel(reason);
+      },
+    }),
+    writable: socket.writable,
+    opened: socket.opened,
+    close: () => socket.close?.(),
+    socket,
+  };
+}