tunnelfetch 1.9.0 → 1.11.0

package/README.md

@@ -470,18 +470,45 @@ extension **set** that is curl's. A test reads that gap straight out of the comm
 capture and fails if it changes, so it cannot drift quietly — but it is a real difference and a
 JA3/JA4 hash sees it.
 
+`tls.omitExtensions` is the subtractive counterpart, and `status_request` (5) is what it exists for:
+that is the one extension this package sends which curl does not, so an identity matching a sample
+without it had no way to drop it. Dropping it gives up OCSP stapling — the only revocation signal
+this package consumes — so pairing it with `trust.revocation: 'require-staple'` is refused at
+configuration time rather than left to fail every connection on a certificate that was never asked
+to carry a staple.
+
 `tls.extraExtensions` takes pre-encoded extensions and is the only way to close it. They are ordered
 like any other and reproduced on a HelloRetryRequest retry, because a second hello that changed its
 extension set would be both malformed (RFC 8446 §4.1.2) and a signal in itself. Encoding them
 correctly is the caller's job; this package does not parse what it did not build.
 
-**Every offered cipher must be one this package can perform.** `tls.ciphers` used to be taken
-verbatim, so a list containing a CBC or RSA-key-exchange suite put a number on the wire that a
-server could select — after which the AEAD layer had nothing to build and the connection died
-mid-handshake. Such a list is now refused at configuration time. An explicit `TLS_CHACHA20_POLY1305_SHA256`
-without an injected implementation is refused too, rather than silently dropped: quietly presenting
-a different fingerprint from the one you asked for is the worst outcome available to a package like
-this one.
+**Every offered cipher must be one this package can perform — unless you say otherwise.**
+`tls.ciphers` used to be taken verbatim, so a list containing a CBC or RSA-key-exchange suite put a
+number on the wire that a server could select, after which the AEAD layer had nothing to build and
+the connection died mid-handshake. Such a list is refused at configuration time by default, and an
+explicit `TLS_CHACHA20_POLY1305_SHA256` with no injected implementation is refused rather than
+silently dropped — quietly presenting a different fingerprint from the one you asked for is the
+worst outcome available to a package like this one.
+
+But refusing outright is the wrong default to have no escape from, because **the restriction is
+itself a fingerprint**:
+
+| | suites offered | performable here |
+| --- | --- | --- |
+| curl 8.21.0 | 30 | 7 |
+| Chromium | 15 | 7 |
+
+A hello restricted to what can be honoured carries a cipher list less than half the length of any
+real client's, and list length and contents are exactly what a JA3 hash reads. `tls.allowUnperformableCiphers`
+offers the accurate list. The trade is narrow: the first unperformable suite sits at index 5 of
+curl's list and 7 of Chromium's, behind the TLS 1.3 suites, so a server with 1.3 available never
+reaches one. If a server does select one the handshake fails, and the error names the option rather
+than reading like a defect here.
+
+Most of the gap is not a missing feature. Sixteen of curl's twenty-three are CBC — MAC-then-encrypt,
+which cannot be implemented without a Lucky13 padding oracle in JavaScript — and two more are RSA key
+exchange with no forward secrecy. Both are refusals this package intends to keep. The three that
+*are* implementable are the TLS 1.2 ChaCha20-Poly1305 suites, and they are not implemented yet.
 
 Extension order matters because JA3 and JA4 hash the extension list **in wire order**, so it is most
 of what a fingerprinter reads. `pre_shared_key` is forced last whatever you ask for: RFC 8446
@@ -578,6 +605,20 @@ new Client({ connect, proxy, maxBodyBytes: Infinity });   // or any number you h
 The trade is deliberate: an unasked-for limit is discoverable the first time it bites, and names the
 option in its error. An unasked-for OOM is neither.
 
+#### The proxy sees a fingerprint too
+
+`Proxy-Connection` is a pre-standard hop header that never reached a spec. The origin never sees it;
+the proxy always does. Clients disagree — some send `keep-alive`, some `close`, some omit it — so for
+anyone matching a client's behaviour *at the proxy* it is part of the fingerprint, and it used to be
+hard-coded.
+
+```js
+new Client({ connect, proxy: { ...cfg, proxyConnection: 'close' } });  // or null to omit it
+```
+
+The default stays `keep-alive`, which avoids a class of proxy that closes the tunnel after one
+request. Omitting the header is not the same as sending `close`.
+
 ### Trust — the `verify=` knob
 
 ```js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tunnelfetch",
-  "version": "1.9.0",
+  "version": "1.11.0",
   "description": "A fetch-shaped HTTP client that can route through HTTP CONNECT / HTTPS / SOCKS5 proxies on runtimes with only raw TCP, such as Cloudflare Workers. Implements TLS in userland because the runtime cannot verify a tunnelled peer.",
   "keywords": [
     "fetch",

package/src/proxy/http-connect.js

@@ -122,9 +122,15 @@ function buildConnectRequest(proxy, target) {
   if (proxy.username) {
     lines.push(`Proxy-Authorization: Basic ${basicCredentials(proxy.username, proxy.password)}`);
   }
-  // Some proxies still key off the pre-standard hop header; sending it is harmless and avoids a
-  // class of proxy that closes the tunnel after one request without it.
-  lines.push('Proxy-Connection: keep-alive');
+  // `Proxy-Connection` is a pre-standard hop header that never made it into a spec, and clients
+  // disagree about it: some send `keep-alive`, some send `close`, some omit it. The proxy sees this
+  // even though the origin never does, so for anyone matching a client's behaviour AT THE PROXY it
+  // is part of the fingerprint. `keep-alive` stays the default — it avoids a class of proxy that
+  // closes the tunnel after one request — but it is no longer fixed.
+  //
+  // `null` omits the header entirely, which is not the same as sending `close`.
+  const pc = proxy.proxyConnection === undefined ? 'keep-alive' : proxy.proxyConnection;
+  if (pc !== null) lines.push(`Proxy-Connection: ${pc}`);
   return `${lines.join('\r\n')}\r\n\r\n`;
 }
 

package/src/proxy/index.js

@@ -19,7 +19,13 @@ import { openSocks5 } from './socks5.js';
  * @typedef {(addr: {hostname: string, port: number},
  *            opts?: {secureTransport?: 'off'|'on'|'starttls', allowHalfOpen?: boolean}) => Duplex} ConnectFn
  * @typedef {{ protocol: 'http'|'https'|'socks5'|'socks5h', hostname: string, port: number,
- *             username?: string, password?: string }} ProxyConfig
+ *             username?: string, password?: string,
+ *             proxyConnection?: string | null }} ProxyConfig
+ *
+ * `proxyConnection` sets the pre-standard `Proxy-Connection` header on a CONNECT request, or
+ * omits it entirely when null. Default 'keep-alive'. The origin never sees this header; the
+ * proxy does, so it belongs to whatever fingerprint the proxy is reading. Clients disagree —
+ * some send keep-alive, some close, some nothing — and omitting is not the same as 'close'.
  */
 
 const DEFAULT_PORTS = { http: 8080, https: 443, socks5: 1080, socks5h: 1080 };
@@ -114,6 +120,10 @@ function normalise(cfg) {
     port,
     username: cfg.username || undefined,
     password: cfg.password || undefined,
+    // Listed explicitly because this function REBUILDS the config rather than copying it, so a
+    // field not named here is dropped without a word. That is how `http2ConnectionWindow` came to
+    // be declared in a profile and read by nothing.
+    proxyConnection: cfg.proxyConnection,
   });
 }
 

package/src/tls/connect.js

@@ -137,9 +137,23 @@ function expectServerHello(msg, offers12) {
  * @property {number[]} [groups] supported_groups, in preference order.
  * @property {number[]} [offerGroups] groups to send an actual key_share for. Default the first
  *   supported group; a HelloRetryRequest recovers any other choice at the cost of a round trip.
- * @property {number[]} [ciphers] cipher suites to offer, in preference order. Every suite must be
- *   one this package can perform; an offer it cannot honour is a dead connection the moment a
- *   server selects it, so an unknown suite is refused here rather than on the wire.
+ * @property {number[]} [ciphers] cipher suites to offer, in preference order. By default every
+ *   suite must be one this package can perform: an offer it cannot honour is a dead connection the
+ *   moment a server selects it, so an unknown suite is refused here rather than on the wire.
+ * @property {number[]} [omitExtensions] extension types to leave out of the ClientHello, the
+ *   subtractive counterpart to `extraExtensions`. `status_request` (5) is the one extension this
+ *   package sends that curl does not, so an identity matching a sample without it needs this.
+ *   Dropping it gives up OCSP stapling, which is the only revocation signal this package can
+ *   consume — pairing it with `trust.revocation: 'require-staple'` is refused rather than left to
+ *   fail every connection.
+ * @property {boolean} [allowUnperformableCiphers] offer suites this package cannot complete.
+ *   For fingerprint fidelity only. Real clients offer far more than this package implements — curl
+ *   8.21.0 offers thirty against seven performable here, Chromium fifteen against seven — so a
+ *   hello restricted to what it can honour carries a cipher list shorter than any real client's,
+ *   which is exactly what a JA3 hash reads. With this set, a server that selects an unperformable
+ *   suite fails the handshake; the first such suite sits behind the TLS 1.3 ones in both real
+ *   lists, so a 1.3-capable server does not reach it. Knowingly trading a rare failure for an
+ *   accurate fingerprint is a legitimate choice; making it silently is not.
  * @property {Uint8Array[]} [extraExtensions] pre-encoded ClientHello extensions, appended before
  *   ordering. `extensionOrder` can only arrange extensions that were BUILT — it filters to what
  *   exists and sorts that — so ordering alone cannot produce an extension this package does not
@@ -347,7 +361,16 @@ async function drive({ record, hostname, verifyPeer, options, deps, versions })
   // package cannot perform is an offer a server may take and then find unhonoured, which fails the
   // connection rather than merely looking wrong". It applies just as much to `tls.ciphers`, and
   // now does. CIPHER_PARAMS is the set this package knows how to key and seal.
-  if (options.ciphers) {
+  //
+  // `allowUnperformableCiphers` opts out, and it exists because refusing outright was the wrong
+  // default to have no escape from. Every real client offers far more suites than this package
+  // implements — curl 8.21.0 offers thirty and seven are performable here; Chromium offers fifteen
+  // and seven are — so a hello restricted to what can be honoured has a cipher list shorter than
+  // any real client's, and that is itself what a JA3 hash reads. A caller matching a fingerprint
+  // may rationally prefer the accurate list: the first unperformable suite sits at index 5 of
+  // curl's and 7 of Chromium's, behind the TLS 1.3 suites, so a server with 1.3 available never
+  // reaches it. The risk is real but narrow, and it is the caller's to take knowingly.
+  if (options.ciphers && !options.allowUnperformableCiphers) {
     const unperformable = ciphers.filter((c) => !CIPHER_PARAMS[c]);
     if (unperformable.length) {
       throw new TlsError(
@@ -414,6 +437,7 @@ async function drive({ record, hostname, verifyPeer, options, deps, versions })
     alpn,
     ciphers,
     extraExtensions: options.extraExtensions ?? [],
+    omitExtensions: options.omitExtensions ?? [],
     versions,
     extensionOrder: options.extensionOrder,
     sigSchemes: options.sigSchemes,

package/src/tls/handshake-messages.js

@@ -192,6 +192,7 @@ export async function deriveSharedSecret(group, privateKey, peerKey, deps = {})
  * @property {string[]} [alpn] default ['http/1.1']; empty array omits the extension
  * @property {number[]} [versions] default [TLS13, TLS12]
  * @property {Uint8Array[]} [extraExtensions] pre-encoded, sent verbatim (the HRR cookie)
+ * @property {number[]} [omitExtensions] extension types to leave out of the hello
  * @property {{ identity: Uint8Array, obfuscatedTicketAge: number, binderLen: number }} [psk]
  *   offer this resumption PSK. Encoded with a zeroed binder placeholder; the caller MUST derive
  *   the real binder over `message.subarray(0, truncatedLength)` and patch it in at
@@ -299,6 +300,7 @@ export function buildClientHello({
   versions = [TLS13, TLS12],
   extensionOrder = CURL_EXTENSION_ORDER,
   extraExtensions = [],
+  omitExtensions = [],
   psk = null,
   grease = false,
   randomBytes = defaultRandom,
@@ -341,11 +343,15 @@ export function buildClientHello({
       'a resumption PSK was supplied but TLS 1.3 is not among the offered versions');
   }
 
+  const omit = new Set(omitExtensions);
   const extensionParts = [
     encodeServerName(hostname),
-    // Always offered, for either version: without it a server may not staple (RFC 6066 s8), and
-    // a stapled OCSP response is the only revocation signal this package can consume.
-    encodeStatusRequest(),
+    // Offered by default for either version: without it a server may not staple (RFC 6066 s8), and
+    // a stapled OCSP response is the only revocation signal this package can consume. It is also
+    // the one extension this package sends that curl does not, so an identity matching a sample
+    // without it needs a way to drop it — `omitExtensions` is the subtractive counterpart to
+    // `extraExtensions`, and dropping this one gives up stapling.
+    omit.has(EXTENSION.status_request) ? null : encodeStatusRequest(),
     encodeSupportedGroups(g ? [g.take(), ...groups] : groups),
     encodeSignatureAlgorithms(sigSchemes),
     alpn.length ? encodeAlpn(alpn) : null,
@@ -589,7 +595,10 @@ export function negotiateCipher(serverHello, { offeredCiphers, version }) {
   if (!params) {
     throw new TlsUnsupportedError(
       codes.TLS_CIPHER_UNSUPPORTED,
-      `cipher suite ${hex16(suite)} has no parameters; this is a bug in the offer list`,
+      `cipher suite ${hex16(suite)} was offered but this package cannot perform it, so the ` +
+        'handshake cannot continue. Either the offer list has a bug, or `allowUnperformableCiphers` ' +
+        'was set to match a real client\'s cipher list and a server has selected one of the suites ' +
+        'that choice knowingly put on the wire.',
       { cipherSuite: suite },
     );
   }

package/src/tls/handshake.js

@@ -250,6 +250,7 @@ export async function continueTls13(ctx) {
       // not permit a second hello to change its extension SET, and a retry that quietly dropped
       // them would present one fingerprint on the first flight and a different one on the second —
       // a difference that is itself a signal.
+      omitExtensions: options.omitExtensions ?? [],
       extraExtensions: [
         ...(options.extraExtensions ?? []),
         ...(cookie ? [cookieExtension(cookie)] : []),

package/src/transport.js

@@ -124,6 +124,20 @@ export async function openConnection({
 }) {
   const target = targetFromUrl(url);
   const proxyConfig = parseProxy(proxy);
+
+  // Dropping status_request gives up OCSP stapling, and stapling is the only revocation signal this
+  // package consumes. Combined with `require-staple` that is not a weakened check, it is a policy
+  // that can never be satisfied: every connection would fail with OCSP_REQUIRED, on a certificate
+  // that was never asked to carry a staple. Refused here, where both settings are visible.
+  if ((tls?.omitExtensions ?? []).includes(5) && trust?.revocation === 'require-staple') {
+    throw new ConfigError(
+      codes.CONFIG_INVALID,
+      "tls.omitExtensions drops status_request (5) while trust.revocation is 'require-staple'. " +
+        'Without status_request a server is not asked to staple, so no staple can arrive and every ' +
+        'connection would fail. Keep the extension, or relax the revocation policy — the two are ' +
+        'a contradiction rather than a stricter setting.',
+    );
+  }
   const owns = !deadlines;
   const dl = deadlines ?? new DeadlineController({}, { signal });
 

package/types/proxy/index.d.ts

@@ -49,12 +49,19 @@ export type ConnectFn = (addr: {
     secureTransport?: "off" | "on" | "starttls";
     allowHalfOpen?: boolean;
 }) => Duplex;
+/**
+ * `proxyConnection` sets the pre-standard `Proxy-Connection` header on a CONNECT request, or
+ * omits it entirely when null. Default 'keep-alive'. The origin never sees this header; the
+ * proxy does, so it belongs to whatever fingerprint the proxy is reading. Clients disagree —
+ * some send keep-alive, some close, some nothing — and omitting is not the same as 'close'.
+ */
 export type ProxyConfig = {
     protocol: "http" | "https" | "socks5" | "socks5h";
     hostname: string;
     port: number;
     username?: string;
     password?: string;
+    proxyConnection?: string | null;
 };
 import { openDirect } from './direct.js';
 import { openHttpConnect } from './http-connect.js';

package/types/tls/connect.d.ts

@@ -13,9 +13,23 @@
  * @property {number[]} [groups] supported_groups, in preference order.
  * @property {number[]} [offerGroups] groups to send an actual key_share for. Default the first
  *   supported group; a HelloRetryRequest recovers any other choice at the cost of a round trip.
- * @property {number[]} [ciphers] cipher suites to offer, in preference order. Every suite must be
- *   one this package can perform; an offer it cannot honour is a dead connection the moment a
- *   server selects it, so an unknown suite is refused here rather than on the wire.
+ * @property {number[]} [ciphers] cipher suites to offer, in preference order. By default every
+ *   suite must be one this package can perform: an offer it cannot honour is a dead connection the
+ *   moment a server selects it, so an unknown suite is refused here rather than on the wire.
+ * @property {number[]} [omitExtensions] extension types to leave out of the ClientHello, the
+ *   subtractive counterpart to `extraExtensions`. `status_request` (5) is the one extension this
+ *   package sends that curl does not, so an identity matching a sample without it needs this.
+ *   Dropping it gives up OCSP stapling, which is the only revocation signal this package can
+ *   consume — pairing it with `trust.revocation: 'require-staple'` is refused rather than left to
+ *   fail every connection.
+ * @property {boolean} [allowUnperformableCiphers] offer suites this package cannot complete.
+ *   For fingerprint fidelity only. Real clients offer far more than this package implements — curl
+ *   8.21.0 offers thirty against seven performable here, Chromium fifteen against seven — so a
+ *   hello restricted to what it can honour carries a cipher list shorter than any real client's,
+ *   which is exactly what a JA3 hash reads. With this set, a server that selects an unperformable
+ *   suite fails the handshake; the first such suite sits behind the TLS 1.3 ones in both real
+ *   lists, so a 1.3-capable server does not reach it. Knowingly trading a rare failure for an
+ *   accurate fingerprint is a legitimate choice; making it silently is not.
  * @property {Uint8Array[]} [extraExtensions] pre-encoded ClientHello extensions, appended before
  *   ordering. `extensionOrder` can only arrange extensions that were BUILT — it filters to what
  *   exists and sorts that — so ordering alone cannot produce an extension this package does not
@@ -169,11 +183,31 @@ export type TlsOptions = {
      */
     offerGroups?: number[] | undefined;
     /**
-     * cipher suites to offer, in preference order. Every suite must be
-     * one this package can perform; an offer it cannot honour is a dead connection the moment a
-     * server selects it, so an unknown suite is refused here rather than on the wire.
+     * cipher suites to offer, in preference order. By default every
+     * suite must be one this package can perform: an offer it cannot honour is a dead connection the
+     * moment a server selects it, so an unknown suite is refused here rather than on the wire.
      */
     ciphers?: number[] | undefined;
+    /**
+     * extension types to leave out of the ClientHello, the
+     * subtractive counterpart to `extraExtensions`. `status_request` (5) is the one extension this
+     * package sends that curl does not, so an identity matching a sample without it needs this.
+     * Dropping it gives up OCSP stapling, which is the only revocation signal this package can
+     * consume — pairing it with `trust.revocation: 'require-staple'` is refused rather than left to
+     * fail every connection.
+     */
+    omitExtensions?: number[] | undefined;
+    /**
+     * offer suites this package cannot complete.
+     * For fingerprint fidelity only. Real clients offer far more than this package implements — curl
+     * 8.21.0 offers thirty against seven performable here, Chromium fifteen against seven — so a
+     * hello restricted to what it can honour carries a cipher list shorter than any real client's,
+     * which is exactly what a JA3 hash reads. With this set, a server that selects an unperformable
+     * suite fails the handshake; the first such suite sits behind the TLS 1.3 ones in both real
+     * lists, so a 1.3-capable server does not reach it. Knowingly trading a rare failure for an
+     * accurate fingerprint is a legitimate choice; making it silently is not.
+     */
+    allowUnperformableCiphers?: boolean | undefined;
     /**
      * pre-encoded ClientHello extensions, appended before
      * ordering. `extensionOrder` can only arrange extensions that were BUILT — it filters to what

package/types/tls/handshake-messages.d.ts

@@ -32,7 +32,7 @@ export function generateKeyShare(group: number, deps?: import("./connect.js").Tl
  * @returns {Promise<Uint8Array>} throws on any degenerate or malformed peer key
  */
 export function deriveSharedSecret(group: number, privateKey: CryptoKey | import("./hybrid.js").HybridPrivate, peerKey: Uint8Array, deps?: import("./connect.js").TlsDeps): Promise<Uint8Array>;
-export function buildClientHello({ hostname, keyShares, random, legacySessionId, ciphers, groups, sigSchemes, alpn, versions, extensionOrder, extraExtensions, psk, grease, randomBytes, }: {
+export function buildClientHello({ hostname, keyShares, random, legacySessionId, ciphers, groups, sigSchemes, alpn, versions, extensionOrder, extraExtensions, omitExtensions, psk, grease, randomBytes, }: {
     hostname: any;
     keyShares: any;
     random: any;
@@ -44,6 +44,7 @@ export function buildClientHello({ hostname, keyShares, random, legacySessionId,
     versions?: number[] | undefined;
     extensionOrder?: readonly number[] | undefined;
     extraExtensions?: never[] | undefined;
+    omitExtensions?: never[] | undefined;
     psk?: null | undefined;
     grease?: boolean | undefined;
     randomBytes?: ((n: any) => Uint8Array<any>) | undefined;
@@ -305,6 +306,7 @@ export function checkAlpn(extensions: Map<number, Uint8Array>, offeredAlpn: stri
  * @property {string[]} [alpn] default ['http/1.1']; empty array omits the extension
  * @property {number[]} [versions] default [TLS13, TLS12]
  * @property {Uint8Array[]} [extraExtensions] pre-encoded, sent verbatim (the HRR cookie)
+ * @property {number[]} [omitExtensions] extension types to leave out of the hello
  * @property {{ identity: Uint8Array, obfuscatedTicketAge: number, binderLen: number }} [psk]
  *   offer this resumption PSK. Encoded with a zeroed binder placeholder; the caller MUST derive
  *   the real binder over `message.subarray(0, truncatedLength)` and patch it in at
@@ -432,6 +434,10 @@ export type ClientHelloOptions = {
      * pre-encoded, sent verbatim (the HRR cookie)
      */
     extraExtensions?: Uint8Array<ArrayBufferLike>[] | undefined;
+    /**
+     * extension types to leave out of the hello
+     */
+    omitExtensions?: number[] | undefined;
     /**
      * offer this resumption PSK. Encoded with a zeroed binder placeholder; the caller MUST derive
      * the real binder over `message.subarray(0, truncatedLength)` and patch it in at