tr-fetch 0.9.0 → 0.9.2

package/README.md

@@ -28,6 +28,7 @@ const response = await trFetch(url, {
 	trFetchDebug: false,
 	trFetchCrlPolicy: {
 		disabled: false,
+		maxCrlBytes: 16777216,
 		missingCrlDistributionPoint: 'ignore',
 		unreachableCrlDistributionPoint: 'reject',
 		invalidCrl: 'reject',
@@ -65,6 +66,23 @@ continues. `reject` rejects the fetch promise before sending the HTTP request
 on that connection. These policies apply only to revocation checks; they cannot
 relax normal TLS verification.
 
+### CRL size limit
+
+`trFetchCrlPolicy.maxCrlBytes` is the largest CRL accepted, in bytes. It
+defaults to `16777216` (16 MiB) and must be a positive safe integer; there is
+no value for unlimited. Some public CAs publish much larger CRLs, so raise it
+explicitly when needed:
+
+```js
+await trFetch('https://example.com/', { trFetchCrlPolicy: { maxCrlBytes: 64 * 1024 * 1024 } });
+```
+
+A larger download is an unreachable distribution point; larger
+`trFetchCrlOverride` data is an invalid CRL. A cached CRL above the caller's
+limit is not used, even if an earlier caller with a higher limit cached it.
+Revoked entries are scanned in place, so memory use stays close to the CRL's
+size.
+
 ### Disabling a check
 
 Both policy objects accept `disabled`. It defaults to `false`; `undefined` and
@@ -305,7 +323,7 @@ CRL retrieval:
   for HTTPS downloads. Download failures use `unreachableCrlDistributionPoint`.
 - Does not copy application cookies, authorization headers or request bodies
   into CRL requests.
-- Allows at most five redirects, 16 MiB of decoded response data and ten seconds
+- Allows at most five redirects, `maxCrlBytes` of decoded response data and ten seconds
   per download including redirects and body reading. Caller cancellation also
   cancels CRL retrieval. At most 32 distribution URIs are tried per certificate.
 - Allows private-network HTTP(S) endpoints, including loopback, for private PKI.
@@ -388,7 +406,14 @@ checks. Only `http:` and `https:` URLs are supported; a URL without a scheme
 defaults to `http://`.
 
 ```sh
-npx tr-curl -v https://example.com/
+npx -p tr-fetch tr-curl -v https://example.com/
+```
+
+The command is in the `tr-fetch` package, so `npx` needs `-p tr-fetch`.
+After `npm install -g tr-fetch`, or inside a project that depends on it,
+`tr-curl` can be run directly:
+
+```sh
 tr-curl -fsSL -o page.html https://example.com/
 tr-curl --json '{"a":1}' -u user:password https://api.example.com/items
 tr-curl --cacert private-ca.pem --crlfile current.crl https://internal.example/

package/check.js

@@ -92,8 +92,14 @@ async function checkCertificate(peer, hostname, options, signal, override, debug
                     const url = networkUrl(source);
                     key = issuerId + ':' + url.href;
                     crl = cache.get(key, options.cacheSize, options.cacheTTL, Date.now(), debug);
+                    // A CRL cached under a higher limit must not bypass this one.
+                    if (crl && (crl.size > options.policy.maxCrlBytes)) {
+                        debug?.('CRL cache entry not used', { source: debugUrl(url), reason: 'exceeds maxCrlBytes',
+                                                              bytes: crl.size, maxCrlBytes: options.policy.maxCrlBytes });
+                        crl = undefined;
+                    }
                     if (! crl) {
-                        bytes = await downloadCrl(url, signal, debug);
+                        bytes = await downloadCrl(url, signal, debug, options.policy.maxCrlBytes);
                     }
                 } catch (cause) {
                     signal?.throwIfAborted();
@@ -105,9 +111,9 @@ async function checkCertificate(peer, hostname, options, signal, override, debug
             let result;
             try {
                 if (! crl) {
-                    crl = parseCrl(bytes);
+                    crl = parseCrl(bytes, options.policy.maxCrlBytes);
                     debug?.('CRL parsed', { source: (source === undefined) ? 'trFetchCrlOverride' : debugUrl(source),
-                                            revokedEntries: crl.revokedCertificates?.length ?? 0 });
+                                            revokedEntries: crl.revokedCount });
                 }
                 result = await validateCrl(crl, certificate, issuer, point.urls);
             } catch (cause) {

package/crl.js

@@ -2,9 +2,11 @@
 
 const asn1 = require('asn1js');
 const pki = require('pkijs');
-const { cryptoEngine, parseDer, extensionsById, extensionValue, parseCertificate } = require('./pkiutils');
+const { cryptoEngine, parseDer, derElement, derHeader, oidString, extensionsById, extensionValue,
+    parseCertificate } = require('./pkiutils');
+const { DEFAULT_MAX_CRL_BYTES } = require('./options');
 
-const MAX_CRL_BYTES = 16 * 1024 * 1024;
+const TIME_TAGS = [ 0x17, 0x18 ];
 
 function distributionPoints(certificate) {
     const extension = extensionsById(certificate.extensions).get('2.5.29.31');
@@ -24,10 +26,11 @@ function distributionPoints(certificate) {
     });
 }
 
-function parseCrl(bytes) {
-    if (bytes.length > MAX_CRL_BYTES) {
-        throw new Error('CRL exceeds the 16 MiB size limit');
+function parseCrl(bytes, maxBytes = DEFAULT_MAX_CRL_BYTES) {
+    if (bytes.length > maxBytes) {
+        throw new Error(`CRL exceeds maxCrlBytes (${maxBytes} bytes)`);
     }
+    const size = bytes.length;
     if (bytes.toString('ascii', 0, 32).trimStart().startsWith('-----BEGIN')) {
         const match = /^\s*-----BEGIN X509 CRL-----\s*([A-Za-z0-9+/=\r\n\t ]+)\s*-----END X509 CRL-----\s*$/.exec(bytes.toString('ascii'));
         if (! match) {
@@ -39,7 +42,110 @@ function parseCrl(bytes) {
             throw new Error('Malformed base64 in PEM CRL');
         }
     }
-    return parseDer(bytes, pki.CertificateRevocationList);
+    const outer = derElement(bytes, 0);
+    if (outer.end !== bytes.length) {
+        throw new Error('Malformed ASN.1 data: trailing bytes');
+    }
+    const tbs = derElement(bytes, outer.start, outer.end);
+    if ((outer.tag !== 0x30) || (tbs.tag !== 0x30)) {
+        throw new Error('Malformed CRL structure');
+    }
+    const fields = [];
+    for (let offset = tbs.start; offset < tbs.end; offset = fields.at(-1).end) {
+        fields.push(derElement(bytes, offset, tbs.end));
+    }
+    // version, signature, issuer, thisUpdate, nextUpdate, revokedCertificates
+    let index = (fields[0]?.tag === 0x02) ? 3 : 2;
+    index += TIME_TAGS.includes(fields[index + 1]?.tag) ? 2 : 1;
+    const entries = (fields[index]?.tag === 0x30) ? fields[index] : undefined;
+    // Decoding every revoked entry into an ASN.1 object tree takes hundreds of
+    // bytes of memory per encoded byte, so large CRLs could exhaust the heap.
+    // PKI.js decodes the rest; the entries are scanned in place.
+    const headerFields = fields.filter(x => x !== entries).map(x => bytes.subarray(x.offset, x.end));
+    const headerLength = headerFields.reduce((sum, x) => sum + x.length, 0);
+    const trailer = bytes.subarray(tbs.end, outer.end);
+    const tbsHeader = derHeader(0x30, headerLength);
+    const crl = parseDer(Buffer.concat([ derHeader(0x30, tbsHeader.length + headerLength + trailer.length),
+        tbsHeader, ...headerFields, trailer ]), pki.CertificateRevocationList);
+    const scan = scanEntries(bytes, entries);
+    return { crl, size, bytes, tbs: bytes.subarray(tbs.offset, tbs.end), entries,
+        revokedCount: scan.count, entryProblem: scan.problem };
+}
+
+function checkEntryExtensions(bytes, list) {
+    const seen = new Set();
+    let problem;
+    for (let offset = list.start; offset < list.end;) {
+        const extension = derElement(bytes, offset, list.end);
+        offset = extension.end;
+        const id = derElement(bytes, extension.start, extension.end);
+        let value = derElement(bytes, id.end, extension.end);
+        let critical = false;
+        if (value.tag === 0x01) {
+            critical = (value.end > value.start) && (bytes[value.start] !== 0);
+            value = derElement(bytes, value.end, extension.end);
+        }
+        if ((extension.tag !== 0x30) || (id.tag !== 0x06) || (value.tag !== 0x04) || (value.end !== extension.end)) {
+            throw new Error('Malformed CRL entry extension');
+        }
+        const extnID = oidString(bytes.subarray(id.start, id.end));
+        if (seen.has(extnID)) {
+            problem ??= `Duplicate extension ${extnID}`;
+        }
+        seen.add(extnID);
+        if (extnID === '2.5.29.29') {
+            problem ??= 'Indirect CRL certificateIssuer entries are unsupported';
+        } else if (critical) {
+            problem ??= `Unsupported critical CRL entry extension ${extnID}`;
+        } else if (extnID === '2.5.29.21') {
+            const reason = derElement(bytes, value.start, value.end);
+            let code = 0;
+            for (let i = reason.start; i < reason.end; i++) {
+                code = (code * 256) + bytes[i];
+            }
+            if ((reason.tag !== 0x0a) || (reason.end !== value.end) || (reason.end === reason.start) || (code === 8)) {
+                problem ??= 'Invalid revocation reason in a complete CRL';
+            }
+        }
+    }
+    return problem;
+}
+
+// Walk the revoked entries without decoding them into objects. Structural
+// errors throw; the first unsupported entry is reported for use after the CRL
+// is authenticated. With a serial, also report whether it is listed.
+function scanEntries(bytes, entries, serial) {
+    let count = 0;
+    let problem;
+    let revoked = false;
+    for (let offset = entries?.start; offset < entries?.end;) {
+        const entry = derElement(bytes, offset, entries.end);
+        offset = entry.end;
+        const number = derElement(bytes, entry.start, entry.end);
+        const date = derElement(bytes, number.end, entry.end);
+        let next = date.end;
+        if ((entry.tag !== 0x30) || (number.tag !== 0x02) || ! TIME_TAGS.includes(date.tag)) {
+            throw new Error('Malformed revoked certificate entry');
+        }
+        if (next < entry.end) {
+            const extensions = derElement(bytes, next, entry.end);
+            if (extensions.tag !== 0x30) {
+                throw new Error('Malformed revoked certificate entry');
+            }
+            if (serial === undefined) {
+                problem ??= checkEntryExtensions(bytes, extensions);
+            }
+            next = extensions.end;
+        }
+        if (next !== entry.end) {
+            throw new Error('Malformed revoked certificate entry');
+        }
+        count++;
+        if ((serial !== undefined) && bytes.subarray(number.start, number.end).equals(serial)) {
+            revoked = true;
+        }
+    }
+    return { count, problem, revoked };
 }
 
 function checkValidity(crl, now = Date.now()) {
@@ -91,7 +197,8 @@ function checkScope(crl, certificate, urls) {
     return extensions;
 }
 
-async function validateCrl(crl, certificate, issuer, urls, now = Date.now()) {
+async function validateCrl(parsed, certificate, issuer, urls, now = Date.now()) {
+    const crl = parsed.crl;
     const nextUpdate = checkValidity(crl, now);
     const extensions = checkScope(crl, certificate, urls);
     if (! crl.issuer.isEqual(certificate.issuer) || ! crl.issuer.isEqual(issuer.subject)) {
@@ -134,31 +241,17 @@ async function validateCrl(crl, certificate, issuer, urls, now = Date.now()) {
     if (crlNumber && ! (extensionValue(crlNumber) instanceof asn1.Integer)) {
         throw new Error('Malformed CRL number');
     }
-    if (! await crl.verify({ issuerCertificate: issuer }, cryptoEngine)) {
+    // The signature covers the original TBS bytes, including the entries.
+    if (! await cryptoEngine.verifyWithPublicKey(parsed.tbs, crl.signatureValue, issuer.subjectPublicKeyInfo, crl.signatureAlgorithm)) {
         throw new Error('CRL signature verification failed');
     }
-    let revoked = false;
-    for (const entry of crl.revokedCertificates ?? []) {
-        for (const extension of extensionsById(entry.crlEntryExtensions?.extensions).values()) {
-            if (extension.extnID === '2.5.29.29') {
-                throw new Error('Indirect CRL certificateIssuer entries are unsupported');
-            }
-            if (extension.critical) {
-                throw new Error(`Unsupported critical CRL entry extension ${extension.extnID}`);
-            }
-            if (extension.extnID === '2.5.29.21') {
-                const reason = extensionValue(extension);
-                if (! (reason instanceof asn1.Enumerated) || (reason.valueBlock.valueDec === 8)) {
-                    throw new Error('Invalid revocation reason in a complete CRL');
-                }
-            }
-        }
-        if (entry.userCertificate.isEqual(certificate.serialNumber)) {
-            revoked = true;
-        }
+    if (parsed.entryProblem) {
+        throw new Error(parsed.entryProblem);
     }
+    const serial = Buffer.from(certificate.serialNumber.valueBlock.valueHexView);
+    const { revoked } = scanEntries(parsed.bytes, parsed.entries, serial);
     checkValidity(crl);
     return { revoked, nextUpdate };
 }
 
-module.exports = { MAX_CRL_BYTES, parseCertificate, distributionPoints, parseCrl, validateCrl };
+module.exports = { parseCertificate, distributionPoints, parseCrl, validateCrl };

package/download.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const { assertVerifiedResponse, createAgent } = require('./transport');
-const { MAX_CRL_BYTES } = require('./crl');
 const { debugUrl } = require('./debug');
 
 function networkUrl(value, kind = 'CRL distribution point') {
@@ -21,9 +20,9 @@ function networkUrl(value, kind = 'CRL distribution point') {
     return url;
 }
 
-async function download(value, signal, body, debug) {
+async function download(value, signal, body, debug, maxCrlBytes) {
     const kind = (body === undefined) ? 'CRL' : 'OCSP';
-    const maxBytes = (body === undefined) ? MAX_CRL_BYTES : 1024 * 1024;
+    const maxBytes = (body === undefined) ? maxCrlBytes : 1024 * 1024;
     let url = networkUrl(value, kind);
     const timeout = AbortSignal.timeout(10000);
     const downloadSignal = signal ? AbortSignal.any([ signal, timeout ]) : timeout;
@@ -75,7 +74,8 @@ async function download(value, signal, body, debug) {
                 for await (const chunk of response.body) {
                     length += chunk.byteLength;
                     if (length > maxBytes) {
-                        throw new Error(`${kind} download exceeds the ${maxBytes / (1024 * 1024)} MiB size limit`);
+                        throw new Error((body === undefined) ? `CRL download exceeds maxCrlBytes (${maxBytes} bytes)` :
+                            'OCSP download exceeds the 1 MiB size limit');
                     }
                     chunks.push(chunk);
                 }
@@ -89,8 +89,8 @@ async function download(value, signal, body, debug) {
     }
 }
 
-function downloadCrl(value, signal, debug) {
-    return download(value, signal, undefined, debug);
+function downloadCrl(value, signal, debug, maxBytes) {
+    return download(value, signal, undefined, debug, maxBytes);
 }
 
 function downloadOcsp(value, request, signal, debug) {

package/index.js

@@ -18,6 +18,7 @@ async function trFetch(input, options) {
         url: debugUrl(request.url),
         crl: config.policy.disabled ? 'disabled' : 'enabled',
         crlDepth: (config.checkDepth === Infinity) ? 'full-chain' : config.checkDepth,
+        maxCrlBytes: config.policy.maxCrlBytes,
         ocsp: config.ocspPolicy.disabled ? 'disabled' : 'enabled',
         ocspDepth: (config.ocspCheckDepth === Infinity) ? 'full-chain' : config.ocspCheckDepth
     });

package/options.js

@@ -1,7 +1,10 @@
 'use strict';
 
+const DEFAULT_MAX_CRL_BYTES = 16 * 1024 * 1024;
+
 const DEFAULT_POLICY = {
     disabled: false,
+    maxCrlBytes: DEFAULT_MAX_CRL_BYTES,
     missingCrlDistributionPoint: 'ignore',
     unreachableCrlDistributionPoint: 'reject',
     invalidCrl: 'reject',
@@ -54,6 +57,12 @@ function parsePolicy(value, defaults, name) {
                 throw new TypeError(`${name}.disabled must be a boolean, null or undefined`);
             }
             policy.disabled = setting ?? false;
+        } else if (key === 'maxCrlBytes') {
+            // Deliberately no value for unlimited; any limit must be explicit.
+            if (! Number.isSafeInteger(setting) || (setting <= 0)) {
+                throw new TypeError(`${name}.maxCrlBytes must be a positive safe integer (bytes)`);
+            }
+            policy.maxCrlBytes = setting;
         } else {
             if (! [ 'ignore', 'warn', 'reject' ].includes(setting)) {
                 throw new TypeError(`${name}.${key} must be ignore, warn or reject`);
@@ -143,4 +152,4 @@ function splitOptions(options) {
              ocspPolicy, ocspUri, ocspCheckDepth, debugEnabled };
 }
 
-module.exports = { splitOptions };
+module.exports = { DEFAULT_MAX_CRL_BYTES, splitOptions };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "tr-fetch",
-    "version": "0.9.0",
+    "version": "0.9.2",
     "description": "Node.js fetch with CRL and OCSP certificate revocation checking, plus tr-curl, a curl-like test tool",
     "main": "index.js",
     "exports": {

package/pkiutils.js

@@ -8,12 +8,70 @@ const cryptoEngine = new pki.CryptoEngine({ name: 'tr-fetch', crypto: webcrypto
 
 function parseDer(bytes, Type) {
     const decoded = asn1.fromBER(bytes);
-    if ((decoded.offset !== bytes.length) || decoded.result.error) {
-        throw new Error('Malformed ASN.1 data or trailing bytes');
+    if (decoded.result.error) {
+        throw new Error(`Malformed ASN.1 data: ${decoded.result.error}`);
+    }
+    if (decoded.offset !== bytes.length) {
+        throw new Error('Malformed ASN.1 data: trailing bytes');
     }
     return new Type({ schema: decoded.result });
 }
 
+// A minimal DER element reader for data too large to decode into an object
+// tree: single-byte tags and definite lengths only, always within bounds.
+function derElement(bytes, offset, end = bytes.length) {
+    if ((offset + 2) > end) {
+        throw new Error('Malformed ASN.1 data: truncated element');
+    }
+    const tag = bytes[offset];
+    if ((tag & 0x1f) === 0x1f) {
+        throw new Error('Malformed ASN.1 data: unsupported tag');
+    }
+    let start = offset + 2;
+    let length = bytes[offset + 1];
+    if (length & 0x80) {
+        const count = length & 0x7f;
+        if ((count === 0) || (count > 4) || ((start + count) > end)) {
+            throw new Error('Malformed ASN.1 data: unsupported length encoding');
+        }
+        length = 0;
+        for (let i = 0; i < count; i++) {
+            length = (length * 256) + bytes[start + i];
+        }
+        start += count;
+    }
+    if ((start + length) > end) {
+        throw new Error('Malformed ASN.1 data: element exceeds its container');
+    }
+    return { tag, offset, start, end: start + length };
+}
+
+function derHeader(tag, length) {
+    if (length < 0x80) {
+        return Buffer.from([ tag, length ]);
+    }
+    const bytes = [];
+    for (let value = length; value > 0; value = Math.floor(value / 256)) {
+        bytes.unshift(value % 256);
+    }
+    return Buffer.from([ tag, 0x80 | bytes.length, ...bytes ]);
+}
+
+function oidString(bytes) {
+    const parts = [];
+    let value = 0;
+    for (const byte of bytes) {
+        value = (value * 128) + (byte & 0x7f);
+        if (! (byte & 0x80)) {
+            parts.push(value);
+            value = 0;
+        }
+    }
+    const first = parts.shift() ?? 0;
+    const top = Math.min(2, Math.floor(first / 40));
+    return [ top, first - (top * 40), ...parts ].join('.');
+}
+
 function extensionsById(extensions = []) {
     const result = new Map();
     for (const extension of extensions) {
@@ -41,4 +99,4 @@ function parseCertificate(bytes) {
     return parseDer(bytes, pki.Certificate);
 }
 
-module.exports = { cryptoEngine, parseDer, extensionsById, extensionValue, parseCertificate };
+module.exports = { cryptoEngine, parseDer, derElement, derHeader, oidString, extensionsById, extensionValue, parseCertificate };