tr-fetch 0.9.3 → 0.9.4

package/README.md

@@ -80,8 +80,10 @@ await trFetch('https://example.com/', { trFetchCrlPolicy: { maxCrlBytes: 64 * 10
 A larger download is an unreachable distribution point; larger
 `trFetchCrlOverride` data is an invalid CRL. A cached CRL above the caller's
 limit is not used, even if an earlier caller with a higher limit cached it.
-Revoked entries are scanned in place, so memory use stays close to the CRL's
-size.
+The CRL is downloaded into a single buffer, verified and scanned in place,
+and only its serial numbers are kept. Peak memory while fetching a CRL is
+roughly three to four times its size, most of it in Node's HTTP stream
+buffering, and is released afterwards.
 
 ### Disabling a check
 
@@ -210,8 +212,21 @@ and a matching certificate identifier. Intermediates use their own OCSP URIs.
 
 Fetched and validated CRLs share an in-memory LRU cache within the loaded
 module. Cache keys include the issuer certificate and distribution URL.
-Certificate decisions and failed lookups are never cached. A cached CRL is
-validated against the current certificate and current time on every use.
+Certificate decisions, failed lookups and `trFetchCrlOverride` data are never
+cached.
+
+The cache does not keep the CRL itself. After a CRL is authenticated, its
+revoked serial numbers are stored as sorted fixed-width records, one array per
+serial length, together with the CRL's validity period and scope. Memory is
+about the size of the serial numbers (14 MB for a 43 MB CRL of 875,000
+entries), outside the JavaScript heap, and a lookup is a binary search.
+Serials match only by identical DER bytes.
+
+Checks that depend only on the CRL and its issuer, including the signature,
+run once per download. Checks that depend on the certificate or the time run
+on every use: validity dates, scope and distribution point, the certificate's
+issuer name and signature, and the serial lookup. An entry serves only the
+issuer certificate that authenticated it.
 
 - `trFetchCrlCacheSize` defaults to `32` entries. Any integer **0 or less**
   disables caching for that call.
@@ -338,7 +353,8 @@ A successfully authenticated CRL listing the certificate immediately invokes
 the revocation policy; it does not trigger a search for a different answer.
 
 Supported CRLs are direct, complete CRLs with SHA-256, SHA-384 or SHA-512
-signatures supported by PKIjs/Web Crypto, including RSA and ECDSA. Validation
+RSA (PKCS #1 v1.5 or PSS) or ECDSA signatures, verified with Node's crypto
+module. RSA-PSS must use MGF1 with the signature hash. Validation
 checks issuer binding, signature algorithm consistency, `cRLSign` key usage
 when present, authority key identifiers when available, dates and scope.
 `nextUpdate` is required. Matching named issuing distribution points and

package/cache.js

@@ -7,8 +7,9 @@ function cacheEntry(key) {
     return { issuer: key.slice(0, separator), source: debugUrl(key.slice(separator + 1)) };
 }
 
-// Cache verified CRLs, never a policy decision or a failed lookup. A caller's
-// shorter TTL also applies to entries populated by an earlier caller.
+// Cache authenticated revocation lists (see RevocationList in crl.js), never
+// a policy decision, a failed lookup or the CRL bytes. A caller's shorter TTL
+// also applies to entries populated by an earlier caller.
 class CrlCache {
     #entries = new Map();
 

package/check.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { createHash } = require('node:crypto');
-const { parseCertificate, distributionPoints, parseCrl, validateCrl } = require('./crl');
+const { parseCertificate, distributionPoints, parseCrl, authenticateCrl, checkRevocationList } = require('./crl');
 const { networkUrl, downloadCrl } = require('./download');
 const { TrFetchCrlError, TrFetchOcspError, applyPolicy } = require('./errors');
 const { checkOcspCertificate } = require('./ocsp');
@@ -83,7 +83,7 @@ async function checkCertificate(peer, hostname, options, signal, override, debug
                 failures.push(issue('unreachableCrlDistributionPoint', 'CRL distribution point lookup limit (32) exceeded'));
                 break;
             }
-            let crl, key, bytes;
+            let list, key, bytes;
             const fetchedAt = Date.now();
             if (forcedCrl !== undefined) {
                 bytes = forcedCrl;
@@ -91,14 +91,14 @@ async function checkCertificate(peer, hostname, options, signal, override, debug
                 try {
                     const url = networkUrl(source);
                     key = issuerId + ':' + url.href;
-                    crl = cache.get(key, options.cacheSize, options.cacheTTL, Date.now(), debug);
+                    list = cache.get(key, options.cacheSize, options.cacheTTL, Date.now(), debug);
                     // A CRL cached under a higher limit must not bypass this one.
-                    if (crl && (crl.size > options.policy.maxCrlBytes)) {
+                    if (list && (list.size > options.policy.maxCrlBytes)) {
                         debug?.('CRL cache entry not used', { source: debugUrl(url), reason: 'exceeds maxCrlBytes',
-                                                              bytes: crl.size, maxCrlBytes: options.policy.maxCrlBytes });
-                        crl = undefined;
+                                                              bytes: list.size, maxCrlBytes: options.policy.maxCrlBytes });
+                        list = undefined;
                     }
-                    if (! crl) {
+                    if (! list) {
                         bytes = await downloadCrl(url, signal, debug, options.policy.maxCrlBytes);
                     }
                 } catch (cause) {
@@ -110,19 +110,22 @@ async function checkCertificate(peer, hostname, options, signal, override, debug
             }
             let result;
             try {
-                if (! crl) {
-                    crl = parseCrl(bytes, options.policy.maxCrlBytes);
-                    debug?.('CRL parsed', { source: (source === undefined) ? 'trFetchCrlOverride' : debugUrl(source),
-                                            revokedEntries: crl.revokedCount });
+                if (! list) {
+                    const label = (source === undefined) ? 'trFetchCrlOverride' : debugUrl(source);
+                    const parsed = parseCrl(bytes, options.policy.maxCrlBytes);
+                    debug?.('CRL parsed', { source: label, revokedEntries: parsed.revokedCount });
+                    list = await authenticateCrl(parsed, issuer);
+                    debug?.('CRL authenticated and indexed', { source: label, serials: list.serials.count,
+                                                               indexBytes: list.serials.bytes });
                 }
-                result = await validateCrl(crl, certificate, issuer, point.urls);
+                result = await checkRevocationList(list, certificate, issuer, point.urls);
             } catch (cause) {
                 failures.push(issue('invalidCrl',
                                     `Invalid CRL from ${(source === undefined) ? 'trFetchCrlOverride' : sourceLabel(source)}: ${cause.message}`, cause, source));
                 continue;
             }
             if ((key !== undefined) && (bytes !== undefined)) {
-                cache.set(key, crl, result.nextUpdate, fetchedAt, options.cacheSize, options.cacheTTL, Date.now(), debug);
+                cache.set(key, list, result.nextUpdate, fetchedAt, options.cacheSize, options.cacheTTL, Date.now(), debug);
             }
             debug?.('CRL serial lookup completed', { source: (source === undefined) ? 'trFetchCrlOverride' : debugUrl(source),
                                                      result: result.revoked ? 'revoked' : 'not listed', authenticated: true,

package/crl.js

@@ -1,10 +1,12 @@
 'use strict';
 
+const { X509Certificate, constants, verify } = require('node:crypto');
 const asn1 = require('asn1js');
 const pki = require('pkijs');
-const { cryptoEngine, parseDer, derElement, derHeader, oidString, extensionsById, extensionValue,
+const { cryptoEngine, parseDer, derElement, derRead, derHeader, oidString, extensionsById, extensionValue,
     parseCertificate } = require('./pkiutils');
 const { DEFAULT_MAX_CRL_BYTES } = require('./options');
+const { buildSerialIndex } = require('./serials');
 
 const TIME_TAGS = [ 0x17, 0x18 ];
 
@@ -72,33 +74,69 @@ function parseCrl(bytes, maxBytes = DEFAULT_MAX_CRL_BYTES) {
         revokedCount: scan.count, entryProblem: scan.problem };
 }
 
-function checkEntryExtensions(bytes, list) {
-    const seen = new Set();
+const OID_CERTIFICATE_ISSUER = Buffer.from([ 0x55, 0x1d, 0x1d ]);
+const OID_REASON_CODE = Buffer.from([ 0x55, 0x1d, 0x15 ]);
+
+function sameBytes(bytes, start, end, expected) {
+    if ((end - start) !== expected.length) {
+        return false;
+    }
+    for (let i = 0; i < expected.length; i++) {
+        if (bytes[start + i] !== expected[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+function sameRange(bytes, a, b, c, d) {
+    if ((b - a) !== (d - c)) {
+        return false;
+    }
+    for (let i = 0; i < (b - a); i++) {
+        if (bytes[a + i] !== bytes[c + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+// Reusable elements for walking entries: hundreds of thousands of entries
+// must not create an object each. Walks are synchronous, so one set per walk.
+function scratch() {
+    return { entry: {}, number: {}, date: {}, extensions: {}, extension: {}, id: {}, value: {}, reason: {}, ids: [] };
+}
+
+function checkEntryExtensions(bytes, list, s) {
     let problem;
+    s.ids.length = 0;
     for (let offset = list.start; offset < list.end;) {
-        const extension = derElement(bytes, offset, list.end);
+        const extension = derRead(bytes, offset, list.end, s.extension);
         offset = extension.end;
-        const id = derElement(bytes, extension.start, extension.end);
-        let value = derElement(bytes, id.end, extension.end);
+        const id = derRead(bytes, extension.start, extension.end, s.id);
+        let value = derRead(bytes, id.end, extension.end, s.value);
         let critical = false;
         if (value.tag === 0x01) {
             critical = (value.end > value.start) && (bytes[value.start] !== 0);
-            value = derElement(bytes, value.end, extension.end);
+            value = derRead(bytes, value.end, extension.end, s.value);
         }
         if ((extension.tag !== 0x30) || (id.tag !== 0x06) || (value.tag !== 0x04) || (value.end !== extension.end)) {
             throw new Error('Malformed CRL entry extension');
         }
-        const extnID = oidString(bytes.subarray(id.start, id.end));
-        if (seen.has(extnID)) {
-            problem ??= `Duplicate extension ${extnID}`;
+        // Object identifiers compare as bytes; strings only for messages.
+        const extnID = () => oidString(bytes.subarray(id.start, id.end));
+        for (let i = 0; i < s.ids.length; i += 2) {
+            if (sameRange(bytes, s.ids[i], s.ids[i + 1], id.start, id.end)) {
+                problem ??= `Duplicate extension ${extnID()}`;
+            }
         }
-        seen.add(extnID);
-        if (extnID === '2.5.29.29') {
+        s.ids.push(id.start, id.end);
+        if (sameBytes(bytes, id.start, id.end, OID_CERTIFICATE_ISSUER)) {
             problem ??= 'Indirect CRL certificateIssuer entries are unsupported';
         } else if (critical) {
-            problem ??= `Unsupported critical CRL entry extension ${extnID}`;
-        } else if (extnID === '2.5.29.21') {
-            const reason = derElement(bytes, value.start, value.end);
+            problem ??= `Unsupported critical CRL entry extension ${extnID()}`;
+        } else if (sameBytes(bytes, id.start, id.end, OID_REASON_CODE)) {
+            const reason = derRead(bytes, value.start, value.end, s.reason);
             let code = 0;
             for (let i = reason.start; i < reason.end; i++) {
                 code = (code * 256) + bytes[i];
@@ -113,27 +151,28 @@ function checkEntryExtensions(bytes, list) {
 
 // Walk the revoked entries without decoding them into objects. Structural
 // errors throw; the first unsupported entry is reported for use after the CRL
-// is authenticated. With a serial, also report whether it is listed.
-function scanEntries(bytes, entries, serial) {
+// is authenticated. With visit, call visit(start, end) for each serial in
+// bytes instead of checking entry extensions again.
+function scanEntries(bytes, entries, visit) {
+    const s = scratch();
     let count = 0;
     let problem;
-    let revoked = false;
     for (let offset = entries?.start; offset < entries?.end;) {
-        const entry = derElement(bytes, offset, entries.end);
+        const entry = derRead(bytes, offset, entries.end, s.entry);
         offset = entry.end;
-        const number = derElement(bytes, entry.start, entry.end);
-        const date = derElement(bytes, number.end, entry.end);
+        const number = derRead(bytes, entry.start, entry.end, s.number);
+        const date = derRead(bytes, number.end, entry.end, s.date);
         let next = date.end;
-        if ((entry.tag !== 0x30) || (number.tag !== 0x02) || ! TIME_TAGS.includes(date.tag)) {
+        if ((entry.tag !== 0x30) || (number.tag !== 0x02) || ((date.tag !== 0x17) && (date.tag !== 0x18))) {
             throw new Error('Malformed revoked certificate entry');
         }
         if (next < entry.end) {
-            const extensions = derElement(bytes, next, entry.end);
+            const extensions = derRead(bytes, next, entry.end, s.extensions);
             if (extensions.tag !== 0x30) {
                 throw new Error('Malformed revoked certificate entry');
             }
-            if (serial === undefined) {
-                problem ??= checkEntryExtensions(bytes, extensions);
+            if (visit === undefined) {
+                problem ??= checkEntryExtensions(bytes, extensions, s);
             }
             next = extensions.end;
         }
@@ -141,29 +180,103 @@ function scanEntries(bytes, entries, serial) {
             throw new Error('Malformed revoked certificate entry');
         }
         count++;
-        if ((serial !== undefined) && bytes.subarray(number.start, number.end).equals(serial)) {
-            revoked = true;
-        }
+        visit?.(number.start, number.end);
     }
-    return { count, problem, revoked };
+    return { count, problem };
 }
 
-function checkValidity(crl, now = Date.now()) {
-    const start = crl.thisUpdate.value.getTime();
-    const end = crl.nextUpdate?.value.getTime();
-    if (! Number.isFinite(start) || ! Number.isFinite(end) || (end <= start)) {
-        throw new Error('CRL must have a valid thisUpdate and a later nextUpdate');
+const RSA_PSS = '1.2.840.113549.1.1.10';
+const MGF1 = '1.2.840.113549.1.1.8';
+// Signature algorithms and the issuer key types they need. The hash is
+// checked separately and must be SHA-256, SHA-384 or SHA-512.
+const SIGNATURE_KEY_TYPES = {
+    '1.2.840.113549.1.1.11': [ 'rsa' ],
+    '1.2.840.113549.1.1.12': [ 'rsa' ],
+    '1.2.840.113549.1.1.13': [ 'rsa' ],
+    [RSA_PSS]: [ 'rsa', 'rsa-pss' ],
+    '1.2.840.10045.4.3.2': [ 'ec' ],
+    '1.2.840.10045.4.3.3': [ 'ec' ],
+    '1.2.840.10045.4.3.4': [ 'ec' ]
+};
+
+// Node's synchronous verify hashes the TBS bytes where they are. Web Crypto
+// and asynchronous verification copy them first, which for a CRL of tens of
+// megabytes doubles its memory. X.509 ECDSA signatures are already DER, the
+// form Node expects.
+function crlSignatureValid(tbs, crl, issuerDer, hash) {
+    const algorithm = crl.signatureAlgorithm.algorithmId;
+    const key = new X509Certificate(issuerDer).publicKey;
+    if (! SIGNATURE_KEY_TYPES[algorithm]?.includes(key.asymmetricKeyType)) {
+        return false;
+    }
+    const signature = crl.signatureValue.valueBlock;
+    if (signature.unusedBits) {
+        return false;
+    }
+    let options = key;
+    if (algorithm === RSA_PSS) {
+        const params = new pki.RSASSAPSSParams({ schema: crl.signatureAlgorithm.algorithmParams });
+        const mgfHash = (params.maskGenAlgorithm.algorithmId === MGF1) ?
+            new pki.AlgorithmIdentifier({ schema: params.maskGenAlgorithm.algorithmParams }).algorithmId : undefined;
+        // Node applies MGF1 with the signature hash; nothing else is accepted.
+        if ((mgfHash !== params.hashAlgorithm.algorithmId) || (params.trailerField !== 1)) {
+            throw new Error('Unsupported RSA-PSS parameters in CRL signature');
+        }
+        options = { key, padding: constants.RSA_PKCS1_PSS_PADDING, saltLength: params.saltLength };
+    }
+    try {
+        return verify(hash.replace('-', '').toLowerCase(), tbs, options, signature.valueHexView);
+    } catch (_) {
+        return false;
     }
-    if (start > now) {
+}
+
+function checkDates(thisUpdate, nextUpdate, now) {
+    if (thisUpdate > now) {
         throw new Error('CRL is not yet valid (thisUpdate is in the future)');
     }
-    if (end <= now) {
+    if (nextUpdate <= now) {
         throw new Error('CRL has expired (nextUpdate has passed)');
     }
-    return end;
 }
 
-function checkScope(crl, certificate, urls) {
+function certificateDer(certificate) {
+    return Buffer.from(certificate.toSchema().toBER());
+}
+
+// An authenticated CRL reduced to what checking a certificate needs: the
+// revoked serials, the validity period and the scope. It does not retain the
+// CRL itself, and serves only certificates of the issuer that signed it.
+class RevocationList {
+    #issuerCertificate;
+
+    constructor(fields) {
+        this.#issuerCertificate = fields.issuerCertificate;
+        this.issuer = fields.issuer;
+        this.thisUpdate = fields.thisUpdate;
+        this.nextUpdate = fields.nextUpdate;
+        this.scope = fields.scope;
+        this.serials = fields.serials;
+        this.revokedCount = fields.revokedCount;
+        this.size = fields.size;
+        Object.freeze(this);
+    }
+
+    signedBy(issuer) {
+        return certificateDer(issuer).equals(this.#issuerCertificate);
+    }
+}
+
+// Everything that depends only on the CRL and its issuer: structure, scope
+// support, issuer binding, signing permission, algorithms and the signature.
+// Runs once per downloaded CRL; the result can be cached for the issuer.
+async function authenticateCrl(parsed, issuer) {
+    const crl = parsed.crl;
+    const thisUpdate = crl.thisUpdate.value.getTime();
+    const nextUpdate = crl.nextUpdate?.value.getTime();
+    if (! Number.isFinite(thisUpdate) || ! Number.isFinite(nextUpdate) || (nextUpdate <= thisUpdate)) {
+        throw new Error('CRL must have a valid thisUpdate and a later nextUpdate');
+    }
     if (! [ 0, 1 ].includes(crl.version)) {
         throw new Error('Unsupported CRL version');
     }
@@ -176,37 +289,25 @@ function checkScope(crl, certificate, urls) {
             throw new Error(`Unsupported critical CRL extension ${extension.extnID}`);
         }
     }
+    const scope = { onlyUserCertificates: false, onlyCaCertificates: false, distributionPoints: undefined };
     const idpExtension = extensions.get('2.5.29.28');
     if (idpExtension) {
         const idp = extensionValue(idpExtension, pki.IssuingDistributionPoint);
         if (idp.indirectCRL || (idp.onlySomeReasons !== undefined) || idp.onlyContainsAttributeCerts) {
             throw new Error('Indirect, reason-limited and attribute-certificate CRLs are unsupported');
         }
-        const basic = extensionsById(certificate.extensions).get('2.5.29.19');
-        const isCA = basic ? extensionValue(basic, pki.BasicConstraints).cA : false;
-        if ((idp.onlyContainsUserCerts && isCA) || (idp.onlyContainsCACerts && ! isCA)) {
-            throw new Error('CRL scope does not cover this certificate type');
-        }
+        scope.onlyUserCertificates = !! idp.onlyContainsUserCerts;
+        scope.onlyCaCertificates = !! idp.onlyContainsCACerts;
         if (idp.distributionPoint !== undefined) {
-            if (! Array.isArray(idp.distributionPoint) ||
-                ! idp.distributionPoint.some(x => (x.type === 6) && urls.includes(x.value))) {
-                throw new Error('CRL issuing distribution point does not match the effective distribution point');
-            }
+            // Only URI names can match; other name forms never do.
+            scope.distributionPoints = Array.isArray(idp.distributionPoint) ?
+                idp.distributionPoint.filter(x => x.type === 6).map(x => x.value) : [];
         }
     }
-    return extensions;
-}
-
-async function validateCrl(parsed, certificate, issuer, urls, now = Date.now()) {
-    const crl = parsed.crl;
-    const nextUpdate = checkValidity(crl, now);
-    const extensions = checkScope(crl, certificate, urls);
-    if (! crl.issuer.isEqual(certificate.issuer) || ! crl.issuer.isEqual(issuer.subject)) {
+    Object.freeze(scope.distributionPoints);
+    if (! crl.issuer.isEqual(issuer.subject)) {
         throw new Error('CRL issuer does not match the certificate issuer');
     }
-    if (! await certificate.verify(issuer, cryptoEngine)) {
-        throw new Error('CRL signing certificate did not issue the checked certificate');
-    }
     if (! Buffer.from(crl.signature.toSchema().toBER()).equals(Buffer.from(crl.signatureAlgorithm.toSchema().toBER()))) {
         throw new Error('CRL signature algorithm identifiers disagree');
     }
@@ -242,16 +343,49 @@ async function validateCrl(parsed, certificate, issuer, urls, now = Date.now())
         throw new Error('Malformed CRL number');
     }
     // The signature covers the original TBS bytes, including the entries.
-    if (! await cryptoEngine.verifyWithPublicKey(parsed.tbs, crl.signatureValue, issuer.subjectPublicKeyInfo, crl.signatureAlgorithm)) {
+    const issuerDer = certificateDer(issuer);
+    if (! crlSignatureValid(parsed.tbs, crl, issuerDer, hash)) {
         throw new Error('CRL signature verification failed');
     }
     if (parsed.entryProblem) {
         throw new Error(parsed.entryProblem);
     }
-    const serial = Buffer.from(certificate.serialNumber.valueBlock.valueHexView);
-    const { revoked } = scanEntries(parsed.bytes, parsed.entries, serial);
-    checkValidity(crl);
-    return { revoked, nextUpdate };
+    // Only an authenticated CRL is worth indexing.
+    const serials = buildSerialIndex(parsed.bytes, visit => scanEntries(parsed.bytes, parsed.entries, visit));
+    return new RevocationList({ issuerCertificate: issuerDer, issuer: crl.issuer, thisUpdate, nextUpdate,
+                                scope: Object.freeze(scope), serials, revokedCount: parsed.revokedCount, size: parsed.size });
+}
+
+// Everything that depends on the checked certificate or the current time.
+// Runs on every use, including for cached lists.
+async function checkRevocationList(list, certificate, issuer, urls, now = Date.now()) {
+    if (! list.signedBy(issuer)) {
+        throw new Error('CRL was authenticated for a different issuer certificate');
+    }
+    checkDates(list.thisUpdate, list.nextUpdate, now);
+    const basic = extensionsById(certificate.extensions).get('2.5.29.19');
+    const isCA = basic ? extensionValue(basic, pki.BasicConstraints).cA : false;
+    if ((list.scope.onlyUserCertificates && isCA) || (list.scope.onlyCaCertificates && ! isCA)) {
+        throw new Error('CRL scope does not cover this certificate type');
+    }
+    if ((list.scope.distributionPoints !== undefined) && ! list.scope.distributionPoints.some(x => urls.includes(x))) {
+        throw new Error('CRL issuing distribution point does not match the effective distribution point');
+    }
+    if (! list.issuer.isEqual(certificate.issuer)) {
+        throw new Error('CRL issuer does not match the certificate issuer');
+    }
+    if (! await certificate.verify(issuer, cryptoEngine)) {
+        throw new Error('CRL signing certificate did not issue the checked certificate');
+    }
+    const revoked = list.serials.has(Buffer.from(certificate.serialNumber.valueBlock.valueHexView));
+    // Checks above can take time; the list must still be valid now.
+    checkDates(list.thisUpdate, list.nextUpdate, Date.now());
+    return { revoked, nextUpdate: list.nextUpdate };
+}
+
+async function validateCrl(parsed, certificate, issuer, urls, now = Date.now()) {
+    return checkRevocationList(await authenticateCrl(parsed, issuer), certificate, issuer, urls, now);
 }
 
-module.exports = { parseCertificate, distributionPoints, parseCrl, validateCrl };
+module.exports = { RevocationList, parseCertificate, distributionPoints, parseCrl, authenticateCrl, checkRevocationList,
+    validateCrl };

package/download.js

@@ -20,6 +20,35 @@ function networkUrl(value, kind = 'CRL distribution point') {
     return url;
 }
 
+// Read the body into one buffer as it arrives, instead of collecting chunks
+// and concatenating them, which needs twice the memory at the end. An
+// uncompressed body's Content-Length sizes the buffer up front and lets an
+// oversized download fail before it is read; otherwise the buffer grows.
+// Only the bytes received are ever exposed.
+async function readBody(response, maxBytes, tooLarge) {
+    const declared = response.headers.has('content-encoding') ? NaN : Number(response.headers.get('content-length') ?? NaN);
+    const known = Number.isSafeInteger(declared) && (declared >= 0);
+    if (known && (declared > maxBytes)) {
+        await response.body?.cancel();
+        throw new Error(tooLarge);
+    }
+    let buffer = Buffer.allocUnsafeSlow(known ? declared : Math.min(64 * 1024, maxBytes));
+    let length = 0;
+    for await (const chunk of response.body ?? []) {
+        if ((length + chunk.byteLength) > maxBytes) {
+            throw new Error(tooLarge);
+        }
+        if ((length + chunk.byteLength) > buffer.length) {
+            const grown = Buffer.allocUnsafeSlow(Math.min(maxBytes, Math.max(length + chunk.byteLength, buffer.length * 2)));
+            buffer.copy(grown, 0, 0, length);
+            buffer = grown;
+        }
+        buffer.set(chunk, length);
+        length += chunk.byteLength;
+    }
+    return (length === buffer.length) ? buffer : buffer.subarray(0, length);
+}
+
 async function download(value, signal, body, debug, maxCrlBytes) {
     const kind = (body === undefined) ? 'CRL' : 'OCSP';
     const maxBytes = (body === undefined) ? maxCrlBytes : 1024 * 1024;
@@ -68,20 +97,11 @@ async function download(value, signal, body, debug, maxCrlBytes) {
                 await response.body?.cancel();
                 throw new Error(`${kind} download returned HTTP ${response.status}`);
             }
-            const chunks = [];
-            let length = 0;
-            if (response.body) {
-                for await (const chunk of response.body) {
-                    length += chunk.byteLength;
-                    if (length > maxBytes) {
-                        throw new Error((body === undefined) ? `CRL download exceeds maxCrlBytes (${maxBytes} bytes)` :
-                            'OCSP download exceeds the 1 MiB size limit');
-                    }
-                    chunks.push(chunk);
-                }
-            }
-            debug?.((kind === 'CRL') ? 'CRL fetched' : 'OCSP response fetched', { source: debugUrl(url), bytes: length });
-            return Buffer.concat(chunks, length);
+            const tooLarge = (body === undefined) ? `CRL download exceeds maxCrlBytes (${maxBytes} bytes)` :
+                'OCSP download exceeds the 1 MiB size limit';
+            const result = await readBody(response, maxBytes, tooLarge);
+            debug?.((kind === 'CRL') ? 'CRL fetched' : 'OCSP response fetched', { source: debugUrl(url), bytes: result.length });
+            return result;
         }
         throw new Error(`${kind} download exceeded the five-redirect limit`);
     } finally {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "tr-fetch",
-    "version": "0.9.3",
+    "version": "0.9.4",
     "description": "Node.js fetch with CRL and OCSP certificate revocation checking, plus tr-curl, a curl-like test tool",
     "main": "index.js",
     "exports": {
@@ -21,6 +21,7 @@
         "ocsp.js",
         "options.js",
         "pkiutils.js",
+        "serials.js",
         "transport.js",
         "bin/",
         "tr-curl/"

package/pkiutils.js

@@ -19,7 +19,13 @@ function parseDer(bytes, Type) {
 
 // A minimal DER element reader for data too large to decode into an object
 // tree: single-byte tags and definite lengths only, always within bounds.
+// derRead fills and returns a caller-supplied object, so loops over large
+// CRLs create no garbage; derElement returns a new one.
 function derElement(bytes, offset, end = bytes.length) {
+    return derRead(bytes, offset, end, {});
+}
+
+function derRead(bytes, offset, end, element) {
     if ((offset + 2) > end) {
         throw new Error('Malformed ASN.1 data: truncated element');
     }
@@ -43,7 +49,11 @@ function derElement(bytes, offset, end = bytes.length) {
     if ((start + length) > end) {
         throw new Error('Malformed ASN.1 data: element exceeds its container');
     }
-    return { tag, offset, start, end: start + length };
+    element.tag = tag;
+    element.offset = offset;
+    element.start = start;
+    element.end = start + length;
+    return element;
 }
 
 function derHeader(tag, length) {
@@ -99,4 +109,4 @@ function parseCertificate(bytes) {
     return parseDer(bytes, pki.Certificate);
 }
 
-module.exports = { cryptoEngine, parseDer, derElement, derHeader, oidString, extensionsById, extensionValue, parseCertificate };
+module.exports = { cryptoEngine, parseDer, derElement, derRead, derHeader, oidString, extensionsById, extensionValue, parseCertificate };

package/serials.js

@@ -0,0 +1,145 @@
+'use strict';
+
+// Revoked serial numbers as sorted, de-duplicated fixed-width records, one
+// Buffer per serial length. Memory is the serial bytes themselves, outside
+// the JS heap, and a lookup is a binary search among serials of the probe's
+// length. Serials match only when their bytes are identical, as in DER.
+class SerialIndex {
+    #groups;
+    #count;
+    #bytes;
+
+    constructor(groups) {
+        this.#groups = groups;
+        this.#count = 0;
+        this.#bytes = 0;
+        for (const group of groups.values()) {
+            this.#count += group.count;
+            this.#bytes += group.data.length;
+        }
+    }
+
+    // Number of distinct serials.
+    get count() {
+        return this.#count;
+    }
+
+    // Bytes of serial data held.
+    get bytes() {
+        return this.#bytes;
+    }
+
+    has(serial) {
+        const group = this.#groups.get(serial.length);
+        if (! group) {
+            return false;
+        }
+        const { data, width } = group;
+        let low = 0;
+        let high = group.count - 1;
+        while (low <= high) {
+            const middle = (low + high) >>> 1;
+            const base = middle * width;
+            let difference = 0;
+            for (let i = 0; (i < width) && ! difference; i++) {
+                difference = data[base + i] - serial[i];
+            }
+            if (! difference) {
+                return true;
+            }
+            if (difference < 0) {
+                low = middle + 1;
+            } else {
+                high = middle - 1;
+            }
+        }
+        return false;
+    }
+}
+
+function sameRecord(a, x, b, y, width) {
+    for (let i = 0; i < width; i++) {
+        if (a[x + i] !== b[y + i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+// LSD radix sort of serial positions in the source buffer, one stable
+// counting pass per byte position, then a copy of each distinct serial into
+// its own buffer. Sorting the positions avoids an unsorted copy of the
+// serials. The work is proportional to the total serial bytes, so no input
+// can make it degrade. Positions where all serials agree are skipped.
+function sortRecords(bytes, positions, width) {
+    const count = positions.length;
+    let order = positions;
+    let next = new positions.constructor(count);
+    const buckets = new Uint32Array(257);
+    for (let position = width - 1; (position >= 0) && (count > 1); position--) {
+        buckets.fill(0);
+        for (let i = 0; i < count; i++) {
+            buckets[bytes[order[i] + position] + 1]++;
+        }
+        if (buckets.includes(count)) {
+            continue;
+        }
+        for (let i = 1; i < 257; i++) {
+            buckets[i] += buckets[i - 1];
+        }
+        for (let i = 0; i < count; i++) {
+            next[buckets[bytes[order[i] + position]]++] = order[i];
+        }
+        [ order, next ] = [ next, order ];
+    }
+    next = undefined;
+    let unique = 0;
+    for (let i = 0; i < count; i++) {
+        if ((i === 0) || ! sameRecord(bytes, order[i - 1], bytes, order[i], width)) {
+            unique++;
+        }
+    }
+    // Own memory for a long-lived cache entry, not a slice of a shared pool
+    // or of the CRL.
+    const sorted = Buffer.allocUnsafeSlow(width * unique);
+    for (let i = 0, filled = 0; i < count; i++) {
+        if ((i === 0) || ! sameRecord(bytes, order[i - 1], bytes, order[i], width)) {
+            bytes.copy(sorted, filled * width, order[i], order[i] + width);
+            filled++;
+        }
+    }
+    return { width, count: unique, data: sorted };
+}
+
+// Build an index of serials in bytes. forEachSerial(visit) must call
+// visit(start, end) for each serial; it is called twice, once to count the
+// serials of each length and once to record where they are.
+function buildSerialIndex(bytes, forEachSerial) {
+    const counts = new Map();
+    forEachSerial(function(start, end) {
+        counts.set(end - start, (counts.get(end - start) ?? 0) + 1);
+    });
+    const Positions = (bytes.length <= 0xffffffff) ? Uint32Array : Float64Array;
+    const located = new Map();
+    for (const [width, count] of counts) {
+        located.set(width, { positions: new Positions(count), filled: 0 });
+    }
+    forEachSerial(function(start, end) {
+        const group = located.get(end - start);
+        if ((group === undefined) || (group.filled >= group.positions.length)) {
+            throw new Error('Serial enumeration changed between passes');
+        }
+        group.positions[group.filled++] = start;
+    });
+    const groups = new Map();
+    for (const [width, group] of located) {
+        if (group.filled !== group.positions.length) {
+            throw new Error('Serial enumeration changed between passes');
+        }
+        groups.set(width, sortRecords(bytes, group.positions, width));
+        located.set(width, undefined);
+    }
+    return new SerialIndex(groups);
+}
+
+module.exports = { SerialIndex, buildSerialIndex };