tr-fetch 0.9.0

package/tr-curl/options.js

@@ -0,0 +1,193 @@
+'use strict';
+
+const Optist = require('optist');
+
+const TLS_VERSIONS = { '1': 'TLSv1', '1.0': 'TLSv1', '1.1': 'TLSv1.1', '1.2': 'TLSv1.2', '1.3': 'TLSv1.3' };
+const TLS_RANK = [ 'TLSv1', 'TLSv1.1', 'TLSv1.2', 'TLSv1.3' ];
+const SIZE_UNITS = { '': 1, k: 1024, m: 1024 ** 2, g: 1024 ** 3, t: 1024 ** 4, p: 1024 ** 5 };
+
+class UsageError extends Error {}
+
+function sizeCb(value) {
+    const match = value.match(/^(\d+)([kmgtp]?)$/i);
+    const bytes = match ? Number(match[1]) * SIZE_UNITS[match[2].toLowerCase()] : NaN;
+    return Number.isSafeInteger(bytes) ? bytes : undefined;
+}
+
+function secondsCb(value) {
+    return /^(\d+(\.\d*)?|\.\d+)$/.test(value) ? Number(value) : undefined;
+}
+
+function redirectsCb(value) {
+    return /^(-1|0|[1-9]\d{0,8})$/.test(value) ? Number(value) : undefined;
+}
+
+function tlsMaxCb(value) {
+    return (value === 'default') ? value : TLS_VERSIONS[value];
+}
+
+function jsonObjectCb(value) {
+    try {
+        const parsed = JSON.parse(value);
+        return (parsed && (typeof(parsed) === 'object') && ! Array.isArray(parsed)) ? parsed : undefined;
+    } catch (_) {
+        return undefined;
+    }
+}
+
+// Request data options share one ordered list, since curl joins them in the
+// order given regardless of which data option supplied each piece.
+function dataOptions(arg, tag) {
+    return [
+        [ 'd', 'data', 'data', '<data> HTTP POST data; @file reads a file with CR and LF removed' ],
+        [ undefined, 'data-ascii', 'data', '<data> Same as --data' ],
+        [ undefined, 'data-binary', 'binary', '<data> HTTP POST data; @file reads a file as is' ],
+        [ undefined, 'data-raw', 'raw', '<data> HTTP POST data without special meaning for @' ],
+        [ undefined, 'data-urlencode', 'urlencode', '<data> HTTP POST data, URL-encoded' ],
+        [ undefined, 'json', 'json', '<data> HTTP POST JSON; @file reads a file as is' ]
+    ].map(([shortName, longName, kind, description]) => arg(shortName, longName, description, tag(kind), true));
+}
+
+function optionDefinitions() {
+    let seq = 0;
+    const tag = kind => value => ({ seq: ++seq, kind, value });
+    const flag = (shortName, longName, description, multi = false) => ({ shortName, longName, multi, description });
+    const arg = (shortName, longName, description, optArgCb, multi = false) => ({ shortName, longName, hasArg: true, multi, optArgCb,
+                                                                                  argName: description.slice(0, description.indexOf(' ')), description: description.slice(description.indexOf(' ') + 1) });
+    return [
+        arg('A', 'user-agent', '<name> Send User-Agent <name> to server'),
+        arg(undefined, 'cacert', '<file> CA certificates (PEM) to use instead of the default trust store'),
+        arg(undefined, 'ciphers', '<list> TLS 1.2 and older cipher list (OpenSSL format)'),
+        flag(undefined, 'create-dirs', 'Create missing local output directories'),
+        arg(undefined, 'crlfile', '<file> Check the server certificate against this CRL (PEM or DER)'),
+        ...dataOptions(arg, tag),
+        arg('e', 'referer', '<URL> Referrer URL; append ";auto" to update it on redirects'),
+        arg(undefined, 'etag-compare', '<file> Send If-None-Match with the ETag read from <file>'),
+        flag('f', 'fail', 'Fail fast with no output on HTTP errors'),
+        flag(undefined, 'fail-early', 'Stop at the first failed transfer'),
+        flag(undefined, 'fail-with-body', 'Fail on HTTP errors but save the body'),
+        flag('G', 'get', 'Put the post data in the URL and use GET'),
+        arg('H', 'header', '<header/@file> Pass custom header(s) to server', undefined, true),
+        flag('h', 'help', 'Show help and exit'),
+        flag('I', 'head', 'Show document info only'),
+        flag('i', 'include', 'Include response headers in the output'),
+        flag('k', 'insecure', 'Skip TLS verification; also bypasses trFetch revocation checks'),
+        flag('L', 'location', 'Follow redirects'),
+        flag(undefined, 'location-trusted', 'Like --location, and send authentication to other hosts'),
+        arg(undefined, 'max-filesize', '<bytes> Maximum file size to download (suffixes k, M, G, T, P)', sizeCb),
+        arg(undefined, 'max-redirs', '<num> Maximum number of redirects allowed (default 50, -1 unlimited)', redirectsCb),
+        arg('m', 'max-time', '<seconds> Maximum time allowed for each transfer', secondsCb),
+        flag(undefined, 'no-progress-meter', 'Do not show the progress meter'),
+        arg(undefined, 'oauth2-bearer', '<token> OAuth 2 Bearer Token'),
+        arg('o', 'output', '<file> Write to file instead of stdout (one per URL)', undefined, true),
+        arg(undefined, 'output-dir', '<dir> Directory to save files in'),
+        flag('#', 'progress-bar', 'Display transfer progress as a bar'),
+        arg('r', 'range', '<range> Retrieve only the bytes within RANGE'),
+        flag('O', 'remote-name', 'Write output to a file named as the remote file (one per URL)', true),
+        flag(undefined, 'remote-name-all', 'Use the remote file name for all URLs'),
+        flag(undefined, 'remove-on-error', 'Remove output file on errors'),
+        arg('X', 'request', '<method> Specify request method to use'),
+        flag('s', 'silent', 'Silent mode'),
+        flag('S', 'show-error', 'Show error even when -s is used'),
+        flag('1', 'tlsv1', 'Use TLSv1.0 or greater'),
+        flag(undefined, 'tlsv1.0', 'Use TLSv1.0 or greater'),
+        flag(undefined, 'tlsv1.1', 'Use TLSv1.1 or greater'),
+        flag(undefined, 'tlsv1.2', 'Use TLSv1.2 or greater'),
+        flag(undefined, 'tlsv1.3', 'Use TLSv1.3 or greater'),
+        arg(undefined, 'tls-max', '<version> Maximum TLS version: 1.0, 1.1, 1.2, 1.3 or default', tlsMaxCb),
+        arg(undefined, 'tls13-ciphers', '<list> TLS 1.3 cipher suites to use'),
+        arg(undefined, 'trfetch-options', '<json> Extra trFetch options as a JSON object', jsonObjectCb, true),
+        arg('u', 'user', '<user:password> Server user and password (basic authentication)'),
+        arg(undefined, 'url', '<url> URL to work with', undefined, true),
+        flag('v', 'verbose', 'Make the operation more talkative, including trFetch debug output', true),
+        flag('V', 'version', 'Show version number and quit')
+    ];
+}
+
+function helpText() {
+    const lines = optionDefinitions().map(function(o) {
+        const names = [ o.shortName && `-${o.shortName}`, `--${o.longName}` ].filter(Boolean).join(', ');
+        return [ (o.shortName ? ' ' : '     ') + names + (o.hasArg ? ` ${o.argName}` : ''), o.description ];
+    });
+    const width = Math.max(...lines.map(x => x[0].length)) + 2;
+    return 'Usage: tr-curl [options...] <url>...\n' + lines.map(([left, text]) => left.padEnd(width) + text + '\n').join('');
+}
+
+function parseArguments(argv) {
+    const opt = (new Optist()).opts(optionDefinitions()).parsePosix(Array.from(argv));
+    const value = name => opt.value(name);
+    if (value('fail') && value('fail-with-body')) {
+        throw new UsageError('--fail and --fail-with-body cannot be used together');
+    }
+    // Optist cannot preserve the relative order of -o and -O.
+    if (value('output').length && value('remote-name')) {
+        throw new UsageError('--output and --remote-name cannot be used together');
+    }
+    const trFetchOptions = Object.assign({}, ...value('trfetch-options'));
+    for (const key of Object.keys(trFetchOptions)) {
+        if (! /^trFetch/.test(key)) {
+            throw new UsageError(`--trfetch-options accepts only trFetch options, not ${JSON.stringify(key)}`);
+        }
+    }
+    const data = [ 'data', 'data-ascii', 'data-binary', 'data-raw', 'data-urlencode', 'json' ]
+          .flatMap(value).sort((a, b) => a.seq - b.seq);
+    if (value('head') && data.length && ! value('get')) {
+        throw new UsageError('You can only select one HTTP request method! You asked for both POST ' +
+                             '(using --data or --json) and HEAD (using -I/--head).');
+    }
+    const tlsMin = [ 'tlsv1.3', 'tlsv1.2', 'tlsv1.1', 'tlsv1.0', 'tlsv1' ].find(value);
+    let tlsMax = value('tls-max');
+    if (tlsMax === 'default') {
+        tlsMax = undefined;
+    }
+    if (tlsMin && tlsMax && (TLS_RANK.indexOf(TLS_VERSIONS[tlsMin.slice(4)]) > TLS_RANK.indexOf(tlsMax))) {
+        throw new UsageError(`--${tlsMin} conflicts with --tls-max ${tlsMax.slice(4)}`);
+    }
+    return {
+        help: value('help'),
+        version: value('version'),
+        urls: [ ...value('url'), ...opt.rest() ],
+        headers: value('header'),
+        data,
+        json: data.some(x => x.kind === 'json'),
+        get: value('get'),
+        head: value('head'),
+        include: value('include'),
+        request: value('request'),
+        userAgent: value('user-agent'),
+        user: value('user'),
+        bearer: value('oauth2-bearer'),
+        referer: value('referer'),
+        range: value('range'),
+        etagCompare: value('etag-compare'),
+        location: value('location') || value('location-trusted'),
+        locationTrusted: value('location-trusted'),
+        maxRedirs: value('max-redirs') ?? 50,
+        maxTime: value('max-time'),
+        maxFilesize: value('max-filesize') || undefined,
+        fail: value('fail'),
+        failWithBody: value('fail-with-body'),
+        failEarly: value('fail-early'),
+        outputs: value('output'),
+        outputDir: value('output-dir'),
+        createDirs: value('create-dirs'),
+        remoteName: value('remote-name'),
+        remoteNameAll: value('remote-name-all'),
+        removeOnError: value('remove-on-error'),
+        silent: value('silent'),
+        showError: value('show-error'),
+        progressMeter: ! value('no-progress-meter'),
+        progressBar: value('progress-bar'),
+        verbose: value('verbose') > 0,
+        insecure: value('insecure'),
+        cacert: value('cacert'),
+        crlFile: value('crlfile'),
+        ciphers: value('ciphers'),
+        tls13Ciphers: value('tls13-ciphers'),
+        tlsMin: tlsMin && TLS_VERSIONS[tlsMin.slice(4)],
+        tlsMax,
+        trFetchOptions
+    };
+}
+
+module.exports = { parseArguments, helpText, UsageError };

package/tr-curl/progress.js

@@ -0,0 +1,136 @@
+'use strict';
+
+const HEADER = '  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n' +
+      '                                 Dload  Upload   Total   Spent    Left  Speed\n';
+
+const UNITS = [ [ 'k', 1024 ], [ 'M', 1024 ** 2 ], [ 'G', 1024 ** 3 ], [ 'T', 1024 ** 4 ], [ 'P', 1024 ** 5 ] ];
+
+// A byte count in five columns, like curl's meter.
+function size5(value) {
+    value = Math.max(0, Math.floor(value));
+    if (value < 100000) {
+        return String(value).padStart(5);
+    }
+    for (const [unit, scale] of UNITS) {
+        if (value < 100 * scale) {
+            const tenths = Math.floor(value / (scale / 10));
+            return `${Math.floor(tenths / 10)}.${tenths % 10}${unit}`.padStart(5);
+        }
+        if (value < 10000 * scale) {
+            return `${Math.floor(value / scale)}${unit}`.padStart(5);
+        }
+    }
+    return `${Math.floor(value / (1024 ** 5))}P`.padStart(5);
+}
+
+function time8(seconds) {
+    if (! Number.isFinite(seconds)) {
+        return '--:--:--';
+    }
+    seconds = Math.floor(seconds);
+    const hours = Math.floor(seconds / 3600);
+    if (hours >= 100) {
+        return `${Math.floor(hours / 24)}d ${String(hours % 24).padStart(2, '0')}h`.padStart(8);
+    }
+    return `${String(hours).padStart(2)}:${String(Math.floor(seconds / 60) % 60).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`;
+}
+
+function percent(part, total) {
+    return String(total ? Math.floor((part * 100) / total) : 0).padStart(3);
+}
+
+class Progress {
+    #mode;
+    #stream;
+    #started = Date.now();
+    #timer;
+    #total;
+    #received = 0;
+    #uploaded = 0;
+    #samples = [];
+    #bounce = 0;
+    #finished = false;
+
+    constructor(mode, stream = process.stderr) {
+        this.#mode = mode;
+        this.#stream = stream;
+        if (mode === 'meter') {
+            this.#write(HEADER);
+        }
+        this.#timer = setInterval(() => this.#render(), (mode === 'meter') ? 1000 : 200);
+        this.#timer.unref();
+    }
+
+    expect(total) {
+        this.#total = total;
+    }
+
+    upload(bytes) {
+        this.#uploaded = bytes;
+    }
+
+    update(received) {
+        this.#received = received;
+    }
+
+    finish() {
+        if (this.#finished) {
+            return;
+        }
+        this.#finished = true;
+        clearInterval(this.#timer);
+        this.#render(true);
+        this.#write('\n');
+    }
+
+    #write(text) {
+        try {
+            this.#stream.write(text);
+        } catch (_) {
+            // The meter must not change the transfer outcome.
+        }
+    }
+
+    #render(final = false) {
+        const now = Date.now();
+        if (this.#mode === 'bar') {
+            this.#renderBar(final);
+            return;
+        }
+        const spent = (now - this.#started) / 1000;
+        this.#samples.push([ now, this.#received ]);
+        while ((this.#samples.length > 1) && ((now - this.#samples[0][0]) > 5000)) {
+            this.#samples.shift();
+        }
+        const [sampleTime, sampleBytes] = this.#samples[0];
+        const current = (now > sampleTime) ? ((this.#received - sampleBytes) * 1000) / (now - sampleTime) :
+              (spent ? this.#received / spent : 0);
+        const dlSpeed = spent ? this.#received / spent : 0;
+        const ulSpeed = spent ? this.#uploaded / spent : 0;
+        const total = (this.#total ?? this.#received) + this.#uploaded;
+        const done = this.#received + this.#uploaded;
+        const expected = (this.#total !== undefined) && (dlSpeed > 0) ? this.#total / dlSpeed : undefined;
+        const left = (expected === undefined) ? undefined : Math.max(0, expected - spent);
+        this.#write(`\r${percent(done, total)} ${size5(total)}  ${percent(this.#received, this.#total)} ${size5(this.#received)}  ` +
+                    `${percent(this.#uploaded, this.#uploaded)} ${size5(this.#uploaded)}  ${size5(dlSpeed)}  ${size5(ulSpeed)} ` +
+                    `${time8(expected)} ${time8(spent)} ${time8(left)} ${size5(final ? dlSpeed : current)}`);
+    }
+
+    #renderBar(final) {
+        const width = Math.max(20, (this.#stream.columns || Number(process.env.COLUMNS) || 80) - 1);
+        const length = width - 7;
+        if (this.#total) {
+            const fraction = Math.min(1, this.#received / this.#total);
+            const bar = '#'.repeat(Math.round(fraction * length)).padEnd(length);
+            this.#write(`\r${bar} ${(fraction * 100).toFixed(1).padStart(5)}%`);
+        } else if (final) {
+            this.#write(`\r${'#'.repeat(length).padEnd(width)}`);
+        } else {
+            const position = this.#bounce++ % ((length - 5) * 2);
+            const offset = (position < (length - 5)) ? position : ((length - 5) * 2) - position;
+            this.#write(`\r${(' '.repeat(offset) + '-=O=-').padEnd(width)}`);
+        }
+    }
+}
+
+module.exports = Progress;

package/transport.js

@@ -0,0 +1,150 @@
+'use strict';
+
+const { isIP } = require('node:net');
+const { Agent, buildConnector, getGlobalDispatcher } = require('undici');
+const { version: undiciVersion } = require('undici/package.json');
+const { debugUrl } = require('./debug');
+
+// Origins each agent actually connected to, after TLS and revocation checks.
+const verifiedOrigins = new WeakMap();
+
+// Do not silently discard a global dispatcher's pinning, proxy, CA or other
+// security settings. Node's bundled Agent uses different Symbol identities;
+// recognize only its known default configuration and default factory as well.
+// Unknown configurations fail closed when Undici's internals change.
+const referenceAgent = new Agent();
+const factorySymbol = Object.getOwnPropertySymbols(referenceAgent).find(x => x.description === 'factory');
+const defaultFactory = Function.prototype.toString.call(referenceAgent[factorySymbol]).replace(/[\s;]/g, '');
+void referenceAgent.close();
+
+function isDefaultAgent(agent) {
+    if (! agent || (agent.constructor.name !== 'Agent') || Object.hasOwn(agent, 'dispatch')) {
+        return false;
+    }
+    const symbols = Object.getOwnPropertySymbols(agent);
+    const options = agent[symbols.find(x => x.description === 'options')];
+    const factory = agent[symbols.find(x => x.description === 'factory')];
+    return options && (typeof(factory) === 'function') &&
+        (Function.prototype.toString.call(factory).replace(/[\s;]/g, '') === defaultFactory) &&
+        Object.entries(options).every(([key, value]) =>
+            ((key === 'connect') && (value === undefined)) || ((key === 'maxOrigins') && (value === Infinity)));
+}
+
+// System fetch drives this package's Agent through the dispatcher handler
+// API of its own bundled Undici, which changes between major versions.
+function assertCompatibleUndici(bundled = process.versions.undici) {
+    const major = version => String(version).split('.')[0];
+    if (major(bundled) !== major(undiciVersion)) {
+        throw new TypeError(`trFetch uses Undici ${undiciVersion}, which is incompatible with ` +
+            `Undici ${bundled} in the fetch of Node.js ${process.version}`);
+    }
+}
+
+function assertDefaultDispatcher() {
+    assertCompatibleUndici();
+    if (! isDefaultAgent(getGlobalDispatcher())) {
+        throw new TypeError('trFetch requires the default Undici dispatcher; custom or unrecognized global dispatchers cannot be safely replaced');
+    }
+}
+
+function originOf(options) {
+    // Undici passes IPv6 addresses without URL brackets.
+    const host = (isIP(options.hostname) === 6) ? `[${options.hostname}]` : options.hostname;
+    try {
+        return new URL(`${options.protocol}//${host}${options.port ? `:${options.port}` : ''}`).origin;
+    } catch (_) {
+        return undefined;
+    }
+}
+
+function createAgent(checkCertificate, signal) {
+    const connect = buildConnector({ rejectUnauthorized: true, maxCachedSessions: 0, allowH2: false });
+    const origins = new Set();
+    const agent = new Agent({
+        allowH2: false,
+        pipelining: 0,
+        connect(options, callback) {
+            let socket;
+            let finished = false;
+            function finish(error) {
+                if (finished) {
+                    return;
+                }
+                finished = true;
+                signal?.removeEventListener('abort', abort);
+                socket?.removeListener('error', finish);
+                if (error) {
+                    socket?.destroy();
+                    callback(error, null);
+                } else {
+                    origins.add(originOf(options));
+                    callback(null, socket);
+                }
+            }
+            function abort() {
+                finish(signal.reason);
+            }
+            if (signal?.aborted) {
+                abort();
+                return;
+            }
+            signal?.addEventListener('abort', abort, { once: true });
+            // Bind SNI and identity verification to the URL, not a Host header.
+            const hostname = options.hostname;
+            socket = connect({ ...options, host: hostname, servername: isIP(hostname) ? null : hostname }, function(error, connected) {
+                if (finished) {
+                    connected?.destroy();
+                    return;
+                }
+                if (error) {
+                    finish(error);
+                    return;
+                }
+                if (options.protocol !== 'https:') {
+                    finish();
+                    return;
+                }
+                if (! connected.authorized) {
+                    finish(new Error(`TLS certificate verification failed: ${connected.authorizationError}`));
+                    return;
+                }
+                connected.disableRenegotiation();
+                if (! checkCertificate) {
+                    finish();
+                    return;
+                }
+                Promise.resolve().then(function() {
+                    return checkCertificate(connected.getPeerCertificate(true), hostname, signal);
+                }).then(() => finish(), finish);
+            });
+            socket.on('error', finish);
+            return socket;
+        }
+    });
+    verifiedOrigins.set(agent, origins);
+    return agent;
+}
+
+// Fail closed if system fetch was replaced or wrapped by code that dropped
+// the dispatcher: an HTTP(S) response must come from a connection that this
+// agent opened and verified, both for the request and the final URL.
+function assertVerifiedResponse(agent, requestUrl, response) {
+    if (! [ 'http:', 'https:' ].includes(new URL(requestUrl).protocol)) {
+        return;
+    }
+    const origins = verifiedOrigins.get(agent);
+    for (const value of [ requestUrl, response.url || requestUrl ]) {
+        let origin;
+        try {
+            origin = new URL(value).origin;
+        } catch (_) {
+            origin = undefined;
+        }
+        if (! origins?.has(origin)) {
+            throw new TypeError(`trFetch: the response for ${debugUrl(value)} did not come through the verifying ` +
+                'dispatcher; globalThis.fetch may be replaced or wrapped');
+        }
+    }
+}
+
+module.exports = { assertCompatibleUndici, assertDefaultDispatcher, assertVerifiedResponse, createAgent };