sliccy 6.211.0 → 6.212.0

package/dist/node-server/routes/fetch-proxy-raw.js

@@ -0,0 +1,205 @@
+/**
+ * Raw mode of `/api/fetch-proxy` (#3571): the HTTP-client flavor the wasm
+ * realm's proxy forwards curl and git through. The wire contract lives in
+ * `@slicc/shared-ts` `raw-fetch-protocol.ts`; this module is the CLI/cloud
+ * float's side of it. Differences from the default route:
+ *
+ *   - the request head (URL, method, ordered headers) arrives as JSON in
+ *     `X-Slicc-Raw-Request`, so the browser's forbidden-header filter and the
+ *     page's own headers (User-Agent, Sec-Fetch-*) never reach upstream;
+ *   - redirects are manual: the 3xx, `Location` and every `Set-Cookie` go
+ *     back to the caller in an ordered header list;
+ *   - the answer is always `200` + `application/vnd.slicc.raw-fetch`: one
+ *     response-head frame, then the body streamed through (Node `pipe`
+ *     pauses upstream whenever the browser stops reading, though Chrome's
+ *     `fetch` reads ahead of its JS reader, so in practice it rarely does).
+ *
+ * Secret handling is the default route's: masked values in headers, URL
+ * credentials and text bodies are unmasked for allowed domains only (403
+ * otherwise), `x-slicc-hmac-sign` is honored, and response header values and
+ * text bodies are scrubbed, including `Location`.
+ */
+import { decodeRawRequestHead, encodeRawResponseFrame, foldRawRequestHeaders, HMAC_SIGN_HEADER, isDecodedPartialResponse, isTextContentType, RAW_FETCH_BRIDGE_REQUEST_BODY_CAP, RAW_FETCH_CONTENT_TYPE, RAW_FETCH_PROBE_HEADER, RAW_FETCH_PROTOCOL_VERSION, RAW_FETCH_REQUEST_HEADER, rawAcceptEncoding, rawResponseHasBody, rawResponseHeaders, stripRawRequestHeaders, } from '../_shared/index.js';
+import { applyHmacSigning, attachUpstreamAbort, injectRequestSecrets, streamUpstreamBody, unmaskRequestBody, } from './fetch-proxy.js';
+const RAW_REQUEST_HEADER_LOWER = RAW_FETCH_REQUEST_HEADER.toLowerCase();
+const RAW_PROBE_HEADER_LOWER = RAW_FETCH_PROBE_HEADER.toLowerCase();
+class RequestBodyTooLargeError extends Error {
+}
+function sendProxyError(res, status, error) {
+    res.setHeader('X-Proxy-Error', '1');
+    res.status(status).json({ error });
+}
+/** Read the upload, refusing it as soon as it passes `limit`. */
+async function readBoundedBody(req, limit) {
+    const declared = Number(req.headers['content-length']);
+    if (Number.isFinite(declared) && declared > limit)
+        throw new RequestBodyTooLargeError();
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of req) {
+        const buf = Buffer.from(chunk);
+        size += buf.byteLength;
+        if (size > limit)
+            throw new RequestBodyTooLargeError();
+        chunks.push(buf);
+    }
+    return Buffer.concat(chunks);
+}
+/** Upstream response headers as an ordered list, each `Set-Cookie` separate. */
+function upstreamHeaderList(upstream) {
+    const list = [];
+    upstream.headers.forEach((value, name) => {
+        if (name !== 'set-cookie')
+            list.push([name, value]);
+    });
+    for (const cookie of upstream.headers.getSetCookie())
+        list.push(['set-cookie', cookie]);
+    return list;
+}
+/** Build the upstream request with secrets injected, or name the forbidden one. */
+async function prepareUpstream(secretProxy, head, rawBody) {
+    const headers = foldRawRequestHeaders(stripRawRequestHeaders(head.headers));
+    const hmacSpec = headers[HMAC_SIGN_HEADER];
+    delete headers[HMAC_SIGN_HEADER];
+    const acceptEncoding = rawAcceptEncoding(headers);
+    if (acceptEncoding !== undefined)
+        headers['accept-encoding'] = acceptEncoding;
+    let hostname = '';
+    try {
+        hostname = new URL(head.url).hostname;
+    }
+    catch {
+        // Malformed URL: the upstream fetch rejects it below.
+    }
+    const injection = injectRequestSecrets(secretProxy, headers, head.url, hostname);
+    if ('forbidden' in injection)
+        return injection;
+    const method = head.method.toUpperCase();
+    let body;
+    if (rawBody.length > 0 && method !== 'GET' && method !== 'HEAD') {
+        body = unmaskRequestBody(secretProxy, headers, rawBody, hostname);
+    }
+    const signing = await applyHmacSigning(secretProxy, headers, hmacSpec, body, hostname);
+    if (signing)
+        return signing;
+    const init = { method: head.method, headers, redirect: 'manual' };
+    if (body)
+        init.body = body;
+    return { url: injection.cleanedUrl, init };
+}
+/** Write the response-head frame, then stream (or skip) the body. */
+function relayUpstream(res, head, upstream, secretProxy, detachClientClose) {
+    const hasBody = rawResponseHasBody(head.method, upstream.status) && upstream.body !== null;
+    // Text bodies pass the gunzip sniff and the secret scrub, so their length
+    // is not the upstream one; binary bodies pass through byte for byte.
+    const bodyRewritten = isTextContentType(upstream.headers.get('content-type') ?? '');
+    const headers = rawResponseHeaders({
+        method: head.method,
+        status: upstream.status,
+        headers: upstreamHeaderList(upstream),
+        bodyRewritten,
+    }).map(([name, value]) => [name, secretProxy.scrubResponse(value)]);
+    res.status(200);
+    res.setHeader('Content-Type', RAW_FETCH_CONTENT_TYPE);
+    res.setHeader('Cache-Control', 'no-store, no-cache');
+    res.write(encodeRawResponseFrame({
+        status: upstream.status,
+        statusText: upstream.statusText,
+        headers,
+        url: head.url,
+    }));
+    if (!hasBody || !upstream.body) {
+        void upstream.body?.cancel().catch(() => undefined);
+        res.end();
+        detachClientClose();
+        return;
+    }
+    streamUpstreamBody(res, upstream, secretProxy, detachClientClose);
+}
+/** Whether this `/api/fetch-proxy` request selected raw mode. */
+export function isRawFetchProxyRequest(req) {
+    return typeof req.headers[RAW_REQUEST_HEADER_LOWER] === 'string';
+}
+/** Whether this `/api/fetch-proxy` request asks what raw mode can do. */
+export function isRawFetchProbe(req) {
+    return req.headers[RAW_PROBE_HEADER_LOWER] !== undefined;
+}
+/** Answer the capability probe; nothing is fetched upstream. */
+export function answerRawFetchProbe(res, maxRequestBodyBytes) {
+    const reply = {
+        rawFetch: RAW_FETCH_PROTOCOL_VERSION,
+        requestBodyStreaming: false,
+        maxRequestBodyBytes,
+    };
+    res.setHeader('Cache-Control', 'no-store');
+    res.status(200).json(reply);
+}
+export async function handleRawFetchProxy(req, res, deps) {
+    const { secretProxy, logger = console } = deps;
+    const limit = deps.maxRequestBodyBytes ?? RAW_FETCH_BRIDGE_REQUEST_BODY_CAP;
+    const head = decodeRawRequestHead(String(req.headers[RAW_REQUEST_HEADER_LOWER]));
+    if (!head) {
+        sendProxyError(res, 400, `Malformed ${RAW_FETCH_REQUEST_HEADER} header`);
+        return;
+    }
+    let rawBody;
+    try {
+        rawBody = await readBoundedBody(req, limit);
+    }
+    catch (err) {
+        if (!(err instanceof RequestBodyTooLargeError))
+            throw err;
+        logger.warn(`[fetch-proxy:raw] ${head.method} ${head.url} → 413`);
+        res.setHeader('Connection', 'close');
+        sendProxyError(res, 413, `Request body exceeds the ${limit} byte limit of this float`);
+        return;
+    }
+    logger.log(`[fetch-proxy:raw] ${head.method} ${head.url}`);
+    const prepared = await prepareUpstream(secretProxy, head, rawBody);
+    if ('forbidden' in prepared) {
+        const { secretName, hostname } = prepared.forbidden;
+        logger.warn(`[fetch-proxy:raw] ${head.method} ${head.url} → 403 (secret "${secretName}")`);
+        sendProxyError(res, 403, `Secret "${secretName}" is not allowed for domain "${hostname}"`);
+        return;
+    }
+    const upstreamAbort = attachUpstreamAbort(res);
+    prepared.init.signal = upstreamAbort.controller.signal;
+    let upstream;
+    try {
+        upstream = await fetch(prepared.url, prepared.init);
+    }
+    catch (err) {
+        upstreamAbort.detach();
+        const message = err instanceof Error ? err.message : String(err);
+        logger.error(`[fetch-proxy:raw] ${head.method} ${head.url} ← 502 (${message})`);
+        sendProxyError(res, 502, `Proxy fetch failed: ${message}`);
+        return;
+    }
+    logger.log(`[fetch-proxy:raw] ${head.method} ${head.url} ← ${upstream.status}`);
+    if (isDecodedPartialResponse({ status: upstream.status, headers: upstreamHeaderList(upstream) })) {
+        upstreamAbort.detach();
+        await upstream.body?.cancel().catch(() => undefined);
+        logger.warn(`[fetch-proxy:raw] ${head.method} ${head.url} ← 206 encoded despite identity`);
+        sendProxyError(res, 502, 'Upstream answered a range request with an encoded partial body');
+        return;
+    }
+    relayUpstream(res, head, upstream, secretProxy, upstreamAbort.detach);
+}
+/**
+ * Mount raw mode ahead of the default `/api/fetch-proxy` handler; requests
+ * without `X-Slicc-Raw-Request` fall through to it untouched.
+ */
+export function registerRawFetchProxyRoute(app, deps) {
+    app.post('/api/fetch-proxy', (req, res, next) => {
+        if (isRawFetchProbe(req) && !isRawFetchProxyRequest(req)) {
+            answerRawFetchProbe(res, deps.maxRequestBodyBytes ?? RAW_FETCH_BRIDGE_REQUEST_BODY_CAP);
+            return;
+        }
+        if (!isRawFetchProxyRequest(req)) {
+            next();
+            return;
+        }
+        deps.activityTracker.recordActivity();
+        handleRawFetchProxy(req, res, deps).catch(next);
+    });
+}

package/dist/node-server/routes/fetch-proxy.d.ts

@@ -42,6 +42,29 @@ export declare function injectRequestSecrets(secretProxy: SecretProxyManager, he
 } | {
     cleanedUrl: string;
 };
+/**
+ * Apply the `x-slicc-hmac-sign` directive, if present: sign `body` with the
+ * named secret's real value and attach the hex result under the header the
+ * client requested. Mutates `headers` and always removes the sentinel
+ * (already stripped from `headers` by `buildForwardHeaders`'s skip-list;
+ * this only needs to read it off the raw request).
+ */
+export declare function applyHmacSigning(secretProxy: SecretProxyManager, headers: Record<string, string>, hmacSpec: string | undefined, body: Buffer | undefined, targetHostname: string): Promise<{
+    forbidden: ForbiddenSecret;
+} | undefined>;
+/**
+ * Unmask masked secrets in a text request body. Non-text bodies (git
+ * packfiles, octet-stream, images, …) are left untouched — `toString('utf-8')`
+ * on arbitrary bytes corrupts them, and masked values never appear in binary.
+ *
+ * Uses the request-side predicate so form POSTs count as text: swift-server
+ * unmasks them and a masked `client_secret` reaching an upstream OAuth token
+ * endpoint verbatim is a silent auth failure, not a corrupt-bytes risk (#2821).
+ * A form body then takes the encoding-aware `unmaskFormBody` path — a plain
+ * substring splice would corrupt it whenever the real secret carries a
+ * form-reserved character (base64 `+` / `/` / `=`).
+ */
+export declare function unmaskRequestBody(secretProxy: SecretProxyManager, headers: Record<string, string>, rawBody: Buffer, targetHostname: string): Buffer;
 /**
  * Buffer-aware UTF-8 secret scrubber. A `StringDecoder` keeps trailing partial
  * multi-byte sequences out of the scrub so codepoints straddling a chunk

package/dist/node-server/routes/fetch-proxy.js

@@ -116,7 +116,7 @@ export function injectRequestSecrets(secretProxy, headers, targetUrl, targetHost
  * (already stripped from `headers` by `buildForwardHeaders`'s skip-list;
  * this only needs to read it off the raw request).
  */
-async function applyHmacSigning(secretProxy, headers, hmacSpec, body, targetHostname) {
+export async function applyHmacSigning(secretProxy, headers, hmacSpec, body, targetHostname) {
     if (!hmacSpec)
         return undefined;
     const signResult = await secretProxy.signHmac(hmacSpec, body ?? Buffer.alloc(0), targetHostname);
@@ -142,7 +142,7 @@ async function applyHmacSigning(secretProxy, headers, hmacSpec, body, targetHost
  * substring splice would corrupt it whenever the real secret carries a
  * form-reserved character (base64 `+` / `/` / `=`).
  */
-function unmaskRequestBody(secretProxy, headers, rawBody, targetHostname) {
+export function unmaskRequestBody(secretProxy, headers, rawBody, targetHostname) {
     const contentType = headers['content-type'] ?? headers['Content-Type'] ?? '';
     if (!isTextRequestContentType(contentType) || !secretProxy.hasSecrets())
         return rawBody;
@@ -268,7 +268,8 @@ export function streamUpstreamBody(res, upstream, secretProxy, detachClientClose
     const decoded = isText
         ? createMaybeGunzipStream({
             onDecided: (inflating) => {
-                if (inflating)
+                // Raw mode (`fetch-proxy-raw.ts`) has already sent its head frame.
+                if (inflating && !res.headersSent)
                     res.removeHeader(FETCH_PROXY_CONTENT_LENGTH_HEADER);
             },
         })