querysub 0.490.0 → 0.492.0

package/.claude/settings.local.json

@@ -28,7 +28,8 @@
       "Bash(yarn ssh-a-claude *)",
       "WebSearch",
       "WebFetch(domain:blog.cloudflare.com)",
-      "WebFetch(domain:developers.cloudflare.com)"
+      "WebFetch(domain:developers.cloudflare.com)",
+      "PowerShell(yarn type *)"
     ]
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.490.0",
+  "version": "0.492.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-e-certs/certAuthority.ts

@@ -1,5 +1,6 @@
 import acme from "acme-client";
-import { generateRSAKeyPair, parseCert, privateKeyToPem } from "sliftutils/misc/https/certs";
+import crypto from "crypto";
+import { parseCert } from "sliftutils/misc/https/certs";
 import { cache, lazy } from "socket-function/src/caching";
 import { hasDNSWritePermissions, setRecord } from "../-b-authorities/dnsAuthority";
 import { magenta } from "socket-function/src/formatting/logColors";
@@ -13,6 +14,8 @@ import { timeInMinute } from "socket-function/src/misc";
 const archives = lazy(() => getArchives(`https_certs_3/`));
 // Expire EXPIRATION_THRESHOLD% of the way through the certificate's lifetime
 const EXPIRATION_THRESHOLD = 0.4;
+// Let's Encrypt only accepts RSA or ECDSA account keys (not Ed25519, which is all sliftutils/certs generates).
+const ACCOUNT_KEY_MODULUS_BITS = 2048;
 
 export const getHTTPSKeyCert = cache(async (domain: string): Promise<{ key: string; cert: string }> => {
     if (!await hasDNSWritePermissions()) {
@@ -112,13 +115,20 @@ let getAccountKey: () => string = lazy(() => {
     let keyCache = accountKeyCache.value;
     if (!keyCache) {
         console.log(`Generating new letsencrypt account key`);
-        const keyPair = generateRSAKeyPair();
-        keyCache = { key: privateKeyToPem(keyPair.privateKey) };
+        keyCache = { key: generateRSAPrivateKeyPem() };
         accountKeyCache.value = keyCache;
     }
     return keyCache.key;
 });
 
+function generateRSAPrivateKeyPem(): string {
+    return crypto.generateKeyPairSync("rsa", {
+        modulusLength: ACCOUNT_KEY_MODULUS_BITS,
+        publicKeyEncoding: { type: "spki", format: "pem" },
+        privateKeyEncoding: { type: "pkcs8", format: "pem" },
+    }).privateKey;
+}
+
 async function generateCert(config: {
     accountKey: string;
     domain: string;

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -43,9 +43,31 @@ import { isClient } from "../config2";
 
 const DEFAULT_MAX_LOCKS = 1000;
 
-// After this time we allow proxies to be reordered, even if there's flags that tell them not to be. 
+// After this time we allow proxies to be reordered, even if there's flags that tell them not to be.
 const MAX_PROXY_REORDER_BLOCK_TIME = timeInSecond * 10;
 
+// The proxy watcher reads all of its synced data through this indirection instead of importing
+//      authorityStorage directly, so the data source can be swapped out. This is what lets us replay
+//      captured function runs against an in-memory dataset in complete isolation (see functionReplay.ts).
+export interface PathValueReadSource {
+    getValueAtOrBeforeTime(pathStr: string, time: Time | undefined): PathValue | undefined;
+    isSynced(pathStr: string): boolean;
+    isParentSynced(pathStr: string): boolean;
+    getPathsFromParent(pathStr: string): Set<string> | undefined;
+    temporaryOverride<T>(overrides: PathValue[] | undefined, code: () => T): T;
+    DEBUG_hasAnyValues(pathStr: string): boolean;
+}
+let currentReadSource: PathValueReadSource = authorityStorage;
+export function runWithPathValueReadSource<T>(source: PathValueReadSource, code: () => T): T {
+    let prev = currentReadSource;
+    currentReadSource = source;
+    try {
+        return code();
+    } finally {
+        currentReadSource = prev;
+    }
+}
+
 let nextSeqNum = 1;
 let nextOrderSeqNum = 1;
 
@@ -566,7 +588,7 @@ export class PathValueProxyWatcher {
             if (!watcher.permissionsChecker.checkPermissions(pathStr).allowed) {
                 if (
                     !watcher.hasAnyUnsyncedAccesses()
-                    && authorityStorage.DEBUG_hasAnyValues(pathStr)
+                    && currentReadSource.DEBUG_hasAnyValues(pathStr)
                     // HACK: Don't show warnings for some framework paths, because they are filling up the console logs
                     //  and don't really matter. We could just not request them, but at depth 3 is valid,
                     //  so we kind of have to request that. And at depth 2, it would require special case code,
@@ -589,14 +611,14 @@ export class PathValueProxyWatcher {
 
         const currentReadTime = watcher.options.forceReadLatest ? undefined : watcher.currentReadTime;
 
-        let pathValue = authorityStorage.getValueAtOrBeforeTime(pathStr, currentReadTime);
+        let pathValue = currentReadSource.getValueAtOrBeforeTime(pathStr, currentReadTime);
         if (!watcher.options.noSyncing) {
             // NOTE: If we have any value, we are always synced (that's what synced means, as if we aren't syncing,
             //  we delete any values, to prevent stale values from being used).
             if (!pathValue) {
                 // NOTE: We might not have a value simply due to reading too far back in time,
                 //  so we have to call isSynced just to be sure it is actually unsynced
-                if (!authorityStorage.isSynced(pathStr)) {
+                if (!currentReadSource.isSynced(pathStr)) {
                     for (let checker of this.isUnsyncCheckers) {
                         checker.value++;
                     }
@@ -604,7 +626,7 @@ export class PathValueProxyWatcher {
                 }
             }
             if (syncParentKeys) {
-                if (!authorityStorage.isParentSynced(pathStr)) {
+                if (!currentReadSource.isParentSynced(pathStr)) {
                     watcher.pendingUnsyncedParentAccesses.add(pathStr);
                 }
             }
@@ -940,7 +962,7 @@ export class PathValueProxyWatcher {
             return getKeys(pathValue?.value) as string[];
         }
 
-        let childPaths = authorityStorage.getPathsFromParent(pathStr);
+        let childPaths = currentReadSource.getPathsFromParent(pathStr);
 
         // We need to also get keys from pendingWrites
         {
@@ -996,7 +1018,7 @@ export class PathValueProxyWatcher {
         }
 
         if (symbol === syncedSymbol) {
-            return { value: authorityStorage.isSynced(pathStr) };
+            return { value: currentReadSource.isSynced(pathStr) };
         }
 
         // Proxies should be considered atomic, at least for the purpose of other proxies!
@@ -1066,7 +1088,7 @@ export class PathValueProxyWatcher {
             let { watchFunction, ...mostOptions } = options;
             doProxyOptions(mostOptions, () => {
                 try {
-                    let result = authorityStorage.temporaryOverride(options.overrides, () =>
+                    let result = currentReadSource.temporaryOverride(options.overrides, () =>
                         options.watchFunction()
                     );
                     // Clone, otherwise proxies get out of the watcher, which can result in accesses outside
@@ -1353,7 +1375,7 @@ export class PathValueProxyWatcher {
                         }
                     },
                     code() {
-                        return authorityStorage.temporaryOverride(options.overrides, () =>
+                        return currentReadSource.temporaryOverride(options.overrides, () =>
                             runCodeWithDatabase(proxy, baseFunction)
                         );
                     },
@@ -1457,8 +1479,8 @@ export class PathValueProxyWatcher {
                         setTimeout(() => {
                             if (watcher.syncRunCount !== syncRunCount) return;
                             if (watcher.disposed) return;
-                            let remainingUnsynced = Array.from(watcher.lastUnsyncedAccesses).filter(x => !authorityStorage.isSynced(x));
-                            let remainingUnsyncedParent = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !authorityStorage.isParentSynced(x));
+                            let remainingUnsynced = Array.from(watcher.lastUnsyncedAccesses).filter(x => !currentReadSource.isSynced(x));
+                            let remainingUnsyncedParent = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !currentReadSource.isParentSynced(x));
                             console.warn(red(`Did not sync ${watcher.debugName} after 30 seconds. Either we had a lot synchronous block, or we will never received the values we are waiting for.`), { remainingUnsynced, remainingUnsyncedParent }, watcher.options.watchFunction);
                         }, 30 * 1000);
                         setTimeout(() => {
@@ -1472,8 +1494,8 @@ export class PathValueProxyWatcher {
                             //      we have had since this watcher last synced. If it is > 50% of the time...
                             //      then synchronous lag is the issue.
 
-                            let reallyUnsyncedAccesses = Array.from(watcher.lastUnsyncedAccesses).filter(x => !authorityStorage.isSynced(x));
-                            let reallyUnsyncedParentAccesses = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !authorityStorage.isParentSynced(x));
+                            let reallyUnsyncedAccesses = Array.from(watcher.lastUnsyncedAccesses).filter(x => !currentReadSource.isSynced(x));
+                            let reallyUnsyncedParentAccesses = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !currentReadSource.isParentSynced(x));
 
                             if (reallyUnsyncedAccesses.length !== 0 || reallyUnsyncedParentAccesses.length !== 0) {
                                 let notWatchingUnsynced = reallyUnsyncedAccesses.filter(x => !remoteWatcher.debugIsWatchingPath(x));
@@ -1705,12 +1727,12 @@ export class PathValueProxyWatcher {
             if (watcher.options.commitAllRuns) return false;
             // NOTE: We COULD remove any synced values from lastUnsyncedAccesses, however... we will generally sync all values at once, so we don't really need to optimize the cascading case here. Also... deleting values requires cloning while we iterate, as well as mutating the set, which probably makes the non-cascading case slower.
             for (let path of watcher.lastUnsyncedAccesses) {
-                if (!authorityStorage.isSynced(path)) {
+                if (!currentReadSource.isSynced(path)) {
                     return true;
                 }
             }
             for (let path of watcher.lastUnsyncedParentAccesses) {
-                if (!authorityStorage.isParentSynced(path)) {
+                if (!currentReadSource.isParentSynced(path)) {
                     return true;
                 }
             }
@@ -1723,13 +1745,13 @@ export class PathValueProxyWatcher {
         function logUnsynced() {
             let anyLogged = false;
             for (let path of watcher.lastUnsyncedAccesses) {
-                if (!authorityStorage.isSynced(path)) {
+                if (!currentReadSource.isSynced(path)) {
                     console.log(yellow(`    Waiting for ${path}`));
                     anyLogged = true;
                 }
             }
             for (let path of watcher.lastUnsyncedParentAccesses) {
-                if (!authorityStorage.isParentSynced(path)) {
+                if (!currentReadSource.isParentSynced(path)) {
                     console.log(yellow(`    Waiting for parent ${path}`));
                     anyLogged = true;
                 }
@@ -1997,7 +2019,7 @@ export class PathValueProxyWatcher {
     public async commitFunction<Result = void>(
         options: Omit<WatcherOptions<Result>, "onResultUpdated" | "onWriteCommitted">,
         config?: {
-            onWritesCommitted?: (writes: PathValue[]) => void;
+            onWritesCommitted?: (writes: PathValue[], watcher: SyncWatcher) => void;
         }
     ): Promise<Result> {
         options = {
@@ -2029,7 +2051,7 @@ export class PathValueProxyWatcher {
                     onResult(result.result);
                 }
                 if (writes && config?.onWritesCommitted) {
-                    config.onWritesCommitted(writes);
+                    config.onWritesCommitted(writes, watcher);
                 }
             }
         });
@@ -2131,12 +2153,12 @@ export class PathValueProxyWatcher {
     public reuseLastWatches() {
         let watcher = this.getTriggeredWatcher();
         for (let path of watcher.lastUnsyncedAccesses) {
-            if (!authorityStorage.isSynced(path)) {
+            if (!currentReadSource.isSynced(path)) {
                 watcher.pendingUnsyncedAccesses.add(path);
             }
         }
         for (let path of watcher.lastUnsyncedParentAccesses) {
-            if (!authorityStorage.isParentSynced(path)) {
+            if (!currentReadSource.isParentSynced(path)) {
                 watcher.pendingUnsyncedParentAccesses.add(path);
             }
         }

package/src/3-path-functions/PathFunctionRunner.ts

@@ -24,6 +24,7 @@ import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
 import type { DeployProgress } from "../4-deploy/deployFunctions";
 import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
 import { PathRouter } from "../0-path-value-core/PathRouter";
+import { FunctionCaptureController, isCapturing, recordCapturedCall } from "./functionCapture";
 setImmediate(() => import("../4-querysub/Querysub"));
 
 let functionCallOuterCount = 0;
@@ -229,6 +230,7 @@ export class PathFunctionRunner {
         filterSelector?: FilterSelector;
     }) {
         SocketFunction.expose(FunctionPreloadController);
+        SocketFunction.expose(FunctionCaptureController);
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
@@ -777,8 +779,26 @@ export class PathFunctionRunner {
                         }
                     },
                 }, {
-                    onWritesCommitted(writes) {
+                    onWritesCommitted(writes, watcher) {
                         finalWrites = writes;
+                        // Capture only runs that actually authored the Result (not nooped/partial runs),
+                        //  so a replayed call maps to exactly one recorded output state.
+                        if (isCapturing()) {
+                            let resultsPathStr = getProxyPath(() => functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId].Results[callSpec.CallId]);
+                            let resultWrite = writes.find(w => w.path === resultsPathStr);
+                            if (resultWrite) {
+                                recordCapturedCall({
+                                    callSpec,
+                                    functionSpec,
+                                    writes,
+                                    watcher,
+                                    resultsPathStr,
+                                    resultWriteTime: resultWrite.time,
+                                    timeTaken: Date.now() - startTime,
+                                    evalTime,
+                                });
+                            }
+                        }
                     },
                 });
             } catch (e: any) {

package/src/3-path-functions/functionCapture.ts

@@ -0,0 +1,213 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { runInfinitePoll } from "socket-function/src/batching";
+import { timeInSecond } from "socket-function/src/misc";
+import { authorityStorage, compareTime, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getDomain } from "../config";
+import { CaptureFileHeader, CaptureRecord, CAPTURE_FORMAT_VERSION, encodeCaptureFile, encodeCaptureRecord } from "./functionCaptureFormat";
+import type { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import type { CallSpec, FunctionSpec } from "./PathFunctionRunner";
+
+// How long to wait after a run commits its Result before treating that run as final and flushing
+//      it into the capture buffer. Long enough to catch a fast rejection (which re-runs the call and
+//      supersedes this Result), but well under the Result path's event expiry, so the re-check below
+//      can still read the value to confirm it's still ours.
+export const CAPTURE_FLUSH_DELAY = timeInSecond * 5;
+const CAPTURE_FLUSH_POLL_INTERVAL = timeInSecond;
+
+export interface CaptureStatus {
+    capturing: boolean;
+    // Records confirmed final + still awaiting the flush delay.
+    callCount: number;
+    finalizedCount: number;
+    pendingCount: number;
+    // Bytes of the finalized record frames (excludes header + pending).
+    byteCount: number;
+    startedAt: number;
+}
+
+interface PendingCapture {
+    record: CaptureRecord;
+    resultsPathStr: string;
+    resultWriteTime: Time;
+}
+
+let capturing = false;
+let captureStartedAt = 0;
+let finalizedFrames: Buffer[] = [];
+let finalizedCount = 0;
+let finalizedBytes = 0;
+let pendingCaptures: PendingCapture[] = [];
+
+export function isCapturing() {
+    return capturing;
+}
+
+function resetCaptureBuffer() {
+    finalizedFrames = [];
+    finalizedCount = 0;
+    finalizedBytes = 0;
+    pendingCaptures = [];
+}
+
+function readValue(pathValue: PathValue): unknown {
+    return pathValueSerializer.getPathValue(pathValue, "noMutate");
+}
+
+export function recordCapturedCall(config: {
+    callSpec: CallSpec;
+    functionSpec: FunctionSpec;
+    writes: PathValue[];
+    watcher: SyncWatcher;
+    resultsPathStr: string;
+    resultWriteTime: Time;
+    timeTaken: number;
+    evalTime: number;
+}): void {
+    if (!capturing) return;
+
+    let { callSpec, functionSpec, writes, watcher } = config;
+
+    let reads: CaptureRecord["reads"] = [];
+    for (let values of watcher.pendingAccesses.values()) {
+        for (let { pathValue } of values.values()) {
+            reads.push({ path: pathValue.path, value: readValue(pathValue), time: pathValue.time });
+        }
+    }
+
+    let undefinedReads: CaptureRecord["undefinedReads"] = [];
+    for (let [readTime, paths] of watcher.pendingEpochAccesses) {
+        for (let path of paths) {
+            undefinedReads.push({ path, readTime });
+        }
+    }
+
+    let parentKeyReads = Array.from(watcher.lastWatches.parentPaths);
+
+    let writeRecords: CaptureRecord["writes"] = writes.map(w => ({
+        path: w.path,
+        value: readValue(w),
+        time: w.time,
+        isTransparent: w.isTransparent,
+        event: w.event,
+    }));
+
+    let record: CaptureRecord = {
+        callId: callSpec.CallId,
+        functionSpec,
+        argsEncoded: callSpec.argsEncoded,
+        runAtTime: callSpec.runAtTime,
+        callerMachineId: callSpec.callerMachineId,
+        callerIP: callSpec.callerIP,
+        reads,
+        undefinedReads,
+        parentKeyReads,
+        writes: writeRecords,
+        timeTaken: config.timeTaken,
+        evalTime: config.evalTime,
+        capturedAt: Date.now(),
+    };
+
+    pendingCaptures.push({
+        record,
+        resultsPathStr: config.resultsPathStr,
+        resultWriteTime: config.resultWriteTime,
+    });
+}
+
+// A record is only kept if the Result path still holds the exact write from this run. If it was
+//      rejected or clobbered by another runner, the current value's time won't match, and we drop it.
+function isStillOurResult(pending: PendingCapture): boolean {
+    let current = authorityStorage.getValueAtOrBeforeTime(pending.resultsPathStr, undefined);
+    if (!current) return false;
+    return compareTime(current.time, pending.resultWriteTime) === 0;
+}
+
+function finalizePending(force: boolean) {
+    if (pendingCaptures.length === 0) return;
+    let now = Date.now();
+    let stillPending: PendingCapture[] = [];
+    for (let pending of pendingCaptures) {
+        if (!force && now - pending.record.capturedAt < CAPTURE_FLUSH_DELAY) {
+            stillPending.push(pending);
+            continue;
+        }
+        if (!isStillOurResult(pending)) continue;
+        let frame = encodeCaptureRecord(pending.record);
+        finalizedFrames.push(frame);
+        finalizedCount++;
+        finalizedBytes += frame.length;
+    }
+    pendingCaptures = stillPending;
+}
+
+let flushPollStarted = false;
+function ensureFlushPoll() {
+    if (flushPollStarted) return;
+    flushPollStarted = true;
+    runInfinitePoll(CAPTURE_FLUSH_POLL_INTERVAL, () => finalizePending(false));
+}
+
+function getStatus(): CaptureStatus {
+    return {
+        capturing,
+        callCount: finalizedCount + pendingCaptures.length,
+        finalizedCount,
+        pendingCount: pendingCaptures.length,
+        byteCount: finalizedBytes,
+        startedAt: captureStartedAt,
+    };
+}
+
+class FunctionCaptureControllerBase {
+    public async startCapture(): Promise<CaptureStatus> {
+        resetCaptureBuffer();
+        captureStartedAt = Date.now();
+        capturing = true;
+        ensureFlushPoll();
+        return getStatus();
+    }
+
+    public async getStatus(): Promise<CaptureStatus> {
+        return getStatus();
+    }
+
+    // Stops capturing, force-finalizes anything still within the flush delay, and returns the
+    //      encoded capture file. The buffer is retained until clearCapture so it can be re-downloaded.
+    public async stopCapture(): Promise<Buffer> {
+        capturing = false;
+        finalizePending(true);
+        let header: CaptureFileHeader = {
+            version: CAPTURE_FORMAT_VERSION,
+            createdAt: Date.now(),
+            domainName: getDomain(),
+            recordCount: finalizedCount,
+        };
+        return encodeCaptureFile(header, finalizedFrames);
+    }
+
+    public async clearCapture(): Promise<CaptureStatus> {
+        capturing = false;
+        resetCaptureBuffer();
+        captureStartedAt = 0;
+        return getStatus();
+    }
+}
+
+export const FunctionCaptureController = SocketFunction.register(
+    "FunctionCaptureController-2f1b8c4a-6e9d-4a71-9b2e-8c5f0d3a1e77",
+    new FunctionCaptureControllerBase(),
+    () => ({
+        startCapture: {},
+        getStatus: {},
+        stopCapture: {},
+        clearCapture: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);

package/src/3-path-functions/functionCaptureFormat.ts

@@ -0,0 +1,103 @@
+import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
+import type { Time } from "../0-path-value-core/pathValueCore";
+import type { FunctionSpec } from "./PathFunctionRunner";
+
+// Custom binary container for captured function runs. Layout:
+//      [MAGIC (8 bytes)]
+//      [uint32 LE headerLength][header JSON (utf8)]
+//      repeated: [uint32 LE recordLength][record (CBOR)]
+// The header is JSON (not CBOR) so future readers can pull version + metadata without
+//      understanding the record encoding, which lets us evolve the record format while
+//      staying backwards compatible.
+export const CAPTURE_MAGIC = Buffer.from("QSFNCAP\0", "latin1");
+export const CAPTURE_FORMAT_VERSION = 1;
+
+export interface CaptureFileHeader {
+    version: number;
+    createdAt: number;
+    domainName?: string;
+    nodeId?: string;
+    entryPoint?: string;
+    recordCount: number;
+    // Free-form metadata, so we can add fields without a version bump.
+    [key: string]: unknown;
+}
+
+// One captured, fully-resolved function run. Enough to replay it in isolation and compare
+//      the output state.
+export interface CaptureRecord {
+    callId: string;
+    functionSpec: FunctionSpec;
+    argsEncoded: string;
+    // The explicit timestamp the run executed at. Drives all determinism on replay
+    //      (Querysub.time()/Querysub.nextId() are derived from it).
+    runAtTime: Time;
+    callerMachineId: string;
+    callerIP: string;
+
+    // Values the proxy resolved for each read, keyed by the value's own write time so the
+    //      read can be reconstructed via getValueAtOrBeforeTime on replay.
+    reads: { path: string; value: unknown; time: Time }[];
+    // Reads that resolved to no value (undefined) at the given read time.
+    undefinedReads: { path: string; readTime?: Time }[];
+    // Paths whose child keys were enumerated (getKeys / Object.keys), needed so lookup
+    //      enumeration replays identically.
+    parentKeyReads: string[];
+
+    // Committed writes (the output state to compare against on replay).
+    writes: { path: string; value: unknown; time: Time; isTransparent?: boolean; event?: boolean }[];
+
+    timeTaken: number;
+    evalTime: number;
+    capturedAt: number;
+}
+
+function writeUInt32(value: number): Buffer {
+    let buffer = Buffer.alloc(4);
+    buffer.writeUInt32LE(value, 0);
+    return buffer;
+}
+
+export function encodeCaptureRecord(record: CaptureRecord): Buffer {
+    let body = encodeCborx(record);
+    return Buffer.concat([writeUInt32(body.length), body]);
+}
+
+export function encodeCaptureFile(header: CaptureFileHeader, recordFrames: Buffer[]): Buffer {
+    let headerBuffer = Buffer.from(JSON.stringify(header), "utf8");
+    return Buffer.concat([CAPTURE_MAGIC, writeUInt32(headerBuffer.length), headerBuffer, ...recordFrames]);
+}
+
+export function decodeCaptureFile(buffer: Buffer): { header: CaptureFileHeader; records: CaptureRecord[] } {
+    if (buffer.length < CAPTURE_MAGIC.length + 4) {
+        throw new Error(`Capture file too small to be valid (${buffer.length} bytes)`);
+    }
+    let magic = buffer.subarray(0, CAPTURE_MAGIC.length);
+    if (!magic.equals(CAPTURE_MAGIC)) {
+        throw new Error(`Capture file has invalid magic. Expected ${JSON.stringify(CAPTURE_MAGIC.toString("latin1"))}, was ${JSON.stringify(magic.toString("latin1"))}`);
+    }
+    let offset = CAPTURE_MAGIC.length;
+    let headerLength = buffer.readUInt32LE(offset);
+    offset += 4;
+    if (offset + headerLength > buffer.length) {
+        throw new Error(`Capture file header length (${headerLength}) exceeds file size (${buffer.length})`);
+    }
+    let header = JSON.parse(buffer.subarray(offset, offset + headerLength).toString("utf8")) as CaptureFileHeader;
+    offset += headerLength;
+
+    let records: CaptureRecord[] = [];
+    while (offset < buffer.length) {
+        if (offset + 4 > buffer.length) {
+            throw new Error(`Capture file truncated reading record length at offset ${offset}`);
+        }
+        let recordLength = buffer.readUInt32LE(offset);
+        offset += 4;
+        if (offset + recordLength > buffer.length) {
+            throw new Error(`Capture file truncated reading record body at offset ${offset} (length ${recordLength}, file size ${buffer.length})`);
+        }
+        let record = decodeCborx<CaptureRecord>(buffer.subarray(offset, offset + recordLength));
+        records.push(record);
+        offset += recordLength;
+    }
+    return { header, records };
+}