run402 4.55.0 → 4.56.1

package/sdk/dist/node/gitvault-publication.js

@@ -40,6 +40,7 @@ import { LocalError, isRun402Error } from "../errors.js";
 import { fetchGitvaultObjectBytes } from "./gitvault-edge-fetch.js";
 import { GITVAULT_FORMAT, GITVAULT_GENESIS_EPOCH, GITVAULT_GENESIS_GENERATION, GITVAULT_HEX16_RE, GITVAULT_OID40_RE, GITVAULT_SUITE, attemptKeyCommitment, bytesToHex, checkFreshEpochKeyAgainstPriorKeys, checkHPartition, computeRotationId, computeTargetPartitionDigest, deriveDigestKey, deriveObjectKey, ekFingerprint, epochRotationKeyCommitment, formatGitvaultTimestamp, fromBase64url, hexToBytes, jcs, keyEnvelopeLedgerId, keyedCommitment, newGitvaultId, newHex32, nextEpoch, objectsetContent, openBindingPreimage, openEpochRotationForRecipient, openFrame, parseGitvaultStrict, parseRotateEpochPayload, pinManifestLedgerId, randomBytes, sealFrame, sealKeyEnvelope, sha256Hex, signGitvaultObject, storedBytes, storedBytesSha256, toBase64url, verifyGitvaultObject, } from "../namespaces/gitvault.crypto.js";
 import { GITVAULT_ZERO_SHA256_SENTINEL } from "../namespaces/gitvault.types.js";
+import { gitvaultCheckpointStaleness } from "../namespaces/gitvault.js";
 import { crossProfileGitvaultHint } from "./gitvault-profile-scan.js";
 import { GITVAULT_DEPLOY_REF, hardenedGit, hasObject, isAncestor } from "./gitvault-snapshot.js";
 // ─── Constants (constants.json) ──────────────────────────────────────────────
@@ -1189,6 +1190,23 @@ export class GitvaultVault {
     repoDir;
     now;
     budget;
+    /**
+     * gitvault-clone-scaling (bench P2): the CURRENT walk window's prefetched
+     * bytes — head bytes (bounded-concurrent direct reads) and carrier frames
+     * (one batched getObjects), keyed by storage path — filled by
+     * `verifyToNewest`'s per-page prefetch and `restoreObjectsInto`'s
+     * backward-window prefetch, consulted by {@link readCachedHeadBytes} /
+     * {@link openMaterializeCarriers} before they pay a network read. Transient (REPLACED each page, so memory is
+     * bounded by one listing page) and UNTRUSTED: every consumer sha-checks
+     * an entry against the exact value it would check network bytes against
+     * (the listing's `stored_bytes_sha256`, a receipt's `ciphertext_sha256`),
+     * the same discipline as the keystore object cache — a wrong or stale
+     * entry is a MISS, never a verification bypass. Deliberately NOT the
+     * keystore cache: that cache's eviction window is a handful of newest
+     * generations by design, so routing a whole page through it would evict
+     * the very bytes the ordered walk is about to read.
+     */
+    walkPrefetch = null;
     retries;
     servicePublicKey;
     genesisCache = null;
@@ -1228,6 +1246,39 @@ export class GitvaultVault {
     }
     kRepo() { return hexToBytes(this.repoFile().k_repo_hex); }
     epoch() { return this.repoFile().epoch; }
+    /**
+     * gitvault-clone-scaling (P3): staleness of the newest checkpoint coverage
+     * this checkout has locally learned, measured at `newestGeneration`. Reads
+     * the keystore AFTER the caller's own persist (a checkpoint-form push has
+     * already recorded its fresh coverage by the time its result is built), so
+     * a compacting push reports itself current. Pure + never-throwing by way of
+     * the helper; unknown coverage reads as `{0, advised: false}` — silent.
+     */
+    checkpointStalenessNow(newestGeneration) {
+        return gitvaultCheckpointStaleness({ newest_generation: newestGeneration, covers_through_generation: this.repoFile().checkpoint_covers_through ?? null });
+    }
+    /**
+     * gitvault-clone-scaling (bench P2): bounded-concurrent direct reads of
+     * many generation-addressed head paths. Heads deliberately do NOT ride
+     * `getObjects` — the `object-reads` presign batch is CARRIER-ONLY by wire
+     * design (see `getObjectsBytes`'s fail-closed doc comment; the live probe
+     * that caught this recorded `getObjects paths=67 FAILED 1ms` followed by
+     * 25 serial singles) — so head bytes parallelize as the SAME direct GETs
+     * the ordered walk itself would issue, just early and overlapped. A
+     * per-path failure resolves `null` (the walk's own read owns that
+     * failure and its envelope); results are raw and UNTRUSTED — callers
+     * sha-check before use, per `walkPrefetch`'s contract.
+     */
+    async prefetchHeadsConcurrent(paths) {
+        const map = new Map();
+        const fetched = await mapBounded(paths, GITVAULT_TRANSPORT_CONCURRENCY, (path) => this.transport.getObject({ repo_id: this.repoId, path }).catch(() => null));
+        for (let i = 0; i < paths.length; i++) {
+            const b = fetched[i];
+            if (b)
+                map.set(paths[i], b);
+        }
+        return map;
+    }
     git() {
         if (!this.repoDir)
             fail("GITVAULT_REPO_DIR_REQUIRED", "this operation needs the local git repository (repo_dir)", "gitvault publication");
@@ -1441,6 +1492,12 @@ export class GitvaultVault {
         let request = { after_generation: anchor, limit: String(GITVAULT_MAX_HEADS_PER_LISTING_PAGE) };
         let verified = 0;
         const rotations = [];
+        // gitvault-clone-scaling (P3): coverage this walk LEARNS. A checkpoint
+        // head names it outright; a walk anchored at GENESIS that completes
+        // without seeing one proves coverage = genesis. A partial (non-genesis)
+        // walk that sees none proves nothing and persists nothing.
+        let walkCheckpointCoverage = null;
+        const walkedFromGenesis = anchor === GITVAULT_GENESIS_GENERATION;
         // Consumed by (at most) the FIRST for(;;) iteration below — a listHeads
         // page this call never had to ask the network for, because the state
         // read above already proved it (0 entries: pin already current; 1 entry:
@@ -1543,6 +1600,84 @@ export class GitvaultVault {
                 : await this.transport.listHeads({ repo_id: this.repoId, ...request });
             syntheticEntries = null;
             progress = verifyHeadsListingPage(page, request, progress, this.repoId);
+            // gitvault-clone-scaling (bench P2): the page just verified names every
+            // entry's generation + stored_bytes_sha256, and the BYTES reads are
+            // independent — verification (and decryption) order is a LOCAL
+            // obligation. Bounded by the remaining verification budget, this
+            // page's per-head SERIAL reads are replaced by: the cache-missing
+            // HEAD bytes as bounded-CONCURRENT direct reads (heads cannot ride
+            // the object-reads presign batch — it is carrier-only by wire design;
+            // see prefetchHeadsConcurrent), then — the same split one level
+            // deeper — ONE batched getObjects for the ref_state/retention_roots
+            // FRAMES each decrypt-validated head will open (their paths +
+            // expected ciphertext hashes parse out of the head bytes just
+            // fetched; parsing is local CPU). Results land in the transient
+            // `walkPrefetch` map (NOT the keystore cache — its eviction window is
+            // smaller than a page; see the field's doc). Failure fidelity: only a
+            // hash-matching result is kept — an absent, mismatched, or
+            // unparseable entry (or a prefetch that fails outright) leaves its
+            // slot empty, and the ordered loop's own per-head reads reproduce the
+            // exact unbatched envelopes. Single-miss sets skip their prefetch
+            // (one direct read costs the same).
+            {
+                const prefetch = new Map();
+                this.walkPrefetch = prefetch;
+                try {
+                    const wanted = page.heads.slice(0, Math.max(0, this.budget - verified));
+                    const cachedHeadIfMatching = (e) => {
+                        const cached = this.keystore.readCachedHead(this.repoId, e.generation);
+                        return cached && sha256Hex(cached.bytes) === e.stored_bytes_sha256 ? cached.bytes : null;
+                    };
+                    const missingHeads = wanted.filter((e) => cachedHeadIfMatching(e) === null);
+                    if (missingHeads.length > 1) {
+                        const fetched = await this.prefetchHeadsConcurrent(missingHeads.map((e) => gitvaultPaths.head(e.generation)));
+                        for (const e of missingHeads) {
+                            const path = gitvaultPaths.head(e.generation);
+                            const bytes = fetched.get(path) ?? null;
+                            if (bytes && sha256Hex(bytes) === e.stored_bytes_sha256)
+                                prefetch.set(path, bytes);
+                        }
+                    }
+                    if (decryptValidate && wanted.length > 1) {
+                        const carriers = [];
+                        for (const e of wanted) {
+                            const headBytes = prefetch.get(gitvaultPaths.head(e.generation)) ?? cachedHeadIfMatching(e);
+                            if (!headBytes)
+                                continue;
+                            let parsed;
+                            try {
+                                parsed = parseGitvaultStrict(new TextDecoder().decode(headBytes));
+                            }
+                            catch {
+                                continue; // the ordered loop's checkChainLink owns rejecting it
+                            }
+                            for (const w of [
+                                { receipt: parsed.ref_state, path: gitvaultPaths.refState(parsed.ref_state.object_id) },
+                                { receipt: parsed.retention_roots, path: gitvaultPaths.retentionRoots(parsed.retention_roots.object_id) },
+                            ]) {
+                                const cached = this.keystore.readCachedCarrier(this.repoId, w.receipt.object_id);
+                                if (cached && sha256Hex(cached.bytes) === w.receipt.ciphertext_sha256)
+                                    continue;
+                                carriers.push({ path: w.path, sha: w.receipt.ciphertext_sha256 });
+                            }
+                        }
+                        if (carriers.length > 1) {
+                            const fetched = await this.transport.getObjects({ repo_id: this.repoId, paths: carriers.map((c) => c.path) });
+                            for (let i = 0; i < carriers.length; i++) {
+                                const bytes = fetched[i] ?? null;
+                                if (bytes && sha256Hex(bytes) === carriers[i].sha)
+                                    prefetch.set(carriers[i].path, bytes);
+                            }
+                        }
+                    }
+                }
+                catch {
+                    // A batch that fails OUTRIGHT (network, not a per-slot null) must
+                    // not introduce a failure mode the unbatched walk never had — the
+                    // ordered loop below re-reads what it needs itself and fails (or
+                    // succeeds, if the fault was transient) with its own envelopes.
+                }
+            }
             for (const entry of page.heads) {
                 if (verified >= this.budget) {
                     if (persist)
@@ -1581,6 +1716,8 @@ export class GitvaultVault {
                 prevEpoch = head.epoch;
                 pin = { generation: head.generation, head_sha256: entry.stored_bytes_sha256, pinned_at: formatGitvaultTimestamp(this.now()) };
                 lastHead = head;
+                if (head.checkpoint)
+                    walkCheckpointCoverage = head.checkpoint.covers_through_generation;
                 verified += 1;
                 // The chain walk ALWAYS continues below regardless of decrypt
                 // outcome (`tryDecrypt` never throws) — `!decryptFailure` just stops
@@ -1599,6 +1736,15 @@ export class GitvaultVault {
                 break;
             request = next;
         }
+        // P3: reaching here means the walk COMPLETED (a budget pause throws
+        // above) — persist whatever coverage it proved. A genesis-anchored walk
+        // is authoritative for its whole history, so no checkpoint seen means
+        // coverage = genesis, honestly.
+        if (persist) {
+            const learned = walkCheckpointCoverage ?? (walkedFromGenesis ? GITVAULT_GENESIS_GENERATION : null);
+            if (learned !== null)
+                this.keystore.updateRepo(this.repoId, { checkpoint_covers_through: learned });
+        }
         // Catch-up: a call with NOTHING new to walk (this repo's chain-verified
         // pin was already at `pin`/`lastHead` — e.g. an EARLIER, decrypt-blind
         // `verifyToNewest({})` call already advanced `head_pin` to the newest,
@@ -1700,7 +1846,16 @@ export class GitvaultVault {
         const cached = this.keystore.readCachedHead(this.repoId, generation);
         if (cached && sha256Hex(cached.bytes) === expectedSha256)
             return cached.bytes;
-        const bytes = await this.transport.getObject({ repo_id: this.repoId, path: gitvaultPaths.head(generation) });
+        // gitvault-clone-scaling (P2): a page-prefetched head serves exactly as
+        // a network fetch would — sha-checked here against the SAME expected
+        // value, then cache-warmed. A miss/mismatch falls through to the read.
+        const path = gitvaultPaths.head(generation);
+        const prefetched = this.walkPrefetch?.get(path);
+        if (prefetched && sha256Hex(prefetched) === expectedSha256) {
+            this.keystore.writeCachedHead(this.repoId, generation, expectedSha256, prefetched);
+            return prefetched;
+        }
+        const bytes = await this.transport.getObject({ repo_id: this.repoId, path });
         if (bytes && sha256Hex(bytes) === expectedSha256)
             this.keystore.writeCachedHead(this.repoId, generation, expectedSha256, bytes);
         return bytes;
@@ -1795,15 +1950,22 @@ export class GitvaultVault {
         const cachedRoots = this.keystore.readCachedCarrier(this.repoId, rootsReceipt.object_id);
         const refStateHit = Boolean(cachedRefState && sha256Hex(cachedRefState.bytes) === refStateReceipt.ciphertext_sha256);
         const rootsHit = Boolean(cachedRoots && sha256Hex(cachedRoots.bytes) === rootsReceipt.ciphertext_sha256);
+        // gitvault-clone-scaling (P2): a page-prefetched frame serves exactly as
+        // a fetched one — sha-checked against the receipt's ciphertext hash; a
+        // miss/mismatch falls through to the batched network read below.
+        const preRefState = !refStateHit ? (this.walkPrefetch?.get(refStatePath) ?? null) : null;
+        const refStatePre = preRefState && sha256Hex(preRefState) === refStateReceipt.ciphertext_sha256 ? preRefState : null;
+        const preRoots = !rootsHit ? (this.walkPrefetch?.get(rootsPath) ?? null) : null;
+        const rootsPre = preRoots && sha256Hex(preRoots) === rootsReceipt.ciphertext_sha256 ? preRoots : null;
         const missingPaths = [];
-        if (!refStateHit)
+        if (!refStateHit && !refStatePre)
             missingPaths.push(refStatePath);
-        if (!rootsHit)
+        if (!rootsHit && !rootsPre)
             missingPaths.push(rootsPath);
         const fetched = missingPaths.length > 0 ? await this.transport.getObjects({ repo_id: this.repoId, paths: missingPaths }) : [];
         let next = 0;
-        const refStateFrame = refStateHit ? cachedRefState.bytes : (fetched[next++] ?? null);
-        const rootsFrame = rootsHit ? cachedRoots.bytes : (fetched[next++] ?? null);
+        const refStateFrame = refStateHit ? cachedRefState.bytes : (refStatePre ?? fetched[next++] ?? null);
+        const rootsFrame = rootsHit ? cachedRoots.bytes : (rootsPre ?? fetched[next++] ?? null);
         const refState = this.decodeCarrierFrame("ref_state", refStateReceipt, refStateFrame, writerKey, keyOverride);
         const roots = this.decodeCarrierFrame("retention_roots", rootsReceipt, rootsFrame, writerKey, keyOverride);
         if (!refStateHit && refStateFrame)
@@ -2427,7 +2589,7 @@ export class GitvaultVault {
             if (published.outcome === "dry_run")
                 fail("GIT_COMMAND_FAILED", "internal: push() received a dry-run result it never requested", "publishing gitvault head");
             this.keystore.updateRepo(this.repoId, { last_ref_transaction: { generation: published.generation, transaction: options.transaction, at: formatGitvaultTimestamp(this.now()) } });
-            return { generation: published.generation, head_sha256: published.head_sha256, head: published.head, admission_record_sha256: published.admission_record_sha256, capture_receipt: published.capture_receipt, form: published.form, conflicts_retried: conflicts, refs: published.refs };
+            return { generation: published.generation, head_sha256: published.head_sha256, head: published.head, admission_record_sha256: published.admission_record_sha256, capture_receipt: published.capture_receipt, form: published.form, conflicts_retried: conflicts, refs: published.refs, checkpoint_staleness: this.checkpointStalenessNow(published.generation) };
         }
     }
     /**
@@ -2517,7 +2679,7 @@ export class GitvaultVault {
             // unreachable here — narrows `published` to `"admitted"` below.
             if (published.outcome === "dry_run")
                 fail("GIT_COMMAND_FAILED", "internal: publishCheckpoint() received a dry-run result it never requested", "publishing gitvault checkpoint");
-            return { generation: published.generation, head_sha256: published.head_sha256, head: published.head, admission_record_sha256: published.admission_record_sha256, capture_receipt: published.capture_receipt, form: published.form, conflicts_retried: conflicts, refs: published.refs };
+            return { generation: published.generation, head_sha256: published.head_sha256, head: published.head, admission_record_sha256: published.admission_record_sha256, capture_receipt: published.capture_receipt, form: published.form, conflicts_retried: conflicts, refs: published.refs, checkpoint_staleness: this.checkpointStalenessNow(published.generation) };
         }
     }
     /** Request a `retention_cutoff` ticket and check it binds THIS base head (and the service key, when one is pinned). */
@@ -2580,7 +2742,14 @@ export class GitvaultVault {
         // generation's head entirely.
         this.keystore.writeCachedHead(this.repoId, head.generation, hash, back);
         const pin = { generation: head.generation, head_sha256: hash, pinned_at: formatGitvaultTimestamp(this.now()) };
-        this.keystore.updateRepo(this.repoId, { head_pin: pin, materialized_pin: pin, verified_prefix: null });
+        this.keystore.updateRepo(this.repoId, {
+            head_pin: pin,
+            materialized_pin: pin,
+            verified_prefix: null,
+            // gitvault-clone-scaling (P3): a checkpoint-form head IS fresh
+            // coverage this checkout just learned first-hand.
+            ...(head.checkpoint ? { checkpoint_covers_through: head.checkpoint.covers_through_generation } : {}),
+        });
         return { outcome: "admitted", head_sha256: hash, admission_record_sha256: result.admission_record_sha256, capture_receipt: result.capture_receipt };
     }
     // ── epoch rotation (D193-D203, rev 42, change gitvault-human-envelopes) ──
@@ -2748,7 +2917,7 @@ export class GitvaultVault {
             // predecessor manifest back to compute confirmed() (D196) — resolves
             // it locally instead of a network object-reads round trip.
             this.keystore.updateRepo(this.repoId, { known_pin_manifest: { pin_manifest_version: nextVersion, stored_bytes_sha256: manifestSha, pins } });
-            return { generation, head_sha256: admitted.head_sha256, head, admission_record_sha256: admitted.admission_record_sha256, capture_receipt: admitted.capture_receipt, form: "wal", conflicts_retried: conflicts, refs: base.refs };
+            return { generation, head_sha256: admitted.head_sha256, head, admission_record_sha256: admitted.admission_record_sha256, capture_receipt: admitted.capture_receipt, form: "wal", conflicts_retried: conflicts, refs: base.refs, checkpoint_staleness: this.checkpointStalenessNow(generation) };
         }
     }
     /**
@@ -3160,7 +3329,7 @@ export class GitvaultVault {
         const admitted = await this.admit(head);
         if (admitted.outcome === "conflict")
             fail("HEAD_CAS_CONFLICT", "a different head was admitted while the repair was being prepared", "publishing repair head", { winner: admitted.winner }, [{ action: "verify the attached winner from storage, rebase, retry" }]);
-        return { generation: repairGen, head_sha256: admitted.head_sha256, head, admission_record_sha256: admitted.admission_record_sha256, capture_receipt: null, form: "checkpoint", conflicts_retried: 0, refs: repairedRefs };
+        return { generation: repairGen, head_sha256: admitted.head_sha256, head, admission_record_sha256: admitted.admission_record_sha256, capture_receipt: null, form: "checkpoint", conflicts_retried: 0, refs: repairedRefs, checkpoint_staleness: this.checkpointStalenessNow(repairGen) };
     }
     /** Heads `base..newest` (already chain-verified by `verifyToNewest`) re-read + hash-checked from storage. */
     async chainFrom(baseGeneration, newest) {
@@ -3230,6 +3399,50 @@ export class GitvaultVault {
         const heads = [];
         let cur = newest.head;
         let incremental = marker !== null;
+        // gitvault-clone-scaling (bench P2): the backward walk's head PATHS are
+        // all derivable up front (generation N−1, N−2, …) — only each head's
+        // expected hash arrives chain-sequentially. Same fetch-concurrent /
+        // verify-ordered split as the forward walk: page-sized windows of
+        // predecessor head bytes batch into the transient `walkPrefetch` map
+        // (use-time sha-checked by `readCachedHeadBytes`; a miss, mismatch, or
+        // failed batch falls back to that read's own single fetch), refilled as
+        // the walk descends past the window floor. The floor ESTIMATE — the
+        // marker on the incremental path, else locally learned checkpoint
+        // coverage, else genesis — only bounds over-fetch; the walk's own stop
+        // conditions are unchanged, and a wrong estimate costs at most one
+        // window of concurrent GETs, never correctness.
+        let prefetchFloor = null;
+        const prefetchBackwardWindow = async (hi) => {
+            let lo = 1n;
+            if (incremental && marker)
+                lo = generationToBigInt(marker.generation) + 1n;
+            else {
+                const known = this.repoFile().checkpoint_covers_through;
+                if (known && /^[0-9a-f]{16}$/.test(known)) {
+                    const k = generationToBigInt(known);
+                    if (k > lo)
+                        lo = k;
+                }
+            }
+            const capLo = hi - BigInt(GITVAULT_MAX_HEADS_PER_LISTING_PAGE) + 1n;
+            if (capLo > lo)
+                lo = capLo;
+            if (lo < 1n)
+                lo = 1n;
+            prefetchFloor = lo;
+            if (hi - lo < 1n)
+                return; // 0 or 1 path — a single read costs the same
+            const gens = [];
+            for (let g = hi; g >= lo; g--)
+                gens.push(bigIntToGeneration(g));
+            try {
+                this.walkPrefetch = await this.prefetchHeadsConcurrent(gens.map((g) => gitvaultPaths.head(g)));
+            }
+            catch {
+                // Fidelity: an outright prefetch failure leaves the map alone — the
+                // walk's own per-head reads take over with their own envelopes.
+            }
+        };
         while (cur) {
             // `cur` disqualifies incremental (checked BEFORE including it): abort
             // and restart as the wholesale walk. Re-visiting heads already fetched
@@ -3248,11 +3461,14 @@ export class GitvaultVault {
             // Before fetching `cur`'s predecessor, check whether the marker
             // already names it — a pure LOCAL comparison against bytes this
             // client already applied last time, no network round trip.
-            const prevGen = bigIntToGeneration(generationToBigInt(cur.generation) - 1n);
+            const prevBig = generationToBigInt(cur.generation) - 1n;
+            const prevGen = bigIntToGeneration(prevBig);
             if (incremental && cur.prev_sha256 === marker.head_sha256 && prevGen === marker.generation) {
                 cur = null; // the predecessor is the marker's own head — already applied; `heads` already holds everything above it
                 break;
             }
+            if (prefetchFloor === null || prevBig < prefetchFloor)
+                await prefetchBackwardWindow(prevBig);
             const bytes = await this.readCachedHeadBytes(prevGen, cur.prev_sha256);
             if (!bytes || sha256Hex(bytes) !== cur.prev_sha256)
                 fail("CHAIN_BROKEN", `head ${prevGen} does not match the chain during restore`, "restoring gitvault objects");