wicked-crew 0.7.11 → 0.7.13

package/dist/api/routes.js

@@ -34,8 +34,8 @@ import { AcpFoldCache, EngineVersionCache, eventsDirOf, installedPackageVersion,
 import { RetryIndex } from './retry-index.js';
 import { GroupIndex } from './group-index.js';
 import { GuidanceIndex } from './guidance-index.js';
-import { DeliveryIndex, deliveryStateWithVacuity, gitRunBranchIsEmpty, gitWorktreeIsClean, prUrlFrom, } from './delivery-index.js';
-import { runDeliverScript } from './post-hoc-deliver.js';
+import { DeliveryIndex, deliveryStateWithVacuity, gitRunBranchIsEmpty, gitWorktreeIsClean, isDeliverConflictStranded, prUrlFrom, } from './delivery-index.js';
+import { gitReprovisionWorktree, runDeliverScript, } from './post-hoc-deliver.js';
 import { LOCAL_ACTOR } from './auth.js';
 // Re-exported so existing `import { API_PREFIX } from './routes.js'` callers keep working; the
 // value lives in the leaf module api-prefix.ts to keep unit-output.ts out of this file's cycle.
@@ -263,6 +263,9 @@ runtime = {}) {
             }),
     };
     const deliverExec = runtime.deliverExec ?? runDeliverScript;
+    const reprovisionWorktree = runtime.reprovisionWorktree ?? gitReprovisionWorktree;
+    /** Repo root for a repo ref, from the registry — shared by the reprovision path below. */
+    const repoRootOf = async (repoRef) => (await adapter.listRepos()).find((r) => r.id === repoRef)?.root_path;
     // The run-DTO joins (DES-UX-001 §8.2/§8.3, DES-UX-002 §7.2): `project_id` from the membership
     // record — `null` = genuinely unfiled, so the field is ALWAYS present on served runs —
     // `retry_of` from the lineage index and `guidance` from the guidance index, each set only
@@ -275,7 +278,42 @@ runtime = {}) {
     // "done" with nothing produced must be LOUD on the wire, never silently green.
     // Applied at DTO assembly on exactly the two endpoints that serve the run DTO
     // (GET /runs + GET /runs/:id); the internal sessionsDetail() consumers are untouched.
+    // crew#418 A: a `failed` run whose ONLY failure was the deliver phase's LIFT collision (a
+    // rebase conflict the changelog union merge could not clear, or a non-fast-forward push) is
+    // reinterpreted on the wire as `completed` + `delivery: 'stranded'` — recoverable, not a hard
+    // failure. The engine's durable record stays `failed` (audit trail); this is a wire derivation,
+    // exactly like `delivery` itself, applied at the SAME three terminal-status decision points so
+    // the run reads consistently across GET /runs(/:id), resume, and POST /runs/:id/deliver. The
+    // deliver unit stays `rejected` with its marker-bearing `denial_reason`, so WHY it stranded is
+    // still on the wire.
+    //
+    // Returns whether this run IS such a strand — the caller uses it for a BRANCH-based `'stranded'`
+    // read: the engine reaps a failed-deliver run's worktree once the deliver phase has committed
+    // the work (a clean tree), so the work lives on the `wicked/<id>` branch, not the worktree — the
+    // worktree-stat derivation would read `'none'` (and flicker as the async reap lands). The
+    // lift-conflict marker is proof the branch carries the work (the deliver script verified it was
+    // ahead of the default branch before it ever attempted the rebase), so the read is a STABLE
+    // `'stranded'` regardless of whether the worktree survived. Idempotent: a second call sees the
+    // already-flipped `completed` and returns false, so the flip never re-fires.
+    const normalizeStranded = (view) => {
+        const conflictStrand = isDeliverConflictStranded(view); // true only while status === 'failed'
+        if (conflictStrand)
+            view.session.status = 'completed';
+        return conflictStrand;
+    };
+    /** The run's delivery state, honest by construction (crew#393/#311/#418): a recorded PR wins
+     *  (`'delivered'`); else a lift-conflict strand reads `'stranded'` from the branch; else the
+     *  worktree-stat derivation (stranded / vacuous / none). */
+    const resolveDelivery = async (view, conflictStrand) => {
+        const url = deliveryIndex.urlFor(view.session.id);
+        if (url !== undefined)
+            return { delivery: 'delivered', deliverUrl: url };
+        if (conflictStrand)
+            return { delivery: 'stranded' };
+        return deliveryStateWithVacuity(view.session, undefined, vacuityProbes);
+    };
     const decorateRun = async (view) => {
+        const conflictStrand = normalizeStranded(view);
         view.session.project_id = projects.index.projectOf(view.session.id) ?? null;
         const retryOf = retryIndex.retryOfFor(view.session.id);
         if (retryOf !== undefined)
@@ -292,7 +330,7 @@ runtime = {}) {
         const guidance = guidanceIndex.guidanceFor(view.session.id);
         if (guidance !== undefined)
             view.session.guidance = guidance;
-        const state = await deliveryStateWithVacuity(view.session, deliveryIndex.urlFor(view.session.id), vacuityProbes);
+        const state = await resolveDelivery(view, conflictStrand);
         view.session.delivery = state.delivery;
         if (state.deliverUrl !== undefined)
             view.session.deliverUrl = state.deliverUrl;
@@ -906,6 +944,30 @@ runtime = {}) {
     // existing worktree. Idempotent: a delivered run answers its recorded URL, never a second PR.
     // Failure is a loud 4xx/5xx carrying the script's own words — never a silent 200.
     const deliverInFlight = new Set();
+    // Per-REPO serialization for the worktree-admin-sensitive region (reprovision → deliver →
+    // cleanup). deliverInFlight is per-RUN and stops the same run double-delivering, but two
+    // DIFFERENT stranded runs in the same repo would both run `git worktree prune/add/remove`
+    // against one `.git/worktrees` admin state and race. Chaining a promise per repo ref serializes
+    // them; different repos still run in parallel. The tail entry is pruned so the map cannot grow.
+    const repoDeliverChain = new Map();
+    const withRepoDeliverLock = async (repoRef, fn) => {
+        const prior = repoDeliverChain.get(repoRef) ?? Promise.resolve();
+        let release;
+        const mine = new Promise((r) => {
+            release = r;
+        });
+        repoDeliverChain.set(repoRef, mine); // WE are now the tail
+        await prior.catch(() => undefined); // wait our turn; a prior failure never blocks ours
+        try {
+            return await fn();
+        }
+        finally {
+            release(); // unblock whoever chained after us
+            // If nothing chained after us, drop the entry so idle repos leave no trace.
+            if (repoDeliverChain.get(repoRef) === mine)
+                repoDeliverChain.delete(repoRef);
+        }
+    };
     /** The last words of a failed deliver script — enough to name the refusal, bounded so a full
      *  git transcript never becomes an error body. */
     const deliverErrorTail = (output) => {
@@ -934,6 +996,10 @@ runtime = {}) {
         const run = views.find((v) => v.session.id === id);
         if (!run)
             return reply.code(404).send({ error: 'Run not found' });
+        // crew#418 A: a run stranded by a deliver-phase lift collision reads `failed` from the
+        // engine but IS liftable post-hoc — normalize its status to `completed` so this route
+        // accepts it, exactly as the run wire and the resume route see it.
+        const conflictStrand = normalizeStranded(run);
         const s = run.session;
         if (s.status !== 'completed') {
             return reply.code(409).send({
@@ -945,22 +1011,55 @@ runtime = {}) {
                 error: `run ${id} is not repo-scoped — there is no worktree to lift into a PR`,
             });
         }
-        if (!worktreeExists(s.workdir)) {
-            return reply.code(409).send({
-                error: `run ${id}'s worktree is gone (${s.workdir}) — nothing left to deliver`,
-            });
-        }
+        // Capture the ref + workdir while the null-check narrowing is fresh — an intervening await
+        // below invalidates property narrowing, so re-reading `s.repo_ref`/`s.workdir` later widens
+        // back to `| null`.
+        const repoRef = s.repo_ref;
+        const initialWorkdir = s.workdir;
         // One delivery per run at a time: the script pushes and opens a PR, so two concurrent
-        // spawns could race gh into two PRs — the exact double-open idempotency forbids.
+        // spawns could race gh into two PRs — the exact double-open idempotency forbids. The guard
+        // wraps the reprovision too, so a reaped run's `wicked/<id>` branch is only ever checked out
+        // by one throwaway worktree at a time.
         if (deliverInFlight.has(id)) {
             return reply.code(409).send({
                 error: `a delivery for run ${id} is already in progress — wait for it to finish`,
             });
         }
         deliverInFlight.add(id);
+        // The worktree the script runs in. Usually the run's own; but the engine REAPS a
+        // failed-deliver run's worktree once its work is committed (crew#418) — the work then lives
+        // on the `wicked/<id>` branch. For such a strand, stand a throwaway worktree back up from
+        // that branch and lift THAT; the branch (the record) is untouched. `cleanupWorktree` tears
+        // the throwaway down after the lift.
         let result;
+        let worktreeGone = false;
         try {
-            result = await deliverExec(s.workdir, s.problem);
+            // The worktree admin (reprovision), the deliver spawn, AND the throwaway teardown all run
+            // under the per-repo lock, so a second stranded run in this repo cannot touch
+            // `.git/worktrees` until this one has stood its worktree up, lifted, and torn it back
+            // down. `cw` is a LOCAL torn down in the lock body's own finally — nothing captured-mutated
+            // crosses the closure boundary.
+            await withRepoDeliverLock(repoRef, async () => {
+                let workdir = initialWorkdir;
+                let cw = null;
+                try {
+                    if (!worktreeExists(workdir)) {
+                        const root = conflictStrand ? await repoRootOf(repoRef) : undefined;
+                        const reprov = root !== undefined ? await reprovisionWorktree(root, id) : null;
+                        if (reprov === null) {
+                            worktreeGone = true;
+                            return;
+                        }
+                        workdir = reprov.workdir;
+                        cw = reprov.cleanup;
+                    }
+                    result = await deliverExec(workdir, s.problem ?? undefined);
+                }
+                finally {
+                    if (cw !== null)
+                        await cw(); // tear the throwaway down whether the lift succeeded or threw
+                }
+            });
         }
         catch (err) {
             return reply.code(500).send({ error: `deliver script could not run: ${message(err)}` });
@@ -968,6 +1067,15 @@ runtime = {}) {
         finally {
             deliverInFlight.delete(id);
         }
+        if (worktreeGone) {
+            return reply.code(409).send({
+                error: `run ${id}'s worktree is gone (${s.workdir}) — nothing left to deliver`,
+            });
+        }
+        if (result === undefined) {
+            // Unreachable: the lock body assigns `result` on every path that is not `worktreeGone`.
+            return reply.code(500).send({ error: `deliver for run ${id} produced no result` });
+        }
         if (result.spawnFailure === true) {
             // The script never reached its own verdict (spawn failure, timeout kill) — an infra
             // fault, not a refusal: 500 so the caller knows a retry is reasonable.
@@ -1342,6 +1450,10 @@ runtime = {}) {
         const run = views.find((v) => v.session.id === id);
         if (!run)
             return reply.code(404).send({ error: 'Run not found' });
+        // crew#418 A: a run stranded by a deliver-phase lift collision reads `failed` from the engine
+        // but its work is complete and liftable — normalize to `completed` so the terminal branch
+        // below refuses resume with the DELIVER recovery (POST /runs/:id/deliver), not a re-entry.
+        const conflictStrand = normalizeStranded(run);
         // A TERMINAL run is refused LOUDLY, with the actual recovery named (crew#311 defect 2).
         // The engine's `resume_run` no-ops on completed/cancelled and answers the status token, so
         // this route used to reply 200 {"status":"cancelled"} — on the exact runs an operator was
@@ -1351,14 +1463,14 @@ runtime = {}) {
         // The 409 body carries the machine-readable pointer (`ResumeRefusal` in api-types).
         const terminal = run.session.status;
         if (terminal === 'completed' || terminal === 'cancelled') {
-            const state = await deliveryStateWithVacuity(run.session, deliveryIndex.urlFor(id), vacuityProbes);
+            const state = await resolveDelivery(run, conflictStrand);
             const recovery = terminal === 'completed' && state.delivery === 'stranded'
                 ? 'deliver'
                 : 'retry';
             const why = terminal === 'cancelled'
                 ? `run ${id} is cancelled — a terminal run cannot be resumed; relaunch the work as a new run with POST /runs {"retryOf":"${id}"}`
                 : state.delivery === 'stranded'
-                    ? `run ${id} is already completed — nothing to resume; its unlifted work is in the worktree: deliver it with POST /runs/${id}/deliver`
+                    ? `run ${id} is already completed — nothing to resume; its unlifted work is on the wicked/${id} branch: deliver it with POST /runs/${id}/deliver`
                     : state.delivery === 'vacuous'
                         ? `run ${id} is already completed, but VACUOUSLY — its units produced no work to resume or deliver; relaunch with POST /runs {"retryOf":"${id}"}`
                         : `run ${id} is already completed — nothing to resume; relaunch the work as a new run with POST /runs {"retryOf":"${id}"}`;