specrails-desktop 2.57.0 → 2.58.0

package/server/dist/modules/loops/runtime/loop-runs-store.js

@@ -1,5 +1,20 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.saveDefinitionRun = saveDefinitionRun;
+exports.readDefinitionRun = readDefinitionRun;
+exports.recordDefinitionCheckpoint = recordDefinitionCheckpoint;
+exports.markDefinitionRestart = markDefinitionRestart;
+exports.definitionRepositoryMounts = definitionRepositoryMounts;
+exports.claimDefinitionExecution = claimDefinitionExecution;
+exports.readDefinitionSuccessor = readDefinitionSuccessor;
+exports.readDefinitionForkTarget = readDefinitionForkTarget;
+exports.releaseDefinitionExecution = releaseDefinitionExecution;
+exports.readDefinitionExecutionClaim = readDefinitionExecutionClaim;
+exports.readDefinitionLineage = readDefinitionLineage;
+exports.linkDefinitionFork = linkDefinitionFork;
 exports.createLoopRun = createLoopRun;
 exports.finishLoopRunAndJob = finishLoopRunAndJob;
 exports.listPendingLoopTerminalRecoveries = listPendingLoopTerminalRecoveries;
@@ -24,6 +39,218 @@ exports.listActiveLoopRuns = listActiveLoopRuns;
 exports.listLoopRuns = listLoopRuns;
 exports.countRunningForLoop = countRunningForLoop;
 exports.reconcileOrphanLoopRuns = reconcileOrphanLoopRuns;
+const agent_runtime_retention_records_1 = require("../../agent-runtime/runtime/agent-runtime-retention-records");
+/**
+ * Loop runs store — per-project record (jobs.sqlite `loop_runs`) of each
+ * executed loop. The manager creates a row at launch, updates counters as it
+ * iterates, and finalises it at the end. Powers loop analytics + the
+ * active derived state (a loop is active iff a row here has status='running'
+ * or status='paused').
+ */
+const path_1 = __importDefault(require("path"));
+const node_fs_1 = require("node:fs");
+function frozenJson(value) {
+    const serialized = JSON.stringify(value, (_key, item) => {
+        if (typeof item === 'function' || typeof item === 'symbol' || typeof item === 'bigint' || typeof item === 'number' && !Number.isFinite(item))
+            throw new Error('Durable workflow inputs must contain only JSON data');
+        return item;
+    });
+    if (!serialized || Buffer.byteLength(serialized) > 4 * 1024 * 1024)
+        throw new Error('Durable workflow inputs exceed 4 MiB');
+    const sort = (input) => Array.isArray(input) ? input.map(sort) : input && typeof input === 'object'
+        ? Object.fromEntries(Object.entries(input).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, item]) => [key, sort(item)])) : input;
+    return JSON.stringify(sort(JSON.parse(serialized)));
+}
+/** Freeze launch data before spawn; preparation may add fields, never replace them. */
+function saveDefinitionRun(db, runId, input) {
+    return db.transaction(() => {
+        const row = getLoopRun(db, runId);
+        if (!row || row.engine_version !== null && row.engine_version !== undefined && row.engine_version !== 2)
+            throw new Error('Definition run is missing or belongs to another engine');
+        const { request, ...fields } = input;
+        const requestJson = request ? frozenJson({ ...request, runId }) : row.run_request_json;
+        if (!requestJson || row.run_request_json && row.run_request_json !== requestJson)
+            throw new Error('Frozen loop request cannot change');
+        const normalized = JSON.parse(requestJson);
+        if (normalized.projectId !== row.project_id || normalized.loopId !== row.loop_id)
+            throw new Error('Frozen loop request belongs to another project or loop');
+        const prior = JSON.parse(row.runtime_metadata_json ?? '{"source":"definition"}');
+        for (const [key, value] of Object.entries(fields)) {
+            if (value !== undefined && key in prior && frozenJson(prior[key]) !== frozenJson(value))
+                throw new Error('Frozen Core metadata cannot change: ' + key);
+        }
+        const metadata = { ...prior, ...JSON.parse(frozenJson(fields)) };
+        if (metadata.source !== 'definition' || metadata.definitionHash !== undefined && !/^[a-f0-9]{64}$/.test(metadata.definitionHash))
+            throw new Error('Invalid Core definition identity');
+        db.prepare('UPDATE loop_runs SET engine_version=2,run_request_json=?,runtime_metadata_json=? WHERE id=?').run(requestJson, frozenJson(metadata), runId);
+        return { row: getLoopRun(db, runId), request: normalized, metadata };
+    })();
+}
+function readDefinitionRun(db, runId) {
+    const row = getLoopRun(db, runId);
+    if (!row || row.engine_version !== 2 || !row.run_request_json)
+        return undefined;
+    const request = JSON.parse(row.run_request_json), metadata = JSON.parse(row.runtime_metadata_json ?? '{"source":"definition"}');
+    if (request.runId !== runId || request.projectId !== row.project_id || request.loopId !== row.loop_id || metadata.source !== 'definition')
+        throw new Error('Frozen workflow ownership is inconsistent');
+    return { row, request, metadata };
+}
+/** The projection cursor is independent of the aggregate accounting totals. */
+function recordDefinitionCheckpoint(db, runId, checkpoint) {
+    for (const value of [checkpoint.revision, checkpoint.eventCursor])
+        if (value !== undefined && (!Number.isSafeInteger(value) || value < 0))
+            throw new Error('Core checkpoint cursors must be nonnegative integers');
+    const serialized = frozenJson(checkpoint);
+    db.transaction(() => {
+        const row = readDefinitionRun(db, runId)?.row;
+        if (!row)
+            throw new Error('Frozen definition run is missing');
+        if (checkpoint.revision !== undefined && checkpoint.revision < (row.core_revision ?? 0) || checkpoint.eventCursor !== undefined && checkpoint.eventCursor < (row.core_event_cursor ?? 0))
+            return;
+        db.prepare('UPDATE loop_runs SET core_revision=?,core_event_cursor=?,runtime_status_json=? WHERE id=?')
+            .run(checkpoint.revision ?? row.core_revision ?? null, checkpoint.eventCursor ?? row.core_event_cursor ?? null, serialized, runId);
+    })();
+}
+/** Startup preserves the job and launch ownership; no terminal callback is queued. */
+function markDefinitionRestart(db, runId, checkpoint = {}) {
+    db.transaction(() => {
+        if (!readDefinitionRun(db, runId))
+            throw new Error('Frozen definition run is missing');
+        recordDefinitionCheckpoint(db, runId, checkpoint);
+        db.prepare("UPDATE loop_runs SET status='paused',final_outcome=NULL,finished_at=NULL,restart_reason='restart' WHERE id=? AND status IN ('running','paused')").run(runId);
+    })();
+}
+function normalizeMount(mount) {
+    const absolute = path_1.default.resolve(mount);
+    const resolved = (0, node_fs_1.existsSync)(absolute) ? (0, node_fs_1.realpathSync)(absolute) : absolute;
+    const trimmed = resolved === path_1.default.parse(resolved).root ? resolved : resolved.replace(/[\\/]+$/, '');
+    return process.platform === 'win32' ? trimmed.toLowerCase() : trimmed;
+}
+function mountsOverlap(a, b) {
+    const contains = (parent, child) => {
+        const relative = path_1.default.relative(parent, child);
+        return relative === '' || !path_1.default.isAbsolute(relative) && relative !== '..' && !relative.startsWith('..' + path_1.default.sep);
+    };
+    return contains(a, b) || contains(b, a);
+}
+function parseMounts(raw) {
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed) || parsed.some(item => typeof item !== 'string' || !path_1.default.isAbsolute(item)))
+        throw new Error('Invalid saved definition execution mounts');
+    return parsed;
+}
+/** Direct fork ancestry of a run: itself, its parents up the fork_of chain and every descendant. */
+function definitionLineageIds(db, runId, parentRunId) {
+    const lineage = new Set();
+    const parents = db.prepare('SELECT fork_of FROM loop_runs WHERE id = ?');
+    const children = db.prepare('SELECT id FROM loop_runs WHERE fork_of = ?');
+    const queue = [runId, ...(parentRunId ? [parentRunId] : [])];
+    while (queue.length) {
+        const current = queue.shift();
+        if (lineage.has(current))
+            continue;
+        lineage.add(current);
+        const parent = parents.get(current)?.fork_of;
+        if (parent && !lineage.has(parent))
+            queue.push(parent);
+        for (const child of children.all(current))
+            if (!lineage.has(child.id))
+                queue.push(child.id);
+    }
+    return lineage;
+}
+/** The checkouts a frozen request may write to. Nested paths share a mount. */
+function definitionRepositoryMounts(request) {
+    const mounts = [
+        ...(request.executionManifest?.repositories.map(repository => repository.worktreePath) ?? []),
+        ...(request.isolation ? [request.isolation.worktreePath] : []),
+        ...(request.executionManifest ? [] : [request.repoDir ?? request.cwd]),
+    ].filter((mount) => typeof mount === 'string' && mount.length > 0);
+    return [...new Set(mounts.map(normalizeMount))].sort();
+}
+/**
+ * Synchronous, atomic admission for one Core process over a worktree. A live
+ * claim of another run that overlaps any repository mount rejects: a parent
+ * and its fork share the checkout, and two owners of the same run are never
+ * allowed. Paused rows are not claims; only a resident process holds one, and
+ * settlement, cancellation and restart reconciliation release it.
+ */
+function claimDefinitionExecution(db, runId, claim) {
+    if (!runId || !claim.owner)
+        throw new Error('Execution claims require a run and an owner');
+    const mounts = [...new Set(claim.repositoryMounts.map(normalizeMount))].sort();
+    return db.transaction(() => {
+        const collecting = claim.owner.startsWith('retention:') || claim.owner.startsWith('retention-recovery:');
+        if ((0, agent_runtime_retention_records_1.readRuntimeExpiration)(db, runId) && !claim.owner.startsWith('retention-recovery:'))
+            return { ok: false, reason: 'expired', conflictingRunId: runId };
+        const successor = readDefinitionSuccessor(db, runId);
+        if (successor && !(collecting && (0, agent_runtime_retention_records_1.readRuntimeExpiration)(db, successor)))
+            return { ok: false, reason: 'lineage_conflict', conflictingRunId: successor };
+        const pendingFork = readDefinitionForkTarget(db, runId);
+        if (pendingFork && !claim.owner.startsWith('fork:') && !(collecting && (0, agent_runtime_retention_records_1.readRuntimeExpiration)(db, pendingFork)))
+            return { ok: false, reason: 'lineage_conflict', conflictingRunId: pendingFork };
+        const existing = db.prepare('SELECT run_id, owner, repository_mounts_json FROM definition_execution_claims WHERE run_id = ?').get(runId);
+        if (existing)
+            return { ok: false, reason: 'active_owner', conflictingRunId: runId };
+        if (mounts.length) {
+            const lineage = definitionLineageIds(db, runId, claim.parentRunId);
+            const live = db.prepare('SELECT run_id, owner, repository_mounts_json FROM definition_execution_claims ORDER BY claimed_at, run_id').all();
+            for (const row of live) {
+                const overlap = parseMounts(row.repository_mounts_json).some(mount => mounts.some(candidate => mountsOverlap(candidate, mount)));
+                if (!overlap)
+                    continue;
+                return { ok: false, reason: lineage.has(row.run_id) ? 'lineage_conflict' : 'active_owner', conflictingRunId: row.run_id };
+            }
+        }
+        const at = new Date().toISOString();
+        db.prepare('INSERT INTO definition_execution_claims (run_id, owner, repository_mounts_json, claimed_at, heartbeat_at) VALUES (?, ?, ?, ?, ?)')
+            .run(runId, claim.owner, JSON.stringify(mounts), at, at);
+        return { ok: true, release: () => releaseDefinitionExecution(db, runId, claim.owner) };
+    })();
+}
+/** The historical run remains intact; only its child owns shared checkout effects. */
+function readDefinitionSuccessor(db, runId) {
+    const row = db.prepare('SELECT child_run_id FROM definition_fork_operations WHERE source_run_id=? AND adopted=1').get(runId);
+    return row?.child_run_id;
+}
+function readDefinitionForkTarget(db, runId) {
+    return db.prepare('SELECT child_run_id FROM definition_fork_operations WHERE source_run_id=? ORDER BY adopted DESC,created_at LIMIT 1').get(runId)?.child_run_id;
+}
+/** Idempotent: only the owner that holds the claim can release it. */
+function releaseDefinitionExecution(db, runId, owner) {
+    db.prepare('DELETE FROM definition_execution_claims WHERE run_id = ? AND owner = ?').run(runId, owner);
+}
+function readDefinitionExecutionClaim(db, runId) {
+    const row = db.prepare('SELECT * FROM definition_execution_claims WHERE run_id = ?').get(runId);
+    return row ? { runId: row.run_id, owner: row.owner, repositoryMounts: parseMounts(row.repository_mounts_json), claimedAt: row.claimed_at, heartbeatAt: row.heartbeat_at } : undefined;
+}
+function readDefinitionLineage(db, runId) {
+    const row = getLoopRun(db, runId);
+    let forkCut = null;
+    if (row?.fork_cut_json) {
+        try {
+            forkCut = JSON.parse(row.fork_cut_json);
+        }
+        catch {
+            forkCut = null;
+        }
+    }
+    const children = db.prepare('SELECT id FROM loop_runs WHERE fork_of = ? ORDER BY started_at, id').all(runId).map(child => child.id);
+    return { runId, forkOf: row?.fork_of ?? null, children, forkCut };
+}
+/** Persist fork ancestry and Core cut provenance on a NEW child row. The source row is never touched. */
+function linkDefinitionFork(db, childRunId, link) {
+    db.transaction(() => {
+        const child = readDefinitionRun(db, childRunId);
+        if (!child)
+            throw new Error('Forked definition run is missing');
+        if (childRunId === link.forkOf || !getLoopRun(db, link.forkOf))
+            throw new Error('Fork source is missing or identical to the child');
+        if (child.row.fork_of && child.row.fork_of !== link.forkOf)
+            throw new Error('Fork ancestry cannot change');
+        db.prepare('UPDATE loop_runs SET fork_of = ?, fork_cut_json = ? WHERE id = ?').run(link.forkOf, frozenJson(link.forkCut), childRunId);
+    })();
+}
 function createLoopRun(db, input) {
     db.prepare(`
     INSERT INTO loop_runs (
@@ -116,6 +343,7 @@ function listPendingLoopTerminalRecoveries(db) {
     SELECT run_id, payload, callback_completed, created_at
       FROM loop_terminal_recovery
      WHERE callback_completed = 0
+       AND NOT EXISTS (SELECT 1 FROM definition_fork_operations f WHERE f.source_run_id=loop_terminal_recovery.run_id AND f.adopted=1)
      ORDER BY created_at, run_id
   `).all();
 }
@@ -253,7 +481,7 @@ function pauseLoopRun(db, id, counters) {
     return getLoopRun(db, id);
 }
 function resumeLoopRun(db, id) {
-    db.prepare(`UPDATE loop_runs SET status = 'running' WHERE id = ? AND status = 'paused'`).run(id);
+    db.prepare(`UPDATE loop_runs SET status = 'running', restart_reason = NULL WHERE id = ? AND status = 'paused'`).run(id);
     return getLoopRun(db, id);
 }
 function getLoopRun(db, id) {
@@ -300,17 +528,19 @@ function getRunEventCounts(db, runId) {
     }
     return { steps, lines };
 }
+// Adopted parents stay in history but their child owns execution and edit guards.
+const ownsLoopExecution = 'NOT EXISTS (SELECT 1 FROM definition_fork_operations AS fork WHERE fork.source_run_id = loop_runs.id AND fork.adopted = 1)';
 /** All currently-running loop runs for a project, straight from the DB (NOT the
  *  in-memory rail map, which is cleared on every server restart). Authoritative
  *  source for seeding the dashboard's live rail metrics after a refresh. */
 function listRunningLoopRuns(db, projectId) {
     return db
-        .prepare("SELECT * FROM loop_runs WHERE project_id = ? AND status = 'running' ORDER BY started_at ASC")
+        .prepare(`SELECT * FROM loop_runs WHERE project_id = ? AND status = 'running' AND ${ownsLoopExecution} ORDER BY started_at ASC`)
         .all(projectId);
 }
 function listActiveLoopRuns(db, projectId) {
     return db
-        .prepare("SELECT * FROM loop_runs WHERE project_id = ? AND status IN ('running','paused') ORDER BY started_at ASC")
+        .prepare(`SELECT * FROM loop_runs WHERE project_id = ? AND status IN ('running','paused') AND ${ownsLoopExecution} ORDER BY started_at ASC`)
         .all(projectId);
 }
 function listLoopRuns(db, projectId, limit = 100) {
@@ -323,7 +553,7 @@ function listLoopRuns(db, projectId, limit = 100) {
  *  Running?" guard that blocks edit/delete of a loop while it executes. */
 function countRunningForLoop(db, loopId) {
     const row = db
-        .prepare(`SELECT COUNT(*) AS n FROM loop_runs WHERE loop_id = ? AND status IN ('running','paused')`)
+        .prepare(`SELECT COUNT(*) AS n FROM loop_runs WHERE loop_id = ? AND status IN ('running','paused') AND ${ownsLoopExecution}`)
         .get(loopId);
     return row.n;
 }
@@ -334,12 +564,16 @@ function countRunningForLoop(db, loopId) {
  *  not sweep those rows: LoopRunManager is the sole invocation authority, so
  *  treating the accumulated step total as a second surface='job' invocation
  *  would double-count spend. A paused row is active too: its resumable context
- *  lived in the old process. Returns the number of loop rows reconciled. */
+ *  lived in the old process. Execution claims of every active row are released
+ *  in the same transaction. Returns the number of loop rows reconciled. */
 function reconcileOrphanLoopRuns(db, finishedAt, 
 /** Compatibility only for pre-migration rows whose exact launch ticket set
  * was never persisted. New rows always use ticket_ids_json. */
-legacyTicketIdsByRun) {
+legacyTicketIdsByRun, definitionStates) {
     const reconcile = db.transaction(() => {
+        // Admission can crash before allocating a run row, or after its terminal
+        // commit but before releasing the reservation. Neither has a live executor.
+        db.prepare("DELETE FROM definition_execution_claims WHERE run_id NOT IN (SELECT id FROM loop_runs WHERE status IN ('running','paused'))").run();
         const active = db.prepare(`
       SELECT * FROM loop_runs WHERE status IN ('running', 'paused')
     `).all();
@@ -366,7 +600,37 @@ legacyTicketIdsByRun) {
       VALUES (?, ?, 0)
       ON CONFLICT(run_id) DO NOTHING
     `);
+        // No process survived the restart: every execution claim of an active row is
+        // stale, whether the row is retained as paused or settled as failed.
+        const releaseClaim = db.prepare('DELETE FROM definition_execution_claims WHERE run_id = ?');
+        let reconciled = 0;
         for (const run of active) {
+            // Adoption already transferred execution/delivery ownership to the child.
+            // The parent's paused row is immutable history, not another orphan launch.
+            if (readDefinitionSuccessor(db, run.id)) {
+                releaseClaim.run(run.id);
+                continue;
+            }
+            reconciled += 1;
+            const probe = definitionStates?.get(run.id);
+            const lease = probe?.lease;
+            if (!lease?.active && probe?.coreStatus !== 'unavailable')
+                releaseClaim.run(run.id);
+            else if (!readDefinitionExecutionClaim(db, run.id)) {
+                const request = readDefinitionRun(db, run.id)?.request;
+                if (request) {
+                    const claim = claimDefinitionExecution(db, run.id, { owner: `retained:${run.id}`, repositoryMounts: definitionRepositoryMounts(request) });
+                    if (!claim.ok)
+                        throw new Error(`Cannot reserve retained Core worktree: ${claim.conflictingRunId}`);
+                }
+            }
+            if (run.engine_version === 2 && run.run_request_json) {
+                const state = definitionStates?.get(run.id);
+                if (state !== null && (!state?.status || ['running', 'paused', 'interrupted'].includes(state.status))) {
+                    markDefinitionRestart(db, run.id, state ?? { status: 'unavailable', reason: 'Original Core status could not be read' });
+                    continue;
+                }
+            }
             updateJob.run(finishedAt, run.id);
             updateRun.run(finishedAt, run.id);
             const persistedIds = parseTicketIdsJson(run.ticket_ids_json);
@@ -386,7 +650,7 @@ legacyTicketIdsByRun) {
             };
             insertIntent.run(run.id, JSON.stringify(payload));
         }
-        return active.length;
+        return reconciled;
     });
     return reconcile();
 }