pi-orchestrator-config 4.3.12 → 4.3.13

package/extensions/orchestrator/async-agents.ts

@@ -27,9 +27,11 @@ const log = createLogger("async_agents");
 
 export { autoCompleteTask, autoMarkInProgress } from "./task-lifecycle.js";
 import { autoCompleteTask, autoMarkInProgress } from "./task-lifecycle.js";
+import { openAsyncStatusOverlay } from "./async-status-ui.js";
 import { setSlot } from "./status-bar.js";
 import { getSetting } from "./project-settings.js";
 import { substituteSettingsPlaceholders } from "./rule-placeholders.js";
+import { isLiveExtensionCtx } from "../shared/live-ctx.js";
 
 const SETTINGS_KEYS: Record<string, unknown> = JSON.parse(
   fs.readFileSync(
@@ -174,10 +176,20 @@ export function registerAsyncAgents(
   }
 
   let lastWidgetKey = "";
+  function getLiveAsyncCtx(): any | null {
+    const ctx = asyncState.lastCtx;
+    if (!isLiveExtensionCtx(ctx)) {
+      log.debug("async_context_stale", { hadContext: ctx !== null });
+      asyncState.lastCtx = null;
+      return null;
+    }
+    return ctx;
+  }
+
   function updateAsyncWidget() {
-    if (!asyncState.lastCtx?.hasUI) return;
     try {
-      const ctx = asyncState.lastCtx;
+      const ctx = getLiveAsyncCtx();
+      if (!ctx?.hasUI) return;
       const running = Array.from(asyncState.jobs.values()).filter(j => j.status === "running" || j.status === "queued");
       const names = running.map(j => j.name || j.agent).join(", ");
       const widgetKey = `${running.length}:${names}`;
@@ -207,10 +219,15 @@ export function registerAsyncAgents(
   function ensureAsyncPoller() {
     if (asyncState.poller) return;
     asyncState.poller = setInterval(() => {
+      const liveCtx = getLiveAsyncCtx();
+      if (!liveCtx) {
+        // ctx is stale (session ended) — stop polling
+        if (asyncState.poller) { clearInterval(asyncState.poller); asyncState.poller = null; }
+        return;
+      }
       try {
-        if (!asyncState.lastCtx?.hasUI) return;
+        if (!liveCtx.hasUI) return;
       } catch {
-        // ctx is stale (session ended) — stop polling
         if (asyncState.poller) { clearInterval(asyncState.poller); asyncState.poller = null; }
         return;
       }
@@ -401,7 +418,7 @@ export function registerAsyncAgents(
               deliverGroupResults(groupJobs);
             }
           // Delivery: retry for non-grouped jobs (zombie-ingest path may skip delivery)
-          } else if (asyncState.lastCtx && !job.fireAndForget) {
+          } else if (getLiveAsyncCtx() && !job.fireAndForget) {
             const displayName = job.name || job.agent;
             const resultStatus = job.status === "complete" ? "✅ completed" : "❌ failed";
             const duration = job.durationMs || (job.updatedAt ? job.updatedAt - job.startedAt : 0);
@@ -437,7 +454,7 @@ export function registerAsyncAgents(
               job.delivered = true;
               log.debug(`reconcile: skipping already-delivered result for ${job.id}`);
             } else {
-              log.debug("reconcile_deliver", job.id, "hasCtx", !!asyncState.lastCtx);
+              log.debug("reconcile_deliver", job.id, "hasCtx", true);
               try {
                 pi.sendMessage({
                   customType: "async-agent-result",
@@ -494,8 +511,6 @@ export function registerAsyncAgents(
     if (gid && groupDeliveryInProgress.has(gid)) return;
     if (gid) groupDeliveryInProgress.add(gid);
     try {
-    if (!asyncState.lastCtx) return;
-
     // Ingest any unprocessed result files — zombie/kill paths may trigger delivery
     // before processResultFile() has read all group members' outputs.
     // Wait up to 2s total (not per-job) for all missing result files.
@@ -603,6 +618,12 @@ export function registerAsyncAgents(
     }
 
     if (sections.length > 0) {
+      // A group may have waited for result files while its session was replaced.
+      // Keep persisted/in-memory results for reconciliation rather than sending into it.
+      if (!getLiveAsyncCtx()) {
+        log.debug("deliver_group_results_deferred", { groupId: groupJobs[0]?.groupId, jobs: groupJobs.length, deliverableJobs: deliverableJobs.length, status: groupJobs[0]?.status });
+        return;
+      }
       try {
         pi.sendMessage({
           customType: "async-agent-result",
@@ -649,7 +670,7 @@ export function registerAsyncAgents(
     try {
       const data = JSON.parse(fs.readFileSync(resultPath, "utf-8"));
       const job = asyncState.jobs.get(data.id);
-      log.debug(`processResultFile: ${path.basename(resultPath)} job=${!!job} delivered=${job?.delivered} hasCtx=${!!asyncState.lastCtx} fireAndForget=${job?.fireAndForget} groupId=${job?.groupId}`);
+      log.debug(`processResultFile: ${path.basename(resultPath)} job=${!!job} delivered=${job?.delivered} hasCtx=${!!getLiveAsyncCtx()} fireAndForget=${job?.fireAndForget} groupId=${job?.groupId}`);
       if (!job) {
         // Orphan result file — job not in current session. Clean up to prevent re-delivery on reload.
         try { fs.unlinkSync(resultPath); } catch (e: any) { log.debug(`orphan cleanup failed: ${resultPath}: ${e?.message}`); }
@@ -694,11 +715,11 @@ export function registerAsyncAgents(
         // Use model context window from the agent's effective model.
         // If agent uses a custom model, we pass its context window via the config.
         // Fall back to orchestrator's model context window.
-        const contextWindow = data.contextWindow || asyncState.lastCtx?.model?.contextWindow || 0;
+        const contextWindow = data.contextWindow || getLiveAsyncCtx()?.model?.contextWindow || 0;
         if (data.lastUsage?.totalTokens && data.lastUsage.totalTokens > 0 && contextWindow > 0) {
           const pct = (data.lastUsage.totalTokens / contextWindow) * 100;
           if (pct > 80) {
-            const sessId = `async-${job.agent}-${djb2Hash(job.agent + ':' + (jobCwd(job)) + ':' + (asyncState.lastCtx?.sessionManager?.getSessionId?.() || '')).toString(36)}`;
+            const sessId = `async-${job.agent}-${djb2Hash(job.agent + ':' + (jobCwd(job)) + ':' + (getLiveAsyncCtx()?.sessionManager?.getSessionId?.() || '')).toString(36)}`;
             try {
               // Find and delete the session file to force fresh on next cycle
               const sessDir = path.join(os.homedir(), '.pi', 'agent', 'sessions');
@@ -776,7 +797,7 @@ export function registerAsyncAgents(
 
       // Non-grouped job: deliver immediately (existing behavior)
       let sendSucceeded = false;
-      if (asyncState.lastCtx && !job.fireAndForget) {
+      if (getLiveAsyncCtx() && !job.fireAndForget) {
         const resultStatus = data.success ? "✅ completed" : "❌ failed";
         let autoCompleteError = "";
         // Auto-complete linked task directly in the store file (no AI involvement)
@@ -852,12 +873,13 @@ export function registerAsyncAgents(
     watchedDir = ASYNC_RESULTS_DIR;
     try {
       fs.mkdirSync(ASYNC_RESULTS_DIR, { recursive: true, mode: 0o700 });
-      asyncState.watcher = fs.watch(ASYNC_RESULTS_DIR, (ev, file) => {
+      const resultsDir = ASYNC_RESULTS_DIR;
+      asyncState.watcher = fs.watch(resultsDir, (ev, file) => {
         if (ev !== "rename" || !file) return;
         const fileName = file.toString();
         log.debug("watcher_event", ev, fileName);
         if (!fileName.endsWith(".json")) return;
-        const resultPath = path.join(ASYNC_RESULTS_DIR, fileName);
+        const resultPath = path.join(resultsDir, fileName);
         setTimeout(() => processResultFile(resultPath), 100);
       });
       if (asyncState.watcher.unref) asyncState.watcher.unref();
@@ -900,8 +922,8 @@ export function registerAsyncAgents(
       parentStartTime,
       taskId: options?.taskId || null,
       cwd,
-      projectCwd: asyncState.lastCtx?.sessionManager?.getCwd?.() || null,
-      sessionId: asyncState.lastCtx?.sessionManager?.getSessionId?.() || null,
+      projectCwd: getLiveAsyncCtx()?.sessionManager?.getCwd?.() || null,
+      sessionId: getLiveAsyncCtx()?.sessionManager?.getSessionId?.() || null,
     }), { mode: 0o600 });
 
     // Track reviewer spawn BEFORE building args — addReviewerPending sets status to
@@ -923,7 +945,7 @@ export function registerAsyncAgents(
     // Deterministic session ID for provider cache affinity + optional session reuse.
     // When persistSession is true, omits --no-session so the session persists to disk.
     // Same agent + cwd gets the same session ID across calls.
-    const parentSessionId = asyncState.lastCtx?.sessionManager?.getSessionId?.() || '';
+    const parentSessionId = getLiveAsyncCtx()?.sessionManager?.getSessionId?.() || '';
     const sessionIdSource = persistSession
       ? agentName + ':' + cwd + ':' + parentSessionId
       : agentName + ':' + task.slice(0, 100);
@@ -956,7 +978,7 @@ export function registerAsyncAgents(
       task,
       cwd,
       model: effectiveModel,
-      contextWindow: asyncState.lastCtx?.model?.contextWindow || 0,
+      contextWindow: getLiveAsyncCtx()?.model?.contextWindow || 0,
       resultPath,
       reviewerOutputPath: agentName.startsWith("code-reviewer-")
         ? reviewerOutputArchivePath(PROJECT_TMP_DIR, id)
@@ -1094,8 +1116,8 @@ export function registerAsyncAgents(
       groupId: options?.groupId,
       taskId: options?.taskId,
       cwd,
-      projectCwd: asyncState.lastCtx?.sessionManager?.getCwd?.(),
-      sessionId: asyncState.lastCtx?.sessionManager?.getSessionId?.(),
+      projectCwd: getLiveAsyncCtx()?.sessionManager?.getCwd?.(),
+      sessionId: getLiveAsyncCtx()?.sessionManager?.getSessionId?.(),
       model: effectiveModel,
     };
     asyncState.jobs.set(id, job);
@@ -1185,7 +1207,7 @@ export function registerAsyncAgents(
 
     // Auto-mark linked task as in_progress AFTER successful spawn
     if (options?.taskId && options.taskId !== "-1") {
-      autoMarkInProgress(options.taskId, asyncState.lastCtx?.sessionManager?.getCwd?.() || cwd, asyncState.lastCtx?.sessionManager?.getSessionId?.() || undefined)
+      autoMarkInProgress(options.taskId, getLiveAsyncCtx()?.sessionManager?.getCwd?.() || cwd, getLiveAsyncCtx()?.sessionManager?.getSessionId?.() || undefined)
         .catch(() => {});  // best-effort, don't block spawn
     }
 
@@ -1194,6 +1216,10 @@ export function registerAsyncAgents(
 
   // Start result watcher on session start
   pi.on("session_start", (_event, ctx) => {
+    if (!isLiveExtensionCtx(ctx)) {
+      log.warn("session_start received a stale ctx; preserving async results for a live session");
+      return;
+    }
     asyncState.lastCtx = ctx;
     deliveredResultIds.clear();
 
@@ -1365,6 +1391,7 @@ export function registerAsyncAgents(
   // Clean up on shutdown
   pi.on("session_shutdown", () => {
     log.info(`shutdown: jobs=${asyncState.jobs.size}`);
+    asyncState.lastCtx = null;
     if (asyncState.poller) { clearInterval(asyncState.poller); asyncState.poller = null; }
     if (asyncState.watcher) { asyncState.watcher.close(); asyncState.watcher = null; }
   });
@@ -1373,7 +1400,6 @@ export function registerAsyncAgents(
   async function handleAsyncStatus(ctx: any): Promise<void> {
     if (!ctx.hasUI) return;
     log.debug("opening_async_status_overlay", { jobs: asyncState.jobs.size });
-    const { openAsyncStatusOverlay } = await import("./async-status-ui.js");
     await openAsyncStatusOverlay(ctx, {
       listJobs: () => Array.from(asyncState.jobs.values()),
       killJob: (id) => {
@@ -1461,7 +1487,7 @@ export function registerAsyncAgents(
         if (pending.length === 0) {
           deliverGroupResults(groupJobs);
         }
-      } else if (asyncState.lastCtx && !job.fireAndForget) {
+      } else if (getLiveAsyncCtx() && !job.fireAndForget) {
         // Non-grouped killed job — deliver immediately so AI knows it was killed
         const displayName = job.name || job.agent;
         const duration = job.durationMs || (Date.now() - job.startedAt);
@@ -1582,7 +1608,7 @@ export function registerAsyncAgents(
   // Spawn — create an async agent from pitasks' TaskExecute
   handleRpc<{ requestId: string; type: string; prompt: string; options?: any }>(
     "subagents:rpc:spawn", async ({ type, prompt, options }) => {
-      const ctx = asyncState.lastCtx;
+      const ctx = getLiveAsyncCtx();
       if (!ctx) throw new Error("No active session");
       if (!type || typeof type !== "string") throw new Error("Missing or invalid 'type' parameter");
       if (!prompt || typeof prompt !== "string") throw new Error("Missing or invalid 'prompt' parameter");

package/extensions/orchestrator/cron-status-format.ts

@@ -3,10 +3,10 @@
  */
 
 export interface CronStatusTaskView {
-  /** Unique overlay row id (e.g. "1" or "cron-123-abc.json:1"). */
+  /** Unique scope-qualified overlay row id (e.g. "session:<uuid>"). */
   id: string;
-  /** Numeric cron task id within its session file. */
-  taskId: number;
+  /** Stable UUID within its session or project scope. */
+  taskId: string | number;
   description: string;
   task: string;
   intervalMs?: number;
@@ -15,7 +15,7 @@ export interface CronStatusTaskView {
   createdAt: number;
   lastRun?: number;
   nextRun?: number;
-  /** Shown in list-all rows (e.g. "this session", "PID 42"). */
+  /** Optional scope and leader-state label shown in overlay rows. */
   sessionLabel?: string;
   isLocal?: boolean;
   cronFile?: string;
@@ -91,7 +91,7 @@ export function formatNextRunLabel(
 /** Map a CronTask-like object into an overlay row view. */
 export function toCronStatusTaskView(
   task: {
-    id: number;
+    id: string | number;
     description: string;
     task: string;
     intervalMs?: number;
@@ -110,7 +110,7 @@ export function toCronStatusTaskView(
 ): CronStatusTaskView {
   return {
     id: opts.overlayId,
-    taskId: task.id,
+    taskId: opts.overlayId,
     description: task.description,
     task: task.task,
     intervalMs: task.intervalMs,

package/extensions/orchestrator/cron-status-ui.ts

@@ -1,5 +1,5 @@
 /**
- * /cron list + list-all fullscreen overlay — list dashboard → task detail.
+ * /cron list fullscreen overlay for session and project tasks — list dashboard → task detail.
  */
 
 import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
@@ -25,7 +25,7 @@ export {
 
 export interface CronStatusUiDeps {
   listTasks: () => CronStatusTaskView[];
-  /** Remove by overlay row id. Return true if removed. */
+  /** Remove by scope-qualified task id. Return true if removed. */
   removeTask: (id: string) => boolean;
   title?: string;
   emptyMessage?: string;
@@ -33,7 +33,7 @@ export interface CronStatusUiDeps {
 }
 
 /**
- * Open fullscreen cron list. Loops list → detail until user closes list.
+ * Open fullscreen cron list for the selected scope. Loops list → detail until closed.
  */
 export async function openCronStatusOverlay(
   ctx: ExtensionCommandContext,

package/extensions/orchestrator/cron-store.ts

@@ -0,0 +1,208 @@
+/** Durable cron storage and local-filesystem leader locks. Delivery is at-least-once. */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { execFileSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { parseProcStartTime } from "./utils.js";
+import { createLogger } from "../shared/logger.js";
+
+export type CronScope = "session" | "project";
+export interface DurableCronTask {
+  id: string;
+  scope: "project";
+  cwd: string;
+  description: string;
+  task: string;
+  intervalMs?: number;
+  atHour?: number;
+  atMinute?: number;
+  createdAt: number;
+  lastRun?: number;
+  nextRun?: number;
+}
+export interface DurableCronEnvelope { version: 1; tasks: DurableCronTask[]; }
+export interface CronLockOwner { pid: number; process_start_token: string; instance_id: string; heartbeat_at: string; fd?: number; }
+
+type StartTokenDeps = { readFileSync: (path: string, encoding: BufferEncoding) => string; execFileSync: (file: string, args: readonly string[], options: { encoding: BufferEncoding; windowsHide: boolean }) => string; platform: NodeJS.Platform };
+const startTokenDeps: StartTokenDeps = { readFileSync: fs.readFileSync, execFileSync, platform: process.platform };
+const log = createLogger("cron_store");
+const STALE_LOCK_MS = 30_000;
+const leaderOwners = new Map<string, CronLockOwner>();
+const sleepCell = new Int32Array(new SharedArrayBuffer(4));
+function sleep(ms: number) { Atomics.wait(sleepCell, 0, 0, ms); }
+
+/** Return an OS-issued token which changes when a PID is reused, or null when unavailable. */
+export function resolveProcessStartToken(pid: number, deps: StartTokenDeps = startTokenDeps): string | null {
+  try {
+    const procToken = parseProcStartTime(deps.readFileSync(`/proc/${pid}/stat`, "utf8"));
+    if (procToken) { log.debug("cron_process_token", { pid, platform: deps.platform, source: "proc" }); return `proc:${procToken}`; }
+  } catch (error: any) { log.debug("cron_process_token_proc_failed", { pid, platform: deps.platform, code: error?.code }); }
+  if (deps.platform !== "win32") { log.warn("cron_process_token_unavailable", { pid, platform: deps.platform }); return null; }
+  try {
+    const output = deps.execFileSync("powershell.exe", ["-NoProfile", "-NonInteractive", "-Command", `(Get-Process -Id ${pid}).StartTime.ToFileTimeUtc()`], { encoding: "utf8", windowsHide: true }).trim();
+    const token = /^\d{17,}$/.test(output) ? `win:${output}` : null;
+    log.debug("cron_process_token_windows", { pid, valid: !!token });
+    return token;
+  } catch (error: any) { log.warn("cron_process_token_windows_failed", { pid, code: error?.code }); return null; }
+}
+export function processStartToken(pid = process.pid): string { return resolveProcessStartToken(pid) || "unknown"; }
+export function durableCronSupported(): boolean { const supported = resolveProcessStartToken(process.pid) !== null; log.debug("durable_cron_supported", { platform: process.platform, supported }); return supported; }
+function lockPath(store: string, kind: "mutation" | "leader") { return `${store}.${kind}.lock`; }
+function ownerPath(dir: string) { return path.join(dir, "owner.json"); }
+/** True only when the OS can prove this record cannot still own the lock. */
+function ownerIsProvenDead(owner: CronLockOwner): boolean {
+  if (!owner?.pid || !Number.isInteger(owner.pid) || owner.pid < 1) return false;
+  try {
+    process.kill(owner.pid, 0);
+  } catch (error: any) {
+    // EPERM means the process exists but is not signalable; all other failures
+    // are treated conservatively except the OS's definitive "no such process".
+    return error?.code === "ESRCH";
+  }
+  if (!owner.process_start_token || owner.process_start_token === "unknown") return false;
+  const token = resolveProcessStartToken(owner.pid);
+  // An unavailable identity source fails closed. A different issued token
+  // proves PID reuse, so this record belongs to a dead process.
+  return token !== null && token !== owner.process_start_token;
+}
+function readOwner(dir: string): CronLockOwner | null {
+  try {
+    const owner = JSON.parse(fs.readFileSync(ownerPath(dir), "utf8"));
+    const valid = owner && typeof owner.pid === "number" && typeof owner.process_start_token === "string" && typeof owner.instance_id === "string" && typeof owner.heartbeat_at === "string";
+    if (!valid) log.warn("cron_lock_owner_invalid", { dir, reason: "invalid_owner" });
+    return valid ? owner : null;
+  } catch (error: any) { log.debug("cron_lock_owner_unreadable", { dir, reason: error?.code || "parse_error" }); return null; }
+}
+function sameOwner(a: CronLockOwner | null, b: CronLockOwner) {
+  return !!a && a.instance_id === b.instance_id && a.pid === b.pid && a.process_start_token === b.process_start_token;
+}
+function serializedOwner(owner: CronLockOwner) {
+  const { fd: _fd, ...record } = owner;
+  return JSON.stringify(record);
+}
+function claimIsOld(claim: string) {
+  try {
+    const ageMs = Date.now() - fs.statSync(claim).mtimeMs;
+    const old = ageMs >= STALE_LOCK_MS;
+    log.debug("cron_lock_claim_age", { claim, age_ms: ageMs, old });
+    return old;
+  } catch (error: any) { log.warn("cron_lock_claim_stat_failed", { claim, reason: error?.code || "stat_error" }); return false; }
+}
+function restoreClaim(claim: string, dir: string) {
+  try { fs.renameSync(claim, dir); } catch { log.warn("cron_lock_claim_restore_failed", { claim, dir }); }
+}
+/** Read the canonical record first; detach and delete only that proven-stale record. */
+function reclaimStaleLock(dir: string, instanceId: string, kind: "leader" | "mutation"): boolean {
+  const expected = readOwner(dir);
+  if (!(expected ? ownerIsProvenDead(expected) : claimIsOld(dir))) return false;
+  const claim = `${dir}.claim-${randomUUID()}`;
+  try { fs.renameSync(dir, claim); } catch { return false; }
+  const current = readOwner(claim);
+  if (expected ? !sameOwner(current, expected) : current || !claimIsOld(claim)) { restoreClaim(claim, dir); return false; }
+  try { fs.rmSync(claim, { recursive: true, force: false }); log.info(`cron_${kind}_lock_reclaim`, { dir, instance_id: instanceId, outcome: "reclaimed" }); return true; }
+  catch (error: any) { log.warn(`cron_${kind}_lock_reclaim`, { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "remove_error" }); restoreClaim(claim, dir); return false; }
+}
+function safelyRemoveLock(dir: string, expected?: CronLockOwner): boolean {
+  if (expected && !sameOwner(readOwner(dir), expected)) { log.debug("cron_lock_remove", { dir, outcome: "not_owner" }); return false; }
+  const claim = `${dir}.claim-${randomUUID()}`;
+  try { fs.renameSync(dir, claim); } catch (error: any) { log.warn("cron_lock_remove", { dir, outcome: "claim_failed", reason: error?.code || "rename_error" }); return false; }
+  if (expected && !sameOwner(readOwner(claim), expected)) { restoreClaim(claim, dir); log.warn("cron_lock_remove", { dir, outcome: "claim_not_owned" }); return false; }
+  try { fs.rmSync(claim, { recursive: true, force: false }); log.info("cron_lock_remove", { dir, outcome: "removed" }); return true; }
+  catch (error: any) { restoreClaim(claim, dir); log.warn("cron_lock_remove", { dir, outcome: "remove_failed", reason: error?.code || "remove_error" }); return false; }
+}
+function acquireDirectoryLock(dir: string, instanceId: string, reclaimStale: boolean, deps = startTokenDeps): CronLockOwner | null {
+  const token = resolveProcessStartToken(process.pid, deps);
+  // Without a PID-reuse-safe token, durable leadership is disabled rather than unsafe.
+  if (!token) { log.warn("cron_leader_lock_unavailable", { dir, instance_id: instanceId, reason: "no_process_token" }); return null; }
+  const owner: CronLockOwner = { pid: process.pid, process_start_token: token, instance_id: instanceId, heartbeat_at: new Date().toISOString() };
+  try {
+    fs.mkdirSync(dir, { mode: 0o700 });
+    owner.fd = fs.openSync(ownerPath(dir), "wx", 0o600);
+    fs.writeFileSync(owner.fd, serializedOwner(owner));
+    const acquired = sameOwner(readOwner(dir), owner);
+    log.info("cron_leader_lock_acquire", { dir, instance_id: instanceId, outcome: acquired ? "acquired" : "verification_failed" });
+    if (acquired) { leaderOwners.set(dir, owner); return owner; }
+    safelyRemoveLock(dir, owner);
+    if (owner.fd !== undefined) try { fs.closeSync(owner.fd); } catch {}
+    return null;
+  } catch (error: any) {
+    if (owner.fd !== undefined) try { fs.closeSync(owner.fd); } catch {}
+    if (error?.code !== "EEXIST" || !reclaimStale) { (error?.code === "EEXIST" ? log.debug : log.warn)("cron_leader_lock_acquire", { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "lock_error" }); return null; }
+    if (!reclaimStaleLock(dir, instanceId, "leader")) return null;
+    return acquireDirectoryLock(dir, instanceId, false);
+  }
+}
+export function readDurableCronStore(store: string): DurableCronEnvelope {
+  try {
+    const value = JSON.parse(fs.readFileSync(store, "utf8"));
+    if (value?.version === 1 && Array.isArray(value.tasks)) return value;
+    if (Array.isArray(value)) return { version: 1, tasks: value };
+  } catch {}
+  return { version: 1, tasks: [] };
+}
+export function validateDurableCronTask(task: unknown): asserts task is DurableCronTask {
+  const value = task as DurableCronTask;
+  if (!value || typeof value !== "object" || typeof value.id !== "string" || !value.id || typeof value.cwd !== "string" || !value.cwd || typeof value.description !== "string" || typeof value.task !== "string" || !value.task.trim() || value.scope !== "project" || !Number.isFinite(value.createdAt)) throw new Error("Invalid durable cron task");
+  if (value.intervalMs !== undefined && (!Number.isFinite(value.intervalMs) || value.intervalMs < 10_000)) throw new Error("Invalid durable cron interval");
+  if (value.atHour !== undefined && (!Number.isInteger(value.atHour) || value.atHour < 0 || value.atHour > 23)) throw new Error("Invalid durable cron hour");
+  if (value.atMinute !== undefined && (!Number.isInteger(value.atMinute) || value.atMinute < 0 || value.atMinute > 59 || value.atHour === undefined)) throw new Error("Invalid durable cron minute");
+  if (value.intervalMs === undefined && value.atHour === undefined) throw new Error("Invalid durable cron schedule");
+  if ((value.lastRun !== undefined && !Number.isFinite(value.lastRun)) || (value.nextRun !== undefined && !Number.isFinite(value.nextRun))) throw new Error("Invalid durable cron timestamp");
+}
+function validate(tasks: DurableCronTask[]) { for (const task of tasks) validateDurableCronTask(task); }
+function atomicWrite(file: string, content: string) {
+  fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+  const temp = `${file}.${process.pid}.${randomUUID()}.tmp`;
+  fs.writeFileSync(temp, content, { mode: 0o600 });
+  fs.renameSync(temp, file);
+}
+function acquireMutationLock(dir: string, instanceId = randomUUID()): CronLockOwner | null {
+  const token = resolveProcessStartToken(process.pid);
+  if (!token) { log.warn("cron_mutation_lock_unavailable", { dir, instance_id: instanceId, reason: "no_process_token" }); return null; }
+  const owner: CronLockOwner = { pid: process.pid, process_start_token: token, instance_id: instanceId, heartbeat_at: new Date().toISOString() };
+  try {
+    fs.mkdirSync(dir, { mode: 0o700 });
+    fs.writeFileSync(ownerPath(dir), serializedOwner(owner), { flag: "wx", mode: 0o600 });
+    const acquired = sameOwner(readOwner(dir), owner);
+    log.debug("cron_mutation_lock_acquire", { dir, instance_id: instanceId, outcome: acquired ? "acquired" : "verification_failed" });
+    return acquired ? owner : null;
+  } catch (error: any) {
+    if (error?.code !== "EEXIST") { log.warn("cron_mutation_lock_acquire", { dir, instance_id: instanceId, outcome: "failed", reason: error?.code || "lock_error" }); return null; }
+    if (!reclaimStaleLock(dir, instanceId, "mutation")) return null;
+    return acquireMutationLock(dir, instanceId);
+  }
+}
+function releaseMutationLock(dir: string, owner: CronLockOwner) { safelyRemoveLock(dir, owner); }
+export function mutateDurableCronStore(store: string, change: (tasks: DurableCronTask[]) => DurableCronTask[]): DurableCronEnvelope {
+  const lock = lockPath(store, "mutation"); let owner: CronLockOwner | null = null;
+  for (let attempt = 0; attempt < 100; attempt++) { owner = acquireMutationLock(lock); if (owner) break; sleep(20); }
+  if (!owner) throw new Error("Timed out waiting for cron storage lock");
+  try {
+    // Corrupt records must not prevent valid tasks from being updated. Schedulers
+    // log each skipped record before calling here.
+    const current = readDurableCronStore(store).tasks.filter((task) => { try { validateDurableCronTask(task); return true; } catch { return false; } });
+    const tasks = change(current); validate(tasks); const envelope: DurableCronEnvelope = { version: 1, tasks }; atomicWrite(store, JSON.stringify(envelope)); return envelope;
+  }
+  finally { releaseMutationLock(lock, owner); }
+}
+export function acquireLeaderLock(store: string, instanceId: string, deps: StartTokenDeps = startTokenDeps): CronLockOwner | null { return acquireDirectoryLock(lockPath(store, "leader"), instanceId, true, deps); }
+export function refreshLeaderLock(store: string, owner: CronLockOwner): boolean {
+  const dir = lockPath(store, "leader");
+  if (owner.fd === undefined || !sameOwner(readOwner(dir), owner)) return false;
+  owner.heartbeat_at = new Date().toISOString();
+  try {
+    // Write through the exclusively-created inode, never a lock pathname that
+    // could have been atomically reclaimed and recreated by another process.
+    fs.ftruncateSync(owner.fd, 0); fs.writeSync(owner.fd, serializedOwner(owner), 0, "utf8"); fs.fsyncSync(owner.fd);
+    return sameOwner(readOwner(dir), owner);
+  } catch { return false; }
+}
+export function releaseLeaderLock(store: string, owner: CronLockOwner) {
+  const dir = lockPath(store, "leader");
+  const locallyOwned = leaderOwners.get(dir) === owner;
+  const released = safelyRemoveLock(dir, owner);
+  if (locallyOwned && owner.fd !== undefined) try { fs.closeSync(owner.fd); } catch {}
+  if (locallyOwned) leaderOwners.delete(dir);
+  (released ? log.info : log.warn)("cron_leader_lock_release", { dir, instance_id: owner.instance_id, outcome: released ? "released" : "path_retained" });
+  return released;
+}