pi-oracle 0.3.3 → 0.4.0

package/extensions/oracle/lib/locks.ts

@@ -1,19 +1,31 @@
-import { createHash } from "node:crypto";
-import { existsSync } from "node:fs";
-import { mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
-import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
-import { basename, join } from "node:path";
+// Purpose: Provide typed oracle lock/lease wrappers bound to the configured shared oracle state directory.
+// Responsibilities: Expose extension-facing lock helpers, sweep stale lock dirs, and preserve existing typed APIs over the shared state helper core.
+// Scope: Extension-process wrappers only; atomic filesystem coordination semantics live in shared/state-coordination-helpers.mjs.
+// Usage: Imported by oracle lib modules that need admission locks, job locks, reconcile locks, or lease metadata persistence.
+// Invariants/Assumptions: All lock/lease paths live under the single configured oracle state directory for this machine.
+
+import { mkdirSync } from "node:fs";
+import {
+  acquireStateLock,
+  createStateLease,
+  getStateLeasesDir,
+  getStateLocksDir,
+  listStateLeaseMetadata,
+  ORACLE_METADATA_WRITE_GRACE_MS,
+  ORACLE_TMP_STATE_DIR_GRACE_MS,
+  readStateLeaseMetadata,
+  releaseStateLease,
+  releaseStatePath,
+  sweepStaleStateLocks,
+  withStateLock,
+  writeStateLeaseMetadata,
+} from "../shared/state-coordination-helpers.mjs";
 
 export const DEFAULT_ORACLE_STATE_DIR = "/tmp/pi-oracle-state";
 export const ORACLE_STATE_DIR_ENV = "PI_ORACLE_STATE_DIR";
 const ORACLE_STATE_DIR = process.env[ORACLE_STATE_DIR_ENV]?.trim() || DEFAULT_ORACLE_STATE_DIR;
-const LOCKS_DIR = join(ORACLE_STATE_DIR, "locks");
-const LEASES_DIR = join(ORACLE_STATE_DIR, "leases");
-const DEFAULT_WAIT_MS = 30_000;
-const POLL_MS = 200;
-export const ORACLE_METADATA_WRITE_GRACE_MS = 1_000;
-/** Incomplete `.tmp-*` dirs are in-flight atomic creates; a 1s grace is too short under multi-process sweep + slow FS. */
-export const ORACLE_TMP_STATE_DIR_GRACE_MS = 60_000;
+
+export { ORACLE_METADATA_WRITE_GRACE_MS, ORACLE_TMP_STATE_DIR_GRACE_MS };
 
 export interface OracleLockHandle {
   path: string;
@@ -23,136 +35,25 @@ function ensureDirSync(path: string): void {
   mkdirSync(path, { recursive: true, mode: 0o700 });
 }
 
-function leaseKey(kind: string, key: string): string {
-  return `${kind}-${createHash("sha256").update(key).digest("hex").slice(0, 24)}`;
-}
-
 export function getOracleStateDir(): string {
   ensureDirSync(ORACLE_STATE_DIR);
   return ORACLE_STATE_DIR;
 }
 
 export function getLocksDir(): string {
-  ensureDirSync(LOCKS_DIR);
-  return LOCKS_DIR;
+  const dir = getStateLocksDir(getOracleStateDir());
+  ensureDirSync(dir);
+  return dir;
 }
 
 export function getLeasesDir(): string {
-  ensureDirSync(LEASES_DIR);
-  return LEASES_DIR;
-}
-
-function lockPath(kind: string, key: string): string {
-  return join(getLocksDir(), leaseKey(kind, key));
-}
-
-function leasePath(kind: string, key: string): string {
-  return join(getLeasesDir(), leaseKey(kind, key));
-}
-
-async function sleep(ms: number): Promise<void> {
-  await new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-async function writeMetadata(path: string, metadata: unknown): Promise<void> {
-  const targetPath = join(path, "metadata.json");
-  const tempPath = join(path, `metadata.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`);
-  await writeFile(tempPath, `${JSON.stringify(metadata, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
-  await rename(tempPath, targetPath);
-}
-
-async function createStateDirAtomically(parentDir: string, finalPath: string, metadata: unknown): Promise<void> {
-  const tempPath = join(parentDir, `.tmp-${basename(finalPath)}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`);
-  await mkdir(tempPath, { recursive: false, mode: 0o700 });
-  try {
-    await writeMetadata(tempPath, metadata);
-    await rename(tempPath, finalPath);
-  } catch (error) {
-    await rm(tempPath, { recursive: true, force: true }).catch(() => undefined);
-    throw error;
-  }
-}
-
-function getMetadataPath(path: string): string {
-  return join(path, "metadata.json");
-}
-
-function getMetadataState(path: string): "present" | "missing" | "invalid" {
-  const metadataPath = getMetadataPath(path);
-  if (!existsSync(metadataPath)) return "missing";
-  try {
-    JSON.parse(readFileSync(metadataPath, "utf8"));
-    return "present";
-  } catch {
-    return "invalid";
-  }
-}
-
-function isIncompleteStateDirStale(path: string, now = Date.now()): boolean {
-  try {
-    const stats = statSync(path);
-    const baselineMs = Math.max(stats.mtimeMs, stats.ctimeMs);
-    const graceMs = basename(path).startsWith(".tmp-") ? ORACLE_TMP_STATE_DIR_GRACE_MS : ORACLE_METADATA_WRITE_GRACE_MS;
-    return now - baselineMs >= graceMs;
-  } catch {
-    return false;
-  }
-}
-
-function readLockProcessPid(path: string): number | undefined {
-  const metadataPath = getMetadataPath(path);
-  if (!existsSync(metadataPath)) return undefined;
-  try {
-    const metadata = JSON.parse(readFileSync(metadataPath, "utf8")) as { processPid?: unknown };
-    return typeof metadata.processPid === "number" && Number.isInteger(metadata.processPid) && metadata.processPid > 0
-      ? metadata.processPid
-      : undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-function isProcessAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (error) {
-    if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return false;
-    return true;
-  }
-}
-
-function isStateDirExistsError(error: unknown): boolean {
-  return Boolean(error && typeof error === "object" && "code" in error && (error.code === "EEXIST" || error.code === "ENOTEMPTY"));
-}
-
-async function maybeReclaimIncompleteStateDir(path: string, now = Date.now()): Promise<boolean> {
-  if (getMetadataState(path) === "present") return false;
-  if (!isIncompleteStateDirStale(path, now)) return false;
-  await rm(path, { recursive: true, force: true }).catch(() => undefined);
-  return true;
-}
-
-async function maybeReclaimStaleLock(path: string, now = Date.now()): Promise<boolean> {
-  if (await maybeReclaimIncompleteStateDir(path, now)) return true;
-  const processPid = readLockProcessPid(path);
-  if (!processPid || isProcessAlive(processPid)) return false;
-  await rm(path, { recursive: true, force: true }).catch(() => undefined);
-  return true;
+  const dir = getStateLeasesDir(getOracleStateDir());
+  ensureDirSync(dir);
+  return dir;
 }
 
 export async function sweepStaleLocks(now = Date.now()): Promise<string[]> {
-  const dir = getLocksDir();
-  const removed: string[] = [];
-
-  for (const name of readdirSync(dir)) {
-    const path = join(dir, name);
-    if (await maybeReclaimStaleLock(path, now)) {
-      removed.push(path);
-    }
-  }
-
-  return removed;
+  return sweepStaleStateLocks(getOracleStateDir(), now);
 }
 
 export async function acquireLock(
@@ -161,28 +62,13 @@ export async function acquireLock(
   metadata: unknown,
   options?: { timeoutMs?: number },
 ): Promise<OracleLockHandle> {
-  const parentDir = getLocksDir();
-  const path = join(parentDir, leaseKey(kind, key));
-  const timeoutMs = options?.timeoutMs ?? DEFAULT_WAIT_MS;
-  const deadline = Date.now() + timeoutMs;
-
-  while (Date.now() < deadline) {
-    try {
-      await createStateDirAtomically(parentDir, path, metadata);
-      return { path };
-    } catch (error) {
-      if (!isStateDirExistsError(error)) throw error;
-      if (await maybeReclaimStaleLock(path)) continue;
-    }
-    await sleep(POLL_MS);
-  }
-
-  throw new Error(`Timed out waiting for oracle ${kind} lock: ${key}`);
+  return {
+    path: await acquireStateLock(getOracleStateDir(), kind, key, metadata, options?.timeoutMs),
+  };
 }
 
 export async function releaseLock(handle: OracleLockHandle | undefined): Promise<void> {
-  if (!handle) return;
-  await rm(handle.path, { recursive: true, force: true }).catch(() => undefined);
+  await releaseStatePath(handle?.path);
 }
 
 export async function withLock<T>(
@@ -192,12 +78,7 @@ export async function withLock<T>(
   fn: () => Promise<T>,
   options?: { timeoutMs?: number },
 ): Promise<T> {
-  const handle = await acquireLock(kind, key, metadata, options);
-  try {
-    return await fn();
-  } finally {
-    await releaseLock(handle);
-  }
+  return withStateLock(getOracleStateDir(), kind, key, metadata, fn, options?.timeoutMs);
 }
 
 export function isLockTimeoutError(error: unknown, kind?: string, key?: string): boolean {
@@ -232,70 +113,21 @@ export async function withJobLock<T>(jobId: string, metadata: unknown, fn: () =>
 }
 
 export async function createLease(kind: string, key: string, metadata: unknown): Promise<string> {
-  const parentDir = getLeasesDir();
-  const path = join(parentDir, leaseKey(kind, key));
-  const deadline = Date.now() + DEFAULT_WAIT_MS;
-
-  while (Date.now() < deadline) {
-    try {
-      await createStateDirAtomically(parentDir, path, metadata);
-      return path;
-    } catch (error) {
-      if (!isStateDirExistsError(error)) throw error;
-      if (await maybeReclaimIncompleteStateDir(path)) continue;
-      if (getMetadataState(path) === "present") throw error;
-    }
-    await sleep(POLL_MS);
-  }
-
-  throw new Error(`Timed out waiting for oracle ${kind} lease: ${key}`);
+  return createStateLease(getOracleStateDir(), kind, key, metadata);
 }
 
 export async function writeLeaseMetadata(kind: string, key: string, metadata: unknown): Promise<string> {
-  const parentDir = getLeasesDir();
-  const path = join(parentDir, leaseKey(kind, key));
-  if (existsSync(path)) {
-    await writeMetadata(path, metadata);
-    return path;
-  }
-  try {
-    await createStateDirAtomically(parentDir, path, metadata);
-  } catch (error) {
-    if (!isStateDirExistsError(error)) throw error;
-    if (await maybeReclaimIncompleteStateDir(path)) {
-      await createStateDirAtomically(parentDir, path, metadata);
-    } else {
-      await writeMetadata(path, metadata);
-    }
-  }
-  return path;
+  return writeStateLeaseMetadata(getOracleStateDir(), kind, key, metadata);
 }
 
 export async function readLeaseMetadata<T = unknown>(kind: string, key: string): Promise<T | undefined> {
-  const path = join(leasePath(kind, key), "metadata.json");
-  if (!existsSync(path)) return undefined;
-  try {
-    return JSON.parse(await readFile(path, "utf8")) as T;
-  } catch {
-    return undefined;
-  }
+  return readStateLeaseMetadata<T>(getOracleStateDir(), kind, key);
 }
 
 export async function releaseLease(kind: string, key: string): Promise<void> {
-  await rm(leasePath(kind, key), { recursive: true, force: true }).catch(() => undefined);
+  await releaseStateLease(getOracleStateDir(), kind, key);
 }
 
 export function listLeaseMetadata<T = unknown>(kind: string): T[] {
-  const dir = getLeasesDir();
-  return readdirSync(dir)
-    .filter((name) => name.startsWith(`${kind}-`))
-    .map((name) => join(dir, name, "metadata.json"))
-    .filter((path) => existsSync(path))
-    .flatMap((path) => {
-      try {
-        return [JSON.parse(readFileSync(path, "utf8")) as T];
-      } catch {
-        return [];
-      }
-    });
+  return listStateLeaseMetadata<T>(getOracleStateDir(), kind);
 }

package/extensions/oracle/lib/poller.ts

@@ -1,5 +1,11 @@
-import { execFileSync } from "node:child_process";
+// Purpose: Poll oracle jobs in the background, reconcile stale state, and deliver best-effort wake-up reminders to eligible sessions.
+// Responsibilities: Track live wake-up targets, promote queued jobs, scan terminal jobs for delivery, and keep session status text current.
+// Scope: Poller/orchestration only; durable lifecycle mutations live in jobs.ts and shared observability formatting lives in extensions/oracle/shared.
+// Usage: Imported by the oracle extension entrypoint to start or stop per-session oracle polling.
+// Invariants/Assumptions: Poller scans are serialized per session key, wake-up delivery is best-effort, and terminal-job notifications always re-read durable job state before send.
 import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
+import { buildOracleStatusText, buildOracleWakeupNotificationContent } from "../shared/job-observability-helpers.mjs";
+import { isProcessAlive, readProcessStartedAt } from "../shared/process-helpers.mjs";
 import { isLockTimeoutError, listLeaseMetadata, releaseLease, withGlobalReconcileLock, writeLeaseMetadata } from "./locks.js";
 import {
   getJobDir,
@@ -64,26 +70,6 @@ function jobMatchesContext(job: { projectId: string; sessionId: string }, sessio
   return job.projectId === projectId && job.sessionId === sessionId;
 }
 
-function readProcessStartedAt(pid: number | undefined): string | undefined {
-  if (!pid || pid <= 0) return undefined;
-  try {
-    const startedAt = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" }).trim();
-    return startedAt || undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-function isProcessAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (error) {
-    if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return false;
-    return true;
-  }
-}
-
 function parseTimestamp(value: string | undefined): number | undefined {
   if (!value) return undefined;
   const parsed = Date.parse(value);
@@ -157,43 +143,20 @@ export function refreshOracleStatus(ctx: ExtensionContext): void {
     return;
   }
   const counts = getJobCounts(ctx);
-  if (counts.active > 0 && counts.queued > 0) {
-    ctx.ui.setStatus("oracle", ctx.ui.theme.fg("success", `oracle: running (${counts.active}), queued (${counts.queued})`));
-    return;
-  }
-  if (counts.active > 0) {
-    const suffix = counts.active > 1 ? ` (${counts.active})` : "";
-    ctx.ui.setStatus("oracle", ctx.ui.theme.fg("success", `oracle: running${suffix}`));
-    return;
-  }
-  if (counts.queued > 0) {
-    const suffix = counts.queued > 1 ? ` (${counts.queued})` : "";
-    ctx.ui.setStatus("oracle", ctx.ui.theme.fg("accent", `oracle: queued${suffix}`));
-    return;
-  }
-
-  ctx.ui.setStatus("oracle", ctx.ui.theme.fg("accent", "oracle: ready"));
-}
-
-function buildNotificationContent(job: OraclePollerJob): string {
-  const responsePath = job.responsePath || `${getJobDir(job.id)}/response.md`;
-  const artifactsPath = `${getJobDir(job.id)}/artifacts`;
-  return [
-    `Oracle job ${job.id} is ${job.status}.`,
-    `Use oracle_read with jobId ${job.id} to open the response and settle wake-up retries.`,
-    `Response file: ${responsePath}`,
-    `Artifacts: ${artifactsPath}`,
-    job.error ? `Error: ${job.error}` : "After oracle_read, continue from the oracle output.",
-  ].join("\n");
+  const statusText = buildOracleStatusText(counts);
+  const tone = counts.active > 0 ? "success" : "accent";
+  ctx.ui.setStatus("oracle", ctx.ui.theme.fg(tone, statusText));
 }
 
-
 function requestWakeupTurn(pi: ExtensionAPI, job: OraclePollerJob): void {
   pi.sendMessage(
     {
       customType: ORACLE_WAKEUP_REMINDER_CUSTOM_TYPE,
       display: false,
-      content: buildNotificationContent(job),
+      content: buildOracleWakeupNotificationContent(job, {
+        responsePath: job.responsePath || `${getJobDir(job.id)}/response.md`,
+        artifactsPath: `${getJobDir(job.id)}/artifacts`,
+      }),
       details: { jobId: job.id, status: job.status },
     },
     { triggerTurn: true, deliverAs: "followUp" },

package/extensions/oracle/lib/queue.ts

@@ -1,15 +1,21 @@
-import { existsSync } from "node:fs";
+// Purpose: Coordinate queued oracle job ordering and promotion into active worker execution.
+// Responsibilities: List queued jobs, compute queue position, and promote queued work under admission control using shared coordination helpers.
+// Scope: Extension-side queue orchestration only; shared promotion primitives live in extensions/oracle/shared and worker-side autonomous promotion stays in run-job.mjs.
+// Usage: Imported by oracle tools/commands when queued jobs may advance after submission or cancellation.
+// Invariants/Assumptions: Queue promotion runs under the global admission lock and only promotes jobs with durable archives and acquired runtime/conversation leases.
+import {
+  buildConversationLeaseMetadata,
+  buildRuntimeLeaseMetadata,
+  compareQueuedOracleJobs,
+  hasDurableWorkerHandoff,
+  isQueuedOracleJob,
+  runQueuedJobPromotionPass,
+} from "../shared/job-coordination-helpers.mjs";
+import { transitionOracleJobPhase } from "../shared/job-lifecycle-helpers.mjs";
 import { loadOracleConfig } from "./config.js";
 import { withLock } from "./locks.js";
-import { appendCleanupWarnings, createJob, hasDurableWorkerHandoff, isTerminalOracleJob, listOracleJobDirs, readJob, spawnWorker, terminateWorkerPid, updateJob, withJobPhase, type OracleJob } from "./jobs.js";
-import {
-  cleanupRuntimeArtifacts,
-  releaseRuntimeLease,
-  tryAcquireConversationLease,
-  tryAcquireRuntimeLease,
-  type OracleConversationLeaseMetadata,
-  type OracleRuntimeLeaseMetadata,
-} from "./runtime.js";
+import { appendCleanupWarnings, createJob, isTerminalOracleJob, listOracleJobDirs, readJob, spawnWorker, terminateWorkerPid, updateJob, type OracleJob } from "./jobs.js";
+import { cleanupRuntimeArtifacts, releaseRuntimeLease, tryAcquireConversationLease, tryAcquireRuntimeLease } from "./runtime.js";
 
 export interface OracleQueuePosition {
   position: number;
@@ -24,13 +30,11 @@ export interface PromoteQueuedJobsOptions {
 }
 
 function isQueuedJob(job: OracleJob | undefined): job is OracleJob {
-  return Boolean(job && job.status === "queued");
+  return isQueuedOracleJob(job);
 }
 
 export function compareQueuedJobs(left: OracleJob, right: OracleJob): number {
-  const leftKey = left.queuedAt ?? left.createdAt;
-  const rightKey = right.queuedAt ?? right.createdAt;
-  return leftKey.localeCompare(rightKey) || left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id);
+  return compareQueuedOracleJobs(left, right);
 }
 
 export function listQueuedJobs(): OracleJob[] {
@@ -50,129 +54,88 @@ export function getQueuePosition(jobId: string): OracleQueuePosition | undefined
   };
 }
 
-function runtimeLeaseMetadata(job: OracleJob, createdAt: string): OracleRuntimeLeaseMetadata {
-  return {
-    jobId: job.id,
-    runtimeId: job.runtimeId,
-    runtimeSessionName: job.runtimeSessionName,
-    runtimeProfileDir: job.runtimeProfileDir,
-    projectId: job.projectId,
-    sessionId: job.sessionId,
-    createdAt,
-  };
-}
-
-function conversationLeaseMetadata(job: OracleJob, createdAt: string): OracleConversationLeaseMetadata | undefined {
-  if (!job.conversationId) return undefined;
-  return {
-    jobId: job.id,
-    conversationId: job.conversationId,
-    projectId: job.projectId,
-    sessionId: job.sessionId,
-    createdAt,
-  };
-}
-
 async function failQueuedPromotion(job: OracleJob, message: string, at: string): Promise<void> {
-  await updateJob(job.id, (current) => ({
-    ...current,
-    ...withJobPhase("failed", {
-      status: "failed",
-      completedAt: at,
+  await updateJob(job.id, (current) => transitionOracleJobPhase(current, "failed", {
+    at,
+    source: "oracle:queue",
+    message: `Queued promotion failed: ${message}`,
+    clearNotificationClaim: true,
+    patch: {
       heartbeatAt: at,
-      notifyClaimedAt: undefined,
-      notifyClaimedBy: undefined,
       error: message,
-    }, at),
+    },
   })).catch(() => undefined);
 }
 
 export async function promoteQueuedJobsWithinAdmissionLock(options: PromoteQueuedJobsOptions): Promise<{ promotedJobIds: string[] }> {
   const spawnWorkerFn = options.spawnWorkerFn ?? spawnWorker;
   const loadConfigFn = options.loadConfigFn ?? loadOracleConfig;
-  const promotedJobIds: string[] = [];
-
-  for (const queuedJob of listQueuedJobs()) {
-    const now = new Date().toISOString();
-    let runtimeLeaseAcquired = false;
-    let conversationLeaseAcquired = false;
-    let workerSpawned = false;
-    let spawnedWorker: Awaited<ReturnType<typeof spawnWorker>> | undefined;
 
-    try {
-      const current = readJob(queuedJob.id);
-      if (!isQueuedJob(current)) continue;
-      if (!existsSync(current.archivePath)) {
-        await failQueuedPromotion(current, `Queued oracle archive is missing: ${current.archivePath}`, now);
-        continue;
-      }
-
-      const config = current.config ?? loadConfigFn(current.cwd);
-      const runtimeAttempt = await tryAcquireRuntimeLease(config, runtimeLeaseMetadata(current, now));
-      if (!runtimeAttempt.acquired) break;
-      runtimeLeaseAcquired = true;
-
-      const conversationMetadata = conversationLeaseMetadata(current, now);
-      if (conversationMetadata) {
-        const conversationAttempt = await tryAcquireConversationLease(conversationMetadata);
-        if (!conversationAttempt.acquired) {
-          await releaseRuntimeLease(current.runtimeId).catch(() => undefined);
-          runtimeLeaseAcquired = false;
-          continue;
-        }
-        conversationLeaseAcquired = true;
-      }
-
-      await updateJob(current.id, (latest) => {
+  return runQueuedJobPromotionPass<OracleJob, Awaited<ReturnType<typeof spawnWorkerFn>>>({
+    listQueuedJobs,
+    refreshJob: (jobId) => readJob(jobId),
+    readLatestJob: (jobId) => readJob(jobId),
+    isQueuedJob,
+    acquireRuntimeLease: async (job, at) => {
+      const config = job.config ?? loadConfigFn(job.cwd);
+      const attempt = await tryAcquireRuntimeLease(config, buildRuntimeLeaseMetadata(job, at));
+      return attempt.acquired;
+    },
+    acquireConversationLease: async (job, at) => {
+      const metadata = buildConversationLeaseMetadata(job, at);
+      if (!metadata) return true;
+      const attempt = await tryAcquireConversationLease(metadata);
+      return attempt.acquired;
+    },
+    releaseRuntimeLease: async (job) => {
+      await releaseRuntimeLease(job.runtimeId);
+    },
+    markSubmitted: async (job, at) => {
+      const config = job.config ?? loadConfigFn(job.cwd);
+      await updateJob(job.id, (latest) => {
         if (latest.status !== "queued") {
           throw new Error(`Queued job ${latest.id} changed state during promotion (${latest.status})`);
         }
-        return {
+        return transitionOracleJobPhase({
           ...latest,
           config,
-          ...withJobPhase("submitted", {
-            status: "submitted",
-            submittedAt: latest.submittedAt || now,
-          }, now),
-        };
+        }, "submitted", {
+          at,
+          source: "oracle:queue",
+          message: "Queued job admitted for worker launch.",
+          patch: {
+            submittedAt: latest.submittedAt || at,
+          },
+        });
       });
-
-      spawnedWorker = await spawnWorkerFn(options.workerPath, current.id);
-      workerSpawned = true;
-      const worker = spawnedWorker;
-      await updateJob(current.id, (latest) => ({
+    },
+    spawnWorker: async (job) => spawnWorkerFn(options.workerPath, job.id),
+    persistWorker: async (job, worker) => {
+      await updateJob(job.id, (latest) => ({
         ...latest,
         workerPid: worker.pid,
         workerNonce: worker.nonce,
         workerStartedAt: worker.startedAt,
       }));
-      promotedJobIds.push(current.id);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      const latest = readJob(queuedJob.id);
-      if (workerSpawned && latest && hasDurableWorkerHandoff(latest)) {
-        promotedJobIds.push(queuedJob.id);
-        continue;
-      }
-      if (spawnedWorker) {
-        await terminateWorkerPid(spawnedWorker.pid, spawnedWorker.startedAt).catch(() => undefined);
-      }
-      if (latest && !isTerminalOracleJob(latest)) {
-        await failQueuedPromotion(latest, message, now);
-      }
+    },
+    hasDurableWorkerHandoff,
+    isTerminalJob: isTerminalOracleJob,
+    failQueuedPromotion,
+    terminateSpawnedWorker: async (worker) => {
+      await terminateWorkerPid(worker.pid, worker.startedAt);
+    },
+    cleanupAfterFailure: async ({ job, at, spawnedWorker, runtimeLeaseAcquired, conversationLeaseAcquired }) => {
       const cleanupReport = await cleanupRuntimeArtifacts({
-        runtimeId: runtimeLeaseAcquired ? queuedJob.runtimeId : undefined,
-        runtimeProfileDir: runtimeLeaseAcquired ? queuedJob.runtimeProfileDir : undefined,
-        runtimeSessionName: workerSpawned ? queuedJob.runtimeSessionName : undefined,
-        conversationId: conversationLeaseAcquired ? queuedJob.conversationId : undefined,
+        runtimeId: runtimeLeaseAcquired ? job.runtimeId : undefined,
+        runtimeProfileDir: runtimeLeaseAcquired ? job.runtimeProfileDir : undefined,
+        runtimeSessionName: spawnedWorker ? job.runtimeSessionName : undefined,
+        conversationId: conversationLeaseAcquired ? job.conversationId : undefined,
       }).catch(() => ({ attempted: [], warnings: [] }));
       if (cleanupReport.warnings.length > 0) {
-        await appendCleanupWarnings(queuedJob.id, cleanupReport.warnings, now).catch(() => undefined);
+        await appendCleanupWarnings(job.id, cleanupReport.warnings, at).catch(() => undefined);
       }
-    }
-  }
-
-  return { promotedJobIds };
+    },
+  });
 }
 
 export async function promoteQueuedJobs(options: PromoteQueuedJobsOptions): Promise<{ promotedJobIds: string[] }> {