pi-oracle 0.3.4 → 0.4.0

package/extensions/oracle/lib/runtime.ts

@@ -1,8 +1,15 @@
-import { randomUUID, createHash } from "node:crypto";
+// Purpose: Manage oracle browser runtime allocation, lease admission, seed/runtime profile handling, and runtime cleanup for the extension side.
+// Responsibilities: Allocate runtimes, enforce persisted-session requirements, acquire/release runtime and conversation leases, and clean up runtime artifacts safely.
+// Scope: Extension-side runtime coordination only; shared concurrency/process primitives live in extensions/oracle/shared.
+// Usage: Imported by jobs, tools, and queue logic to provision or tear down isolated oracle browser runtimes.
+// Invariants/Assumptions: Lease metadata is the admission source of truth, tracked worker identity checks defend against PID reuse, and runtime cleanup always attempts lease release.
+import { randomUUID } from "node:crypto";
 import { spawn } from "node:child_process";
 import { existsSync, realpathSync, readFileSync } from "node:fs";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
-import { basename, dirname, join } from "node:path";
+import { dirname, join } from "node:path";
+import { jobBlocksAdmission } from "../shared/job-coordination-helpers.mjs";
+import { isTrackedProcessAlive } from "../shared/process-helpers.mjs";
 import type { OracleConfig } from "./config.js";
 import { createLease, listLeaseMetadata, readLeaseMetadata, releaseLease, withAuthLock } from "./locks.js";
 
@@ -12,6 +19,8 @@ const ORACLE_JOBS_DIR = process.env.PI_ORACLE_JOBS_DIR?.trim() || DEFAULT_ORACLE
 const AGENT_BROWSER_BIN = [process.env.AGENT_BROWSER_PATH, "/opt/homebrew/bin/agent-browser", "/usr/local/bin/agent-browser"].find(
   (candidate) => typeof candidate === "string" && candidate && existsSync(candidate),
 ) || "agent-browser";
+const PROFILE_CLONE_TIMEOUT_MS = 120_000;
+const ORACLE_SUBPROCESS_KILL_GRACE_MS = 2_000;
 
 export interface OracleRuntimeLeaseMetadata {
   jobId: string;
@@ -110,8 +119,18 @@ function activeJobExists(jobId: string): boolean {
   const path = join(ORACLE_JOBS_DIR, `oracle-${jobId}`, "job.json");
   if (!existsSync(path)) return false;
   try {
-    const job = JSON.parse(readFileSync(path, "utf8")) as { status?: string; cleanupWarnings?: unknown; cleanupPending?: unknown };
-    return ["preparing", "submitted", "waiting"].includes(job.status || "") || job.cleanupPending === true || (Array.isArray(job.cleanupWarnings) && job.cleanupWarnings.length > 0);
+    const job = JSON.parse(readFileSync(path, "utf8")) as {
+      status?: string;
+      cleanupPending?: unknown;
+      workerPid?: unknown;
+      workerStartedAt?: unknown;
+    };
+    return jobBlocksAdmission({
+      status: typeof job.status === "string" ? job.status : undefined,
+      cleanupPending: job.cleanupPending === true,
+      workerPid: typeof job.workerPid === "number" ? job.workerPid : undefined,
+      workerStartedAt: typeof job.workerStartedAt === "string" ? job.workerStartedAt : undefined,
+    }, isTrackedProcessAlive);
   } catch {
     return false;
   }
@@ -198,22 +217,55 @@ function profileCloneArgs(config: OracleConfig, sourceDir: string, destinationDi
   return ["-R", sourceDir, destinationDir];
 }
 
-async function spawnCp(args: string[]): Promise<void> {
+async function spawnCp(args: string[], options?: { timeoutMs?: number }): Promise<void> {
   await new Promise<void>((resolve, reject) => {
     const child = spawn("cp", args, { stdio: ["ignore", "pipe", "pipe"] });
     let stderr = "";
+    let timedOut = false;
+    let killTimer: NodeJS.Timeout | undefined;
+    let killGraceTimer: NodeJS.Timeout | undefined;
+
+    const clearTimers = () => {
+      if (killTimer) clearTimeout(killTimer);
+      if (killGraceTimer) clearTimeout(killGraceTimer);
+    };
+
+    if ((options?.timeoutMs ?? 0) > 0) {
+      killTimer = setTimeout(() => {
+        timedOut = true;
+        child.kill("SIGTERM");
+        killGraceTimer = setTimeout(() => {
+          child.kill("SIGKILL");
+        }, ORACLE_SUBPROCESS_KILL_GRACE_MS);
+        killGraceTimer.unref?.();
+      }, options?.timeoutMs);
+      killTimer.unref?.();
+    }
+
     child.stderr.on("data", (data) => {
       stderr += String(data);
     });
-    child.on("error", reject);
+    child.on("error", (error) => {
+      clearTimers();
+      reject(error);
+    });
     child.on("close", (code) => {
+      clearTimers();
+      if (timedOut) {
+        reject(new Error(stderr || `cp timed out after ${options?.timeoutMs}ms`));
+        return;
+      }
       if (code === 0) resolve();
       else reject(new Error(stderr || `cp exited with code ${code}`));
     });
   });
 }
 
-export async function cloneSeedProfileToRuntime(config: OracleConfig, runtimeProfileDir: string): Promise<string | undefined> {
+export async function cloneSeedProfileToRuntime(
+  config: OracleConfig,
+  runtimeProfileDir: string,
+  options?: { cpTimeoutMs?: number },
+): Promise<string | undefined> {
   const seedDir = config.browser.authSeedProfileDir;
   if (!existsSync(seedDir)) {
     throw new Error(`Oracle auth seed profile not found: ${seedDir}. Run /oracle-auth first.`);
@@ -222,7 +274,7 @@ export async function cloneSeedProfileToRuntime(config: OracleConfig, runtimePro
   await withAuthLock({ runtimeProfileDir, seedDir }, async () => {
     await rm(runtimeProfileDir, { recursive: true, force: true }).catch(() => undefined);
     await mkdir(dirname(runtimeProfileDir), { recursive: true, mode: 0o700 }).catch(() => undefined);
-    await spawnCp(profileCloneArgs(config, seedDir, runtimeProfileDir));
+    await spawnCp(profileCloneArgs(config, seedDir, runtimeProfileDir), { timeoutMs: options?.cpTimeoutMs ?? PROFILE_CLONE_TIMEOUT_MS });
   });
 
   return getSeedGeneration(config);
@@ -286,9 +338,6 @@ export async function cleanupRuntimeArtifacts(runtime: {
       report.warnings.push(`Failed to remove runtime profile ${runtime.runtimeProfileDir}: ${error.message}`);
     });
   }
-  if (report.warnings.length > 0) {
-    return report;
-  }
   if (runtime.conversationId) {
     report.attempted.push("conversationLease");
   }
@@ -305,6 +354,3 @@ export async function cleanupRuntimeArtifacts(runtime: {
   return report;
 }
 
-export function stableProjectLabel(projectId: string): string {
-  return basename(projectId) || createHash("sha256").update(projectId).digest("hex").slice(0, 8);
-}

package/extensions/oracle/lib/tools.ts

@@ -9,6 +9,8 @@ import { tmpdir } from "node:os";
 import { basename, join, posix } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { Type } from "@sinclair/typebox";
+import { formatOracleJobSummary, formatOracleSubmitResponse } from "../shared/job-observability-helpers.mjs";
+import { transitionOracleJobPhase } from "../shared/job-lifecycle-helpers.mjs";
 import { isLockTimeoutError, withGlobalReconcileLock, withLock } from "./locks.js";
 import {
   coerceOracleSubmitPresetId,
@@ -36,7 +38,6 @@ import {
   spawnWorker,
   terminateWorkerPid,
   updateJob,
-  withJobPhase,
   type OracleJob,
 } from "./jobs.js";
 import { getQueuePosition, promoteQueuedJobs, promoteQueuedJobsWithinAdmissionLock } from "./queue.js";
@@ -78,11 +79,16 @@ const ORACLE_CANCEL_PARAMS = Type.Object({
 const MAX_ARCHIVE_BYTES = 250 * 1024 * 1024;
 const MAX_QUEUED_JOBS_PER_ACTIVE_RUNTIME = 1;
 const MAX_QUEUED_ARCHIVE_BYTES_PER_ACTIVE_RUNTIME = MAX_ARCHIVE_BYTES;
+const ARCHIVE_COMMAND_TIMEOUT_MS = 120_000;
+const ARCHIVE_COMMAND_KILL_GRACE_MS = 2_000;
 
 const DEFAULT_ARCHIVE_EXCLUDED_DIR_NAMES_ANYWHERE = new Set([
   ".git",
   ".hg",
   ".svn",
+  ".pi",
+  ".oracle-context",
+  ".cursor",
   "node_modules",
   "target",
   ".venv",
@@ -118,6 +124,7 @@ const DEFAULT_ARCHIVE_EXCLUDED_FILES = new Set([
   ".netrc",
   ".npmrc",
   ".pypirc",
+  ".scratchpad.md",
   "Thumbs.db",
   "id_dsa",
   "id_ecdsa",
@@ -321,7 +328,13 @@ function formatArchiveOversizeError(args: {
     .join("\n");
 }
 
-async function writeArchiveFile(cwd: string, entries: string[], archivePath: string, listPath: string): Promise<number> {
+async function writeArchiveFile(
+  cwd: string,
+  entries: string[],
+  archivePath: string,
+  listPath: string,
+  options?: { commandTimeoutMs?: number },
+): Promise<number> {
   await writeFile(listPath, Buffer.from(`${entries.join("\0")}\0`), { mode: 0o600 });
   await rm(archivePath, { force: true }).catch(() => undefined);
 
@@ -337,24 +350,57 @@ async function writeArchiveFile(cwd: string, entries: string[], archivePath: str
 
     let stderr = "";
     let settled = false;
+    let timedOut = false;
+    let timeout: NodeJS.Timeout | undefined;
+    let killGraceTimer: NodeJS.Timeout | undefined;
     let tarCode: number | null | undefined;
     let zstdCode: number | null | undefined;
 
+    const clearTimers = () => {
+      if (timeout) clearTimeout(timeout);
+      if (killGraceTimer) clearTimeout(killGraceTimer);
+    };
+
+    const terminateChildren = () => {
+      tar.kill("SIGTERM");
+      zstd.kill("SIGTERM");
+      killGraceTimer = setTimeout(() => {
+        tar.kill("SIGKILL");
+        zstd.kill("SIGKILL");
+      }, ARCHIVE_COMMAND_KILL_GRACE_MS);
+      killGraceTimer.unref?.();
+    };
+
     const finish = (error?: Error) => {
       if (settled) return;
       if (error) {
         settled = true;
-        tar.kill("SIGTERM");
-        zstd.kill("SIGTERM");
+        clearTimers();
+        terminateChildren();
         rejectPromise(error);
         return;
       }
       if (tarCode === undefined || zstdCode === undefined) return;
       settled = true;
+      clearTimers();
+      if (timedOut) {
+        rejectPromise(new Error(stderr || `Oracle archive subprocess timed out after ${options?.commandTimeoutMs ?? ARCHIVE_COMMAND_TIMEOUT_MS}ms`));
+        return;
+      }
       if (tarCode === 0 && zstdCode === 0) resolvePromise();
       else rejectPromise(new Error(stderr || `archive command failed (tar=${tarCode}, zstd=${zstdCode})`));
     };
 
+    const commandTimeoutMs = options?.commandTimeoutMs ?? ARCHIVE_COMMAND_TIMEOUT_MS;
+    if (commandTimeoutMs > 0) {
+      timeout = setTimeout(() => {
+        timedOut = true;
+        stderr = `${stderr}${stderr ? "\n" : ""}Oracle archive subprocess timed out after ${commandTimeoutMs}ms`;
+        terminateChildren();
+      }, commandTimeoutMs);
+      timeout.unref?.();
+    }
+
     tar.stderr.on("data", (data) => {
       stderr += String(data);
     });
@@ -381,7 +427,7 @@ export async function createArchiveForTesting(
   cwd: string,
   files: string[],
   archivePath: string,
-  options?: { maxBytes?: number; adaptivePruneMinBytes?: number },
+  options?: { maxBytes?: number; adaptivePruneMinBytes?: number; commandTimeoutMs?: number },
 ): Promise<ArchiveCreationResult> {
   const archiveInputs = resolveArchiveInputs(cwd, files);
   const wholeRepoSelection = isWholeRepoArchiveSelection(archiveInputs);
@@ -403,7 +449,7 @@ export async function createArchiveForTesting(
         throw new Error("Oracle archive inputs are empty after default exclusions and automatic size pruning");
       }
 
-      const archiveBytes = await writeArchiveFile(cwd, expandedEntries, archivePath, listPath);
+      const archiveBytes = await writeArchiveFile(cwd, expandedEntries, archivePath, listPath, { commandTimeoutMs: options?.commandTimeoutMs });
       if (archiveBytes < maxBytes) {
         return {
           sha256: await sha256File(archivePath),
@@ -543,38 +589,10 @@ function redactJobDetails(job: NonNullable<ReturnType<typeof readJob>>) {
     cleanupWarnings: job.cleanupWarnings,
     lastCleanupAt: job.lastCleanupAt,
     error: job.error,
+    lifecycleEvents: job.lifecycleEvents,
   };
 }
 
-function formatAutoPrunedArchiveMessage(autoPrunedPrefixes: ArchiveCreationResult["autoPrunedPrefixes"]): string | undefined {
-  if (autoPrunedPrefixes.length === 0) return undefined;
-  return `Archive auto-pruned generic generated-output-name dirs to fit size limit: ${autoPrunedPrefixes.map((entry) => `${entry.relativePath}/ (${formatBytes(entry.bytes)})`).join(", ")}`;
-}
-
-function formatSubmitResponse(
-  job: NonNullable<ReturnType<typeof readJob>>,
-  options: {
-    autoPrunedPrefixes: ArchiveCreationResult["autoPrunedPrefixes"];
-    queued: boolean;
-    queuePosition?: number;
-    queueDepth?: number;
-  },
-): string {
-  return [
-    `${options.queued ? "Oracle job queued" : "Oracle job dispatched"}: ${job.id}`,
-    options.queued && options.queuePosition && options.queueDepth ? `Queue position: ${options.queuePosition} of ${options.queueDepth}` : undefined,
-    job.followUpToJobId ? `Follow-up to: ${job.followUpToJobId}` : undefined,
-    `Prompt: ${job.promptPath}`,
-    `Archive: ${job.archivePath}`,
-    formatAutoPrunedArchiveMessage(options.autoPrunedPrefixes),
-    `Response will be written to: ${job.responsePath}`,
-    options.queued ? "The job will start automatically when capacity is available." : undefined,
-    "Stop now and wait for the oracle completion wake-up.",
-  ]
-    .filter(Boolean)
-    .join("\n");
-}
-
 export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void {
   pi.registerTool({
     name: "oracle_submit",
@@ -738,7 +756,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
           content: [
             {
               type: "text",
-              text: formatSubmitResponse(job, {
+              text: formatOracleSubmitResponse(job, {
                 autoPrunedPrefixes: currentArchive.autoPrunedPrefixes,
                 queued,
                 queuePosition: queuePosition?.position,
@@ -769,7 +787,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
             content: [
               {
                 type: "text",
-                text: formatSubmitResponse(latest, {
+                text: formatOracleSubmitResponse(latest, {
                   autoPrunedPrefixes: archive?.autoPrunedPrefixes ?? [],
                   queued: true,
                   queuePosition: queuePosition?.position,
@@ -797,7 +815,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
             content: [
               {
                 type: "text",
-                text: formatSubmitResponse(latest, {
+                text: formatOracleSubmitResponse(latest, {
                   autoPrunedPrefixes: archive?.autoPrunedPrefixes ?? [],
                   queued: false,
                 }),
@@ -820,13 +838,13 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
         }
         if (job && (!latest || !isTerminalOracleJob(latest))) {
           const failedAt = new Date().toISOString();
-          await updateJob(job.id, (current) => ({
-            ...current,
-            ...withJobPhase("failed", {
-              status: "failed",
-              completedAt: failedAt,
+          await updateJob(job.id, (current) => transitionOracleJobPhase(current, "failed", {
+            at: failedAt,
+            source: "oracle:submit",
+            message: `Submission failed before durable worker handoff: ${message}`,
+            patch: {
               error: message,
-            }, failedAt),
+            },
           })).catch(() => undefined);
         }
         const cleanupReport = await cleanupRuntimeArtifacts({
@@ -877,28 +895,11 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
         content: [
           {
             type: "text",
-            text: [
-              `job: ${current.id}`,
-              `status: ${current.status}`,
-              current.queuedAt ? `queued: ${current.queuedAt}` : undefined,
-              current.submittedAt ? `submitted: ${current.submittedAt}` : undefined,
-              ...(current.status === "queued"
-                ? (() => {
-                  const queuePosition = getQueuePosition(current.id);
-                  return queuePosition ? [`queue-position: ${queuePosition.position} of ${queuePosition.depth}`] : [];
-                })()
-                : []),
-              current.followUpToJobId ? `follow-up-to: ${current.followUpToJobId}` : undefined,
-              current.chatUrl ? `chat: ${current.chatUrl}` : undefined,
-              current.responsePath ? `response: ${current.responsePath}` : undefined,
-              current.responseFormat ? `response-format: ${current.responseFormat}` : undefined,
-              `artifacts: ${getJobDir(current.id)}/artifacts`,
-              current.error ? `error: ${current.error}` : undefined,
-              "",
+            text: formatOracleJobSummary(current, {
+              queuePosition: current.status === "queued" ? getQueuePosition(current.id) : undefined,
+              artifactsPath: `${getJobDir(current.id)}/artifacts`,
               responsePreview,
-            ]
-              .filter(Boolean)
-              .join("\n"),
+            }),
           },
         ],
         details: { job: redactJobDetails(current) },

package/extensions/oracle/shared/job-coordination-helpers.d.mts

@@ -0,0 +1,84 @@
+export interface OracleDurableWorkerHandoffJobLike {
+  status?: string;
+  workerPid?: number;
+}
+
+export interface OracleAdmissionBlockingJobLike extends OracleDurableWorkerHandoffJobLike {
+  cleanupPending?: boolean;
+  workerStartedAt?: string;
+}
+
+export interface OracleRuntimeLeaseMetadataLike {
+  jobId: string;
+  runtimeId: string;
+  runtimeSessionName: string;
+  runtimeProfileDir: string;
+  projectId: string;
+  sessionId: string;
+  createdAt: string;
+}
+
+export interface OracleConversationLeaseMetadataLike {
+  jobId: string;
+  conversationId: string;
+  projectId: string;
+  sessionId: string;
+  createdAt: string;
+}
+
+export interface OracleQueuedPromotionFailureContext<TJob, TWorker> {
+  job: TJob;
+  latest?: TJob;
+  error: unknown;
+  at: string;
+  spawnedWorker?: TWorker;
+  runtimeLeaseAcquired: boolean;
+  conversationLeaseAcquired: boolean;
+}
+
+export type OracleQueuedPromotionFailureOutcome = void | "break";
+
+export interface OracleQueuedPromotionOptions<TJob extends { id: string; archivePath: string }, TWorker> {
+  listQueuedJobs: () => TJob[];
+  refreshJob: (jobId: string) => TJob | undefined;
+  readLatestJob: (jobId: string) => TJob | undefined;
+  isQueuedJob?: (job: TJob | undefined) => boolean;
+  acquireRuntimeLease: (job: TJob, at: string) => Promise<boolean>;
+  acquireConversationLease: (job: TJob, at: string) => Promise<boolean>;
+  releaseRuntimeLease: (job: TJob) => Promise<void>;
+  markSubmitted: (job: TJob, at: string) => Promise<void>;
+  spawnWorker: (job: TJob) => Promise<TWorker>;
+  persistWorker: (job: TJob, worker: TWorker) => Promise<void>;
+  hasDurableWorkerHandoff?: (job: TJob | undefined) => boolean;
+  isTerminalJob: (job: TJob) => boolean;
+  failQueuedPromotion: (job: TJob, message: string, at: string) => Promise<void>;
+  terminateSpawnedWorker: (worker: TWorker) => Promise<void>;
+  cleanupAfterFailure: (context: OracleQueuedPromotionFailureContext<TJob, TWorker>) => Promise<OracleQueuedPromotionFailureOutcome>;
+  onDurableHandoff?: (job: TJob, latest?: TJob) => Promise<void> | void;
+}
+
+export declare function isQueuedOracleJob(job: OracleDurableWorkerHandoffJobLike | undefined): boolean;
+export declare function compareQueuedOracleJobs(
+  left: { createdAt: string; queuedAt?: string; id: string },
+  right: { createdAt: string; queuedAt?: string; id: string },
+): number;
+export declare function hasDurableWorkerHandoff(job: OracleDurableWorkerHandoffJobLike | undefined): boolean;
+export declare function hasAdmissionBlockingWorker(
+  job: OracleAdmissionBlockingJobLike | undefined,
+  isTrackedProcessAliveFn?: (pid: number | undefined, startedAt?: string) => boolean,
+): boolean;
+export declare function jobBlocksAdmission(
+  job: OracleAdmissionBlockingJobLike | undefined,
+  isTrackedProcessAliveFn?: (pid: number | undefined, startedAt?: string) => boolean,
+): boolean;
+export declare function buildRuntimeLeaseMetadata(
+  job: { id: string; runtimeId: string; runtimeSessionName: string; runtimeProfileDir: string; projectId: string; sessionId: string },
+  createdAt: string,
+): OracleRuntimeLeaseMetadataLike;
+export declare function buildConversationLeaseMetadata(
+  job: { id: string; conversationId?: string; projectId: string; sessionId: string },
+  createdAt: string,
+): OracleConversationLeaseMetadataLike | undefined;
+export declare function runQueuedJobPromotionPass<TJob extends { id: string; archivePath: string }, TWorker>(
+  options: OracleQueuedPromotionOptions<TJob, TWorker>,
+): Promise<{ promotedJobIds: string[] }>;

package/extensions/oracle/shared/job-coordination-helpers.mjs

@@ -0,0 +1,168 @@
+// Purpose: Provide shared oracle job coordination helpers for admission control, lease metadata, and queued promotion orchestration.
+// Responsibilities: Normalize queue ordering, derive lease metadata, detect durable handoff/admission blockers, and run a single queued-promotion pass.
+// Scope: Pure coordination/state-machine logic only; filesystem I/O and job persistence remain in injected callbacks.
+// Usage: Imported by lib/queue.ts, lib/runtime.ts, lib/jobs.ts, and worker/run-job.mjs to keep concurrency semantics aligned.
+// Invariants/Assumptions: Queued jobs have durable ids/archive paths, and callers provide side-effect callbacks that preserve atomic job updates.
+
+import { existsSync } from "node:fs";
+import { isTrackedProcessAlive } from "./process-helpers.mjs";
+
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleAdmissionBlockingJobLike} OracleAdmissionBlockingJobLike */
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleConversationLeaseMetadataLike} OracleConversationLeaseMetadataLike */
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleDurableWorkerHandoffJobLike} OracleDurableWorkerHandoffJobLike */
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleRuntimeLeaseMetadataLike} OracleRuntimeLeaseMetadataLike */
+
+/**
+ * @param {OracleDurableWorkerHandoffJobLike | undefined} job
+ * @returns {boolean}
+ */
+export function isQueuedOracleJob(job) {
+  return job?.status === "queued";
+}
+
+/**
+ * @param {{ createdAt: string; queuedAt?: string; id: string }} left
+ * @param {{ createdAt: string; queuedAt?: string; id: string }} right
+ * @returns {number}
+ */
+export function compareQueuedOracleJobs(left, right) {
+  const leftKey = left.queuedAt ?? left.createdAt;
+  const rightKey = right.queuedAt ?? right.createdAt;
+  return leftKey.localeCompare(rightKey) || left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id);
+}
+
+/**
+ * @param {OracleDurableWorkerHandoffJobLike | undefined} job
+ * @returns {boolean}
+ */
+export function hasDurableWorkerHandoff(job) {
+  if (!job || job.status === "queued") return false;
+  if (job.workerPid) return true;
+  return false;
+}
+
+/**
+ * @param {OracleAdmissionBlockingJobLike | undefined} job
+ * @param {(pid: number | undefined, startedAt?: string) => boolean} [isTrackedProcessAliveFn]
+ * @returns {boolean}
+ */
+export function hasAdmissionBlockingWorker(job, isTrackedProcessAliveFn = isTrackedProcessAlive) {
+  if (!job?.workerPid) return false;
+  return isTrackedProcessAliveFn(job.workerPid, job.workerStartedAt);
+}
+
+/**
+ * @param {OracleAdmissionBlockingJobLike | undefined} job
+ * @param {(pid: number | undefined, startedAt?: string) => boolean} [isTrackedProcessAliveFn]
+ * @returns {boolean}
+ */
+export function jobBlocksAdmission(job, isTrackedProcessAliveFn = isTrackedProcessAlive) {
+  return ["preparing", "submitted", "waiting"].includes(String(job?.status || "")) ||
+    job?.cleanupPending === true ||
+    hasAdmissionBlockingWorker(job, isTrackedProcessAliveFn);
+}
+
+/**
+ * @param {{ id: string; runtimeId: string; runtimeSessionName: string; runtimeProfileDir: string; projectId: string; sessionId: string }} job
+ * @param {string} createdAt
+ * @returns {OracleRuntimeLeaseMetadataLike}
+ */
+export function buildRuntimeLeaseMetadata(job, createdAt) {
+  return {
+    jobId: job.id,
+    runtimeId: job.runtimeId,
+    runtimeSessionName: job.runtimeSessionName,
+    runtimeProfileDir: job.runtimeProfileDir,
+    projectId: job.projectId,
+    sessionId: job.sessionId,
+    createdAt,
+  };
+}
+
+/**
+ * @param {{ id: string; conversationId?: string; projectId: string; sessionId: string }} job
+ * @param {string} createdAt
+ * @returns {OracleConversationLeaseMetadataLike | undefined}
+ */
+export function buildConversationLeaseMetadata(job, createdAt) {
+  if (!job.conversationId) return undefined;
+  return {
+    jobId: job.id,
+    conversationId: job.conversationId,
+    projectId: job.projectId,
+    sessionId: job.sessionId,
+    createdAt,
+  };
+}
+
+/**
+ * @template {{ id: string; archivePath: string }} TJob
+ * @template TWorker
+ * @param {import("./job-coordination-helpers.d.mts").OracleQueuedPromotionOptions<TJob, TWorker>} options
+ * @returns {Promise<{ promotedJobIds: string[] }>}
+ */
+export async function runQueuedJobPromotionPass(options) {
+  const promotedJobIds = [];
+  const isQueuedJob = options.isQueuedJob ?? isQueuedOracleJob;
+  const durableHandoff = options.hasDurableWorkerHandoff ?? hasDurableWorkerHandoff;
+
+  for (const queuedJob of options.listQueuedJobs()) {
+    const promotedAt = new Date().toISOString();
+    let runtimeLeaseAcquired = false;
+    let conversationLeaseAcquired = false;
+    /** @type {TWorker | undefined} */
+    let spawnedWorker;
+
+    try {
+      const current = options.refreshJob(queuedJob.id);
+      if (!isQueuedJob(current)) continue;
+      if (!existsSync(current.archivePath)) {
+        await options.failQueuedPromotion(current, `Queued oracle archive is missing: ${current.archivePath}`, promotedAt);
+        continue;
+      }
+
+      const runtimeAttempt = await options.acquireRuntimeLease(current, promotedAt);
+      if (!runtimeAttempt) break;
+      runtimeLeaseAcquired = true;
+
+      const conversationAttempt = await options.acquireConversationLease(current, promotedAt);
+      if (!conversationAttempt) {
+        await options.releaseRuntimeLease(current).catch(() => undefined);
+        runtimeLeaseAcquired = false;
+        continue;
+      }
+      conversationLeaseAcquired = true;
+
+      await options.markSubmitted(current, promotedAt);
+      spawnedWorker = await options.spawnWorker(current);
+      await options.persistWorker(current, spawnedWorker);
+      promotedJobIds.push(current.id);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const latest = options.readLatestJob(queuedJob.id);
+      if (spawnedWorker && durableHandoff(latest)) {
+        promotedJobIds.push(queuedJob.id);
+        await options.onDurableHandoff?.(queuedJob, latest);
+        continue;
+      }
+      if (spawnedWorker) {
+        await options.terminateSpawnedWorker(spawnedWorker).catch(() => undefined);
+      }
+      if (latest && !options.isTerminalJob(latest)) {
+        await options.failQueuedPromotion(latest, message, promotedAt);
+      }
+      const failureOutcome = await options.cleanupAfterFailure({
+        job: queuedJob,
+        latest,
+        error,
+        at: promotedAt,
+        spawnedWorker,
+        runtimeLeaseAcquired,
+        conversationLeaseAcquired,
+      });
+      if (failureOutcome === "break") break;
+    }
+  }
+
+  return { promotedJobIds };
+}