pi-oracle 0.3.3 → 0.4.0

package/extensions/oracle/lib/runtime.ts

@@ -1,8 +1,15 @@
-import { randomUUID, createHash } from "node:crypto";
+// Purpose: Manage oracle browser runtime allocation, lease admission, seed/runtime profile handling, and runtime cleanup for the extension side.
+// Responsibilities: Allocate runtimes, enforce persisted-session requirements, acquire/release runtime and conversation leases, and clean up runtime artifacts safely.
+// Scope: Extension-side runtime coordination only; shared concurrency/process primitives live in extensions/oracle/shared.
+// Usage: Imported by jobs, tools, and queue logic to provision or tear down isolated oracle browser runtimes.
+// Invariants/Assumptions: Lease metadata is the admission source of truth, tracked worker identity checks defend against PID reuse, and runtime cleanup always attempts lease release.
+import { randomUUID } from "node:crypto";
 import { spawn } from "node:child_process";
 import { existsSync, realpathSync, readFileSync } from "node:fs";
 import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
-import { basename, dirname, join } from "node:path";
+import { dirname, join } from "node:path";
+import { jobBlocksAdmission } from "../shared/job-coordination-helpers.mjs";
+import { isTrackedProcessAlive } from "../shared/process-helpers.mjs";
 import type { OracleConfig } from "./config.js";
 import { createLease, listLeaseMetadata, readLeaseMetadata, releaseLease, withAuthLock } from "./locks.js";
 
@@ -12,6 +19,8 @@ const ORACLE_JOBS_DIR = process.env.PI_ORACLE_JOBS_DIR?.trim() || DEFAULT_ORACLE
 const AGENT_BROWSER_BIN = [process.env.AGENT_BROWSER_PATH, "/opt/homebrew/bin/agent-browser", "/usr/local/bin/agent-browser"].find(
   (candidate) => typeof candidate === "string" && candidate && existsSync(candidate),
 ) || "agent-browser";
+const PROFILE_CLONE_TIMEOUT_MS = 120_000;
+const ORACLE_SUBPROCESS_KILL_GRACE_MS = 2_000;
 
 export interface OracleRuntimeLeaseMetadata {
   jobId: string;
@@ -110,8 +119,18 @@ function activeJobExists(jobId: string): boolean {
   const path = join(ORACLE_JOBS_DIR, `oracle-${jobId}`, "job.json");
   if (!existsSync(path)) return false;
   try {
-    const job = JSON.parse(readFileSync(path, "utf8")) as { status?: string; cleanupWarnings?: unknown; cleanupPending?: unknown };
-    return ["preparing", "submitted", "waiting"].includes(job.status || "") || job.cleanupPending === true || (Array.isArray(job.cleanupWarnings) && job.cleanupWarnings.length > 0);
+    const job = JSON.parse(readFileSync(path, "utf8")) as {
+      status?: string;
+      cleanupPending?: unknown;
+      workerPid?: unknown;
+      workerStartedAt?: unknown;
+    };
+    return jobBlocksAdmission({
+      status: typeof job.status === "string" ? job.status : undefined,
+      cleanupPending: job.cleanupPending === true,
+      workerPid: typeof job.workerPid === "number" ? job.workerPid : undefined,
+      workerStartedAt: typeof job.workerStartedAt === "string" ? job.workerStartedAt : undefined,
+    }, isTrackedProcessAlive);
   } catch {
     return false;
   }
@@ -198,22 +217,55 @@ function profileCloneArgs(config: OracleConfig, sourceDir: string, destinationDi
   return ["-R", sourceDir, destinationDir];
 }
 
-async function spawnCp(args: string[]): Promise<void> {
+async function spawnCp(args: string[], options?: { timeoutMs?: number }): Promise<void> {
   await new Promise<void>((resolve, reject) => {
     const child = spawn("cp", args, { stdio: ["ignore", "pipe", "pipe"] });
     let stderr = "";
+    let timedOut = false;
+    let killTimer: NodeJS.Timeout | undefined;
+    let killGraceTimer: NodeJS.Timeout | undefined;
+
+    const clearTimers = () => {
+      if (killTimer) clearTimeout(killTimer);
+      if (killGraceTimer) clearTimeout(killGraceTimer);
+    };
+
+    if ((options?.timeoutMs ?? 0) > 0) {
+      killTimer = setTimeout(() => {
+        timedOut = true;
+        child.kill("SIGTERM");
+        killGraceTimer = setTimeout(() => {
+          child.kill("SIGKILL");
+        }, ORACLE_SUBPROCESS_KILL_GRACE_MS);
+        killGraceTimer.unref?.();
+      }, options?.timeoutMs);
+      killTimer.unref?.();
+    }
+
     child.stderr.on("data", (data) => {
       stderr += String(data);
     });
-    child.on("error", reject);
+    child.on("error", (error) => {
+      clearTimers();
+      reject(error);
+    });
     child.on("close", (code) => {
+      clearTimers();
+      if (timedOut) {
+        reject(new Error(stderr || `cp timed out after ${options?.timeoutMs}ms`));
+        return;
+      }
       if (code === 0) resolve();
       else reject(new Error(stderr || `cp exited with code ${code}`));
     });
   });
 }
 
-export async function cloneSeedProfileToRuntime(config: OracleConfig, runtimeProfileDir: string): Promise<string | undefined> {
+export async function cloneSeedProfileToRuntime(
+  config: OracleConfig,
+  runtimeProfileDir: string,
+  options?: { cpTimeoutMs?: number },
+): Promise<string | undefined> {
   const seedDir = config.browser.authSeedProfileDir;
   if (!existsSync(seedDir)) {
     throw new Error(`Oracle auth seed profile not found: ${seedDir}. Run /oracle-auth first.`);
@@ -222,7 +274,7 @@ export async function cloneSeedProfileToRuntime(config: OracleConfig, runtimePro
   await withAuthLock({ runtimeProfileDir, seedDir }, async () => {
     await rm(runtimeProfileDir, { recursive: true, force: true }).catch(() => undefined);
     await mkdir(dirname(runtimeProfileDir), { recursive: true, mode: 0o700 }).catch(() => undefined);
-    await spawnCp(profileCloneArgs(config, seedDir, runtimeProfileDir));
+    await spawnCp(profileCloneArgs(config, seedDir, runtimeProfileDir), { timeoutMs: options?.cpTimeoutMs ?? PROFILE_CLONE_TIMEOUT_MS });
   });
 
   return getSeedGeneration(config);
@@ -286,9 +338,6 @@ export async function cleanupRuntimeArtifacts(runtime: {
       report.warnings.push(`Failed to remove runtime profile ${runtime.runtimeProfileDir}: ${error.message}`);
     });
   }
-  if (report.warnings.length > 0) {
-    return report;
-  }
   if (runtime.conversationId) {
     report.attempted.push("conversationLease");
   }
@@ -305,6 +354,3 @@ export async function cleanupRuntimeArtifacts(runtime: {
   return report;
 }
 
-export function stableProjectLabel(projectId: string): string {
-  return basename(projectId) || createHash("sha256").update(projectId).digest("hex").slice(0, 8);
-}

package/extensions/oracle/lib/tools.ts

@@ -9,6 +9,8 @@ import { tmpdir } from "node:os";
 import { basename, join, posix } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { Type } from "@sinclair/typebox";
+import { formatOracleJobSummary, formatOracleSubmitResponse } from "../shared/job-observability-helpers.mjs";
+import { transitionOracleJobPhase } from "../shared/job-lifecycle-helpers.mjs";
 import { isLockTimeoutError, withGlobalReconcileLock, withLock } from "./locks.js";
 import {
   coerceOracleSubmitPresetId,
@@ -36,7 +38,6 @@ import {
   spawnWorker,
   terminateWorkerPid,
   updateJob,
-  withJobPhase,
   type OracleJob,
 } from "./jobs.js";
 import { getQueuePosition, promoteQueuedJobs, promoteQueuedJobsWithinAdmissionLock } from "./queue.js";
@@ -78,11 +79,16 @@ const ORACLE_CANCEL_PARAMS = Type.Object({
 const MAX_ARCHIVE_BYTES = 250 * 1024 * 1024;
 const MAX_QUEUED_JOBS_PER_ACTIVE_RUNTIME = 1;
 const MAX_QUEUED_ARCHIVE_BYTES_PER_ACTIVE_RUNTIME = MAX_ARCHIVE_BYTES;
+const ARCHIVE_COMMAND_TIMEOUT_MS = 120_000;
+const ARCHIVE_COMMAND_KILL_GRACE_MS = 2_000;
 
 const DEFAULT_ARCHIVE_EXCLUDED_DIR_NAMES_ANYWHERE = new Set([
   ".git",
   ".hg",
   ".svn",
+  ".pi",
+  ".oracle-context",
+  ".cursor",
   "node_modules",
   "target",
   ".venv",
@@ -107,8 +113,10 @@ const DEFAULT_ARCHIVE_EXCLUDED_DIR_NAMES_ANYWHERE = new Set([
   ".pnpm-store",
   ".serverless",
   ".aws-sam",
+  "secrets",
+  ".secrets",
 ]);
-const DEFAULT_ARCHIVE_EXCLUDED_DIR_NAMES_AT_REPO_ROOT = new Set(["coverage", "htmlcov", "tmp", "temp", ".tmp", "dist", "build", "out", "secrets", ".secrets"]);
+const DEFAULT_ARCHIVE_EXCLUDED_DIR_NAMES_AT_REPO_ROOT = new Set(["coverage", "htmlcov", "tmp", "temp", ".tmp", "dist", "build", "out"]);
 const DEFAULT_ARCHIVE_EXCLUDED_FILES = new Set([
   ".coverage",
   ".DS_Store",
@@ -116,6 +124,7 @@ const DEFAULT_ARCHIVE_EXCLUDED_FILES = new Set([
   ".netrc",
   ".npmrc",
   ".pypirc",
+  ".scratchpad.md",
   "Thumbs.db",
   "id_dsa",
   "id_ecdsa",
@@ -319,7 +328,13 @@ function formatArchiveOversizeError(args: {
     .join("\n");
 }
 
-async function writeArchiveFile(cwd: string, entries: string[], archivePath: string, listPath: string): Promise<number> {
+async function writeArchiveFile(
+  cwd: string,
+  entries: string[],
+  archivePath: string,
+  listPath: string,
+  options?: { commandTimeoutMs?: number },
+): Promise<number> {
   await writeFile(listPath, Buffer.from(`${entries.join("\0")}\0`), { mode: 0o600 });
   await rm(archivePath, { force: true }).catch(() => undefined);
 
@@ -335,24 +350,57 @@ async function writeArchiveFile(cwd: string, entries: string[], archivePath: str
 
     let stderr = "";
     let settled = false;
+    let timedOut = false;
+    let timeout: NodeJS.Timeout | undefined;
+    let killGraceTimer: NodeJS.Timeout | undefined;
     let tarCode: number | null | undefined;
     let zstdCode: number | null | undefined;
 
+    const clearTimers = () => {
+      if (timeout) clearTimeout(timeout);
+      if (killGraceTimer) clearTimeout(killGraceTimer);
+    };
+
+    const terminateChildren = () => {
+      tar.kill("SIGTERM");
+      zstd.kill("SIGTERM");
+      killGraceTimer = setTimeout(() => {
+        tar.kill("SIGKILL");
+        zstd.kill("SIGKILL");
+      }, ARCHIVE_COMMAND_KILL_GRACE_MS);
+      killGraceTimer.unref?.();
+    };
+
     const finish = (error?: Error) => {
       if (settled) return;
       if (error) {
         settled = true;
-        tar.kill("SIGTERM");
-        zstd.kill("SIGTERM");
+        clearTimers();
+        terminateChildren();
         rejectPromise(error);
         return;
       }
       if (tarCode === undefined || zstdCode === undefined) return;
       settled = true;
+      clearTimers();
+      if (timedOut) {
+        rejectPromise(new Error(stderr || `Oracle archive subprocess timed out after ${options?.commandTimeoutMs ?? ARCHIVE_COMMAND_TIMEOUT_MS}ms`));
+        return;
+      }
       if (tarCode === 0 && zstdCode === 0) resolvePromise();
       else rejectPromise(new Error(stderr || `archive command failed (tar=${tarCode}, zstd=${zstdCode})`));
     };
 
+    const commandTimeoutMs = options?.commandTimeoutMs ?? ARCHIVE_COMMAND_TIMEOUT_MS;
+    if (commandTimeoutMs > 0) {
+      timeout = setTimeout(() => {
+        timedOut = true;
+        stderr = `${stderr}${stderr ? "\n" : ""}Oracle archive subprocess timed out after ${commandTimeoutMs}ms`;
+        terminateChildren();
+      }, commandTimeoutMs);
+      timeout.unref?.();
+    }
+
     tar.stderr.on("data", (data) => {
       stderr += String(data);
     });
@@ -379,7 +427,7 @@ export async function createArchiveForTesting(
   cwd: string,
   files: string[],
   archivePath: string,
-  options?: { maxBytes?: number; adaptivePruneMinBytes?: number },
+  options?: { maxBytes?: number; adaptivePruneMinBytes?: number; commandTimeoutMs?: number },
 ): Promise<ArchiveCreationResult> {
   const archiveInputs = resolveArchiveInputs(cwd, files);
   const wholeRepoSelection = isWholeRepoArchiveSelection(archiveInputs);
@@ -401,7 +449,7 @@ export async function createArchiveForTesting(
         throw new Error("Oracle archive inputs are empty after default exclusions and automatic size pruning");
       }
 
-      const archiveBytes = await writeArchiveFile(cwd, expandedEntries, archivePath, listPath);
+      const archiveBytes = await writeArchiveFile(cwd, expandedEntries, archivePath, listPath, { commandTimeoutMs: options?.commandTimeoutMs });
       if (archiveBytes < maxBytes) {
         return {
           sha256: await sha256File(archivePath),
@@ -541,38 +589,10 @@ function redactJobDetails(job: NonNullable<ReturnType<typeof readJob>>) {
     cleanupWarnings: job.cleanupWarnings,
     lastCleanupAt: job.lastCleanupAt,
     error: job.error,
+    lifecycleEvents: job.lifecycleEvents,
   };
 }
 
-function formatAutoPrunedArchiveMessage(autoPrunedPrefixes: ArchiveCreationResult["autoPrunedPrefixes"]): string | undefined {
-  if (autoPrunedPrefixes.length === 0) return undefined;
-  return `Archive auto-pruned generic generated-output-name dirs to fit size limit: ${autoPrunedPrefixes.map((entry) => `${entry.relativePath}/ (${formatBytes(entry.bytes)})`).join(", ")}`;
-}
-
-function formatSubmitResponse(
-  job: NonNullable<ReturnType<typeof readJob>>,
-  options: {
-    autoPrunedPrefixes: ArchiveCreationResult["autoPrunedPrefixes"];
-    queued: boolean;
-    queuePosition?: number;
-    queueDepth?: number;
-  },
-): string {
-  return [
-    `${options.queued ? "Oracle job queued" : "Oracle job dispatched"}: ${job.id}`,
-    options.queued && options.queuePosition && options.queueDepth ? `Queue position: ${options.queuePosition} of ${options.queueDepth}` : undefined,
-    job.followUpToJobId ? `Follow-up to: ${job.followUpToJobId}` : undefined,
-    `Prompt: ${job.promptPath}`,
-    `Archive: ${job.archivePath}`,
-    formatAutoPrunedArchiveMessage(options.autoPrunedPrefixes),
-    `Response will be written to: ${job.responsePath}`,
-    options.queued ? "The job will start automatically when capacity is available." : undefined,
-    "Stop now and wait for the oracle completion wake-up.",
-  ]
-    .filter(Boolean)
-    .join("\n");
-}
-
 export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void {
   pi.registerTool({
     name: "oracle_submit",
@@ -583,7 +603,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
     promptSnippet: "Dispatch a background ChatGPT web oracle job after gathering repo context.",
     promptGuidelines: [
       "Gather context before calling oracle_submit.",
-      "By default, archive the whole repo by passing '.'; default archive exclusions apply automatically, including common bulky outputs and obvious credentials/private data like .env files, key material, credential dotfiles, local database files, and root secrets directories.",
+      "By default, archive the whole repo by passing '.'; default archive exclusions apply automatically, including common bulky outputs and obvious credentials/private data like .env files, key material, credential dotfiles, local database files, and nested secrets directories anywhere in the repo.",
       "Only narrow file selection when the user explicitly asks, the task is clearly scoped smaller, or privacy/sensitivity requires it.",
       "For very targeted asks like a single function or stack trace, a smaller archive is preferable.",
       "When files='.' and the post-exclusion archive is still too large, submit automatically prunes the largest nested directories matching generic generated-output names like build/, dist/, out/, coverage/, and tmp/ outside obvious source roots like src/ and lib/ until the archive fits or no candidate remains; successful submissions report what was pruned.",
@@ -736,7 +756,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
           content: [
             {
               type: "text",
-              text: formatSubmitResponse(job, {
+              text: formatOracleSubmitResponse(job, {
                 autoPrunedPrefixes: currentArchive.autoPrunedPrefixes,
                 queued,
                 queuePosition: queuePosition?.position,
@@ -767,7 +787,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
             content: [
               {
                 type: "text",
-                text: formatSubmitResponse(latest, {
+                text: formatOracleSubmitResponse(latest, {
                   autoPrunedPrefixes: archive?.autoPrunedPrefixes ?? [],
                   queued: true,
                   queuePosition: queuePosition?.position,
@@ -795,7 +815,7 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
             content: [
               {
                 type: "text",
-                text: formatSubmitResponse(latest, {
+                text: formatOracleSubmitResponse(latest, {
                   autoPrunedPrefixes: archive?.autoPrunedPrefixes ?? [],
                   queued: false,
                 }),
@@ -818,13 +838,13 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
         }
         if (job && (!latest || !isTerminalOracleJob(latest))) {
           const failedAt = new Date().toISOString();
-          await updateJob(job.id, (current) => ({
-            ...current,
-            ...withJobPhase("failed", {
-              status: "failed",
-              completedAt: failedAt,
+          await updateJob(job.id, (current) => transitionOracleJobPhase(current, "failed", {
+            at: failedAt,
+            source: "oracle:submit",
+            message: `Submission failed before durable worker handoff: ${message}`,
+            patch: {
               error: message,
-            }, failedAt),
+            },
           })).catch(() => undefined);
         }
         const cleanupReport = await cleanupRuntimeArtifacts({
@@ -875,28 +895,11 @@ export function registerOracleTools(pi: ExtensionAPI, workerPath: string): void
         content: [
           {
             type: "text",
-            text: [
-              `job: ${current.id}`,
-              `status: ${current.status}`,
-              current.queuedAt ? `queued: ${current.queuedAt}` : undefined,
-              current.submittedAt ? `submitted: ${current.submittedAt}` : undefined,
-              ...(current.status === "queued"
-                ? (() => {
-                  const queuePosition = getQueuePosition(current.id);
-                  return queuePosition ? [`queue-position: ${queuePosition.position} of ${queuePosition.depth}`] : [];
-                })()
-                : []),
-              current.followUpToJobId ? `follow-up-to: ${current.followUpToJobId}` : undefined,
-              current.chatUrl ? `chat: ${current.chatUrl}` : undefined,
-              current.responsePath ? `response: ${current.responsePath}` : undefined,
-              current.responseFormat ? `response-format: ${current.responseFormat}` : undefined,
-              `artifacts: ${getJobDir(current.id)}/artifacts`,
-              current.error ? `error: ${current.error}` : undefined,
-              "",
+            text: formatOracleJobSummary(current, {
+              queuePosition: current.status === "queued" ? getQueuePosition(current.id) : undefined,
+              artifactsPath: `${getJobDir(current.id)}/artifacts`,
               responsePreview,
-            ]
-              .filter(Boolean)
-              .join("\n"),
+            }),
           },
         ],
         details: { job: redactJobDetails(current) },

package/extensions/oracle/shared/job-coordination-helpers.d.mts

@@ -0,0 +1,84 @@
+export interface OracleDurableWorkerHandoffJobLike {
+  status?: string;
+  workerPid?: number;
+}
+
+export interface OracleAdmissionBlockingJobLike extends OracleDurableWorkerHandoffJobLike {
+  cleanupPending?: boolean;
+  workerStartedAt?: string;
+}
+
+export interface OracleRuntimeLeaseMetadataLike {
+  jobId: string;
+  runtimeId: string;
+  runtimeSessionName: string;
+  runtimeProfileDir: string;
+  projectId: string;
+  sessionId: string;
+  createdAt: string;
+}
+
+export interface OracleConversationLeaseMetadataLike {
+  jobId: string;
+  conversationId: string;
+  projectId: string;
+  sessionId: string;
+  createdAt: string;
+}
+
+export interface OracleQueuedPromotionFailureContext<TJob, TWorker> {
+  job: TJob;
+  latest?: TJob;
+  error: unknown;
+  at: string;
+  spawnedWorker?: TWorker;
+  runtimeLeaseAcquired: boolean;
+  conversationLeaseAcquired: boolean;
+}
+
+export type OracleQueuedPromotionFailureOutcome = void | "break";
+
+export interface OracleQueuedPromotionOptions<TJob extends { id: string; archivePath: string }, TWorker> {
+  listQueuedJobs: () => TJob[];
+  refreshJob: (jobId: string) => TJob | undefined;
+  readLatestJob: (jobId: string) => TJob | undefined;
+  isQueuedJob?: (job: TJob | undefined) => boolean;
+  acquireRuntimeLease: (job: TJob, at: string) => Promise<boolean>;
+  acquireConversationLease: (job: TJob, at: string) => Promise<boolean>;
+  releaseRuntimeLease: (job: TJob) => Promise<void>;
+  markSubmitted: (job: TJob, at: string) => Promise<void>;
+  spawnWorker: (job: TJob) => Promise<TWorker>;
+  persistWorker: (job: TJob, worker: TWorker) => Promise<void>;
+  hasDurableWorkerHandoff?: (job: TJob | undefined) => boolean;
+  isTerminalJob: (job: TJob) => boolean;
+  failQueuedPromotion: (job: TJob, message: string, at: string) => Promise<void>;
+  terminateSpawnedWorker: (worker: TWorker) => Promise<void>;
+  cleanupAfterFailure: (context: OracleQueuedPromotionFailureContext<TJob, TWorker>) => Promise<OracleQueuedPromotionFailureOutcome>;
+  onDurableHandoff?: (job: TJob, latest?: TJob) => Promise<void> | void;
+}
+
+export declare function isQueuedOracleJob(job: OracleDurableWorkerHandoffJobLike | undefined): boolean;
+export declare function compareQueuedOracleJobs(
+  left: { createdAt: string; queuedAt?: string; id: string },
+  right: { createdAt: string; queuedAt?: string; id: string },
+): number;
+export declare function hasDurableWorkerHandoff(job: OracleDurableWorkerHandoffJobLike | undefined): boolean;
+export declare function hasAdmissionBlockingWorker(
+  job: OracleAdmissionBlockingJobLike | undefined,
+  isTrackedProcessAliveFn?: (pid: number | undefined, startedAt?: string) => boolean,
+): boolean;
+export declare function jobBlocksAdmission(
+  job: OracleAdmissionBlockingJobLike | undefined,
+  isTrackedProcessAliveFn?: (pid: number | undefined, startedAt?: string) => boolean,
+): boolean;
+export declare function buildRuntimeLeaseMetadata(
+  job: { id: string; runtimeId: string; runtimeSessionName: string; runtimeProfileDir: string; projectId: string; sessionId: string },
+  createdAt: string,
+): OracleRuntimeLeaseMetadataLike;
+export declare function buildConversationLeaseMetadata(
+  job: { id: string; conversationId?: string; projectId: string; sessionId: string },
+  createdAt: string,
+): OracleConversationLeaseMetadataLike | undefined;
+export declare function runQueuedJobPromotionPass<TJob extends { id: string; archivePath: string }, TWorker>(
+  options: OracleQueuedPromotionOptions<TJob, TWorker>,
+): Promise<{ promotedJobIds: string[] }>;

package/extensions/oracle/shared/job-coordination-helpers.mjs

@@ -0,0 +1,168 @@
+// Purpose: Provide shared oracle job coordination helpers for admission control, lease metadata, and queued promotion orchestration.
+// Responsibilities: Normalize queue ordering, derive lease metadata, detect durable handoff/admission blockers, and run a single queued-promotion pass.
+// Scope: Pure coordination/state-machine logic only; filesystem I/O and job persistence remain in injected callbacks.
+// Usage: Imported by lib/queue.ts, lib/runtime.ts, lib/jobs.ts, and worker/run-job.mjs to keep concurrency semantics aligned.
+// Invariants/Assumptions: Queued jobs have durable ids/archive paths, and callers provide side-effect callbacks that preserve atomic job updates.
+
+import { existsSync } from "node:fs";
+import { isTrackedProcessAlive } from "./process-helpers.mjs";
+
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleAdmissionBlockingJobLike} OracleAdmissionBlockingJobLike */
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleConversationLeaseMetadataLike} OracleConversationLeaseMetadataLike */
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleDurableWorkerHandoffJobLike} OracleDurableWorkerHandoffJobLike */
+/** @typedef {import("./job-coordination-helpers.d.mts").OracleRuntimeLeaseMetadataLike} OracleRuntimeLeaseMetadataLike */
+
+/**
+ * @param {OracleDurableWorkerHandoffJobLike | undefined} job
+ * @returns {boolean}
+ */
+export function isQueuedOracleJob(job) {
+  return job?.status === "queued";
+}
+
+/**
+ * @param {{ createdAt: string; queuedAt?: string; id: string }} left
+ * @param {{ createdAt: string; queuedAt?: string; id: string }} right
+ * @returns {number}
+ */
+export function compareQueuedOracleJobs(left, right) {
+  const leftKey = left.queuedAt ?? left.createdAt;
+  const rightKey = right.queuedAt ?? right.createdAt;
+  return leftKey.localeCompare(rightKey) || left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id);
+}
+
+/**
+ * @param {OracleDurableWorkerHandoffJobLike | undefined} job
+ * @returns {boolean}
+ */
+export function hasDurableWorkerHandoff(job) {
+  if (!job || job.status === "queued") return false;
+  if (job.workerPid) return true;
+  return false;
+}
+
+/**
+ * @param {OracleAdmissionBlockingJobLike | undefined} job
+ * @param {(pid: number | undefined, startedAt?: string) => boolean} [isTrackedProcessAliveFn]
+ * @returns {boolean}
+ */
+export function hasAdmissionBlockingWorker(job, isTrackedProcessAliveFn = isTrackedProcessAlive) {
+  if (!job?.workerPid) return false;
+  return isTrackedProcessAliveFn(job.workerPid, job.workerStartedAt);
+}
+
+/**
+ * @param {OracleAdmissionBlockingJobLike | undefined} job
+ * @param {(pid: number | undefined, startedAt?: string) => boolean} [isTrackedProcessAliveFn]
+ * @returns {boolean}
+ */
+export function jobBlocksAdmission(job, isTrackedProcessAliveFn = isTrackedProcessAlive) {
+  return ["preparing", "submitted", "waiting"].includes(String(job?.status || "")) ||
+    job?.cleanupPending === true ||
+    hasAdmissionBlockingWorker(job, isTrackedProcessAliveFn);
+}
+
+/**
+ * @param {{ id: string; runtimeId: string; runtimeSessionName: string; runtimeProfileDir: string; projectId: string; sessionId: string }} job
+ * @param {string} createdAt
+ * @returns {OracleRuntimeLeaseMetadataLike}
+ */
+export function buildRuntimeLeaseMetadata(job, createdAt) {
+  return {
+    jobId: job.id,
+    runtimeId: job.runtimeId,
+    runtimeSessionName: job.runtimeSessionName,
+    runtimeProfileDir: job.runtimeProfileDir,
+    projectId: job.projectId,
+    sessionId: job.sessionId,
+    createdAt,
+  };
+}
+
+/**
+ * @param {{ id: string; conversationId?: string; projectId: string; sessionId: string }} job
+ * @param {string} createdAt
+ * @returns {OracleConversationLeaseMetadataLike | undefined}
+ */
+export function buildConversationLeaseMetadata(job, createdAt) {
+  if (!job.conversationId) return undefined;
+  return {
+    jobId: job.id,
+    conversationId: job.conversationId,
+    projectId: job.projectId,
+    sessionId: job.sessionId,
+    createdAt,
+  };
+}
+
+/**
+ * @template {{ id: string; archivePath: string }} TJob
+ * @template TWorker
+ * @param {import("./job-coordination-helpers.d.mts").OracleQueuedPromotionOptions<TJob, TWorker>} options
+ * @returns {Promise<{ promotedJobIds: string[] }>}
+ */
+export async function runQueuedJobPromotionPass(options) {
+  const promotedJobIds = [];
+  const isQueuedJob = options.isQueuedJob ?? isQueuedOracleJob;
+  const durableHandoff = options.hasDurableWorkerHandoff ?? hasDurableWorkerHandoff;
+
+  for (const queuedJob of options.listQueuedJobs()) {
+    const promotedAt = new Date().toISOString();
+    let runtimeLeaseAcquired = false;
+    let conversationLeaseAcquired = false;
+    /** @type {TWorker | undefined} */
+    let spawnedWorker;
+
+    try {
+      const current = options.refreshJob(queuedJob.id);
+      if (!isQueuedJob(current)) continue;
+      if (!existsSync(current.archivePath)) {
+        await options.failQueuedPromotion(current, `Queued oracle archive is missing: ${current.archivePath}`, promotedAt);
+        continue;
+      }
+
+      const runtimeAttempt = await options.acquireRuntimeLease(current, promotedAt);
+      if (!runtimeAttempt) break;
+      runtimeLeaseAcquired = true;
+
+      const conversationAttempt = await options.acquireConversationLease(current, promotedAt);
+      if (!conversationAttempt) {
+        await options.releaseRuntimeLease(current).catch(() => undefined);
+        runtimeLeaseAcquired = false;
+        continue;
+      }
+      conversationLeaseAcquired = true;
+
+      await options.markSubmitted(current, promotedAt);
+      spawnedWorker = await options.spawnWorker(current);
+      await options.persistWorker(current, spawnedWorker);
+      promotedJobIds.push(current.id);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      const latest = options.readLatestJob(queuedJob.id);
+      if (spawnedWorker && durableHandoff(latest)) {
+        promotedJobIds.push(queuedJob.id);
+        await options.onDurableHandoff?.(queuedJob, latest);
+        continue;
+      }
+      if (spawnedWorker) {
+        await options.terminateSpawnedWorker(spawnedWorker).catch(() => undefined);
+      }
+      if (latest && !options.isTerminalJob(latest)) {
+        await options.failQueuedPromotion(latest, message, promotedAt);
+      }
+      const failureOutcome = await options.cleanupAfterFailure({
+        job: queuedJob,
+        latest,
+        error,
+        at: promotedAt,
+        spawnedWorker,
+        runtimeLeaseAcquired,
+        conversationLeaseAcquired,
+      });
+      if (failureOutcome === "break") break;
+    }
+  }
+
+  return { promotedJobIds };
+}