pi-oracle 0.3.4 → 0.5.0

package/extensions/oracle/worker/run-job.mjs

@@ -1,9 +1,24 @@
+// Purpose: Execute a single oracle worker job from browser launch through response/artifact extraction and cleanup.
+// Responsibilities: Drive the isolated browser session, update durable job state, coordinate cleanup, and autonomously promote queued work after successful teardown.
+// Scope: Worker runtime behavior only; shared concurrency/process helpers live in extensions/oracle/shared and extension-side policy remains in lib modules.
+// Usage: Spawned as a detached Node process with a job id argument by the oracle extension queue/submission flows.
+// Invariants/Assumptions: Job state is persisted under worker-held locks, browser/session artifacts live under the configured oracle directories, and cleanup preserves durable recovery semantics.
 import { createHash, randomUUID } from "node:crypto";
 import { existsSync, readdirSync, readFileSync } from "node:fs";
 import { appendFile, chmod, mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
 import { basename, dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { spawn, execFileSync } from "node:child_process";
+import { spawn } from "node:child_process";
+import {
+  buildConversationLeaseMetadata,
+  buildRuntimeLeaseMetadata,
+  compareQueuedOracleJobs,
+  hasDurableWorkerHandoff,
+  jobBlocksAdmission,
+  runQueuedJobPromotionPass,
+} from "../shared/job-coordination-helpers.mjs";
+import { applyOracleJobCleanupWarnings, clearOracleJobCleanupState, transitionOracleJobPhase } from "../shared/job-lifecycle-helpers.mjs";
+import { spawnDetachedNodeProcess, terminateTrackedProcess } from "../shared/process-helpers.mjs";
 import { extractArtifactLabels, FILE_LABEL_PATTERN_SOURCE, GENERIC_ARTIFACT_LABELS, parseSnapshotEntries, partitionStructuralArtifactCandidates } from "./artifact-heuristics.mjs";
 import {
   buildAllowedChatGptOrigins,
@@ -17,6 +32,7 @@ import {
   snapshotWeaklyMatchesRequestedModel,
   autoSwitchToThinkingSelectionVisible,
 } from "./chatgpt-ui-helpers.mjs";
+import { assistantSnapshotSlice, nextStableValueState, resolveStableConversationUrlCandidate, stripUrlQueryAndHash } from "./chatgpt-flow-helpers.mjs";
 import { createLease, listLeaseMetadata, readLeaseMetadata, releaseLease, withLock } from "./state-locks.mjs";
 
 const jobId = process.argv[2];
@@ -48,6 +64,7 @@ const ARTIFACT_DOWNLOAD_HEARTBEAT_MS = 10_000;
 const ARTIFACT_DOWNLOAD_TIMEOUT_MS = 90_000;
 const ARTIFACT_DOWNLOAD_MAX_ATTEMPTS = 2;
 const AGENT_BROWSER_CLOSE_TIMEOUT_MS = 10_000;
+const PROFILE_CLONE_TIMEOUT_MS = 120_000;
 const MODEL_CONFIGURATION_SETTLE_TIMEOUT_MS = 20_000;
 const MODEL_CONFIGURATION_SETTLE_POLL_MS = 250;
 const MODEL_CONFIGURATION_CLOSE_RETRY_MS = 1_000;
@@ -68,79 +85,10 @@ async function ensurePrivateDir(path) {
   await chmod(path, 0o700).catch(() => undefined);
 }
 
-function isProcessAlive(pid) {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch (error) {
-    if (error && typeof error === "object" && "code" in error && error.code === "ESRCH") return false;
-    return true;
-  }
-}
-
-function readProcessStartedAt(pid) {
-  if (!pid || pid <= 0) return undefined;
-  try {
-    const startedAt = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" }).trim();
-    return startedAt || undefined;
-  } catch {
-    return undefined;
-  }
-}
-
-async function waitForProcessStartedAt(pid, timeoutMs = 2_000) {
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    const startedAt = readProcessStartedAt(pid);
-    if (startedAt) return startedAt;
-    await sleep(100);
-  }
-  return readProcessStartedAt(pid);
-}
-
 async function terminateWorkerPid(pid, startedAt, options = {}) {
-  if (!pid || pid <= 0) return true;
-  const currentStartedAt = readProcessStartedAt(pid);
-  if (!currentStartedAt) return true;
-  if (startedAt && currentStartedAt !== startedAt) return false;
-
-  const termGraceMs = options.termGraceMs ?? 5_000;
-  const killGraceMs = options.killGraceMs ?? 2_000;
-
-  try {
-    process.kill(pid, "SIGTERM");
-  } catch {
-    return !isProcessAlive(pid);
-  }
-
-  const termDeadline = Date.now() + termGraceMs;
-  while (Date.now() < termDeadline) {
-    const liveStartedAt = readProcessStartedAt(pid);
-    if (!liveStartedAt) return true;
-    if (startedAt && liveStartedAt !== startedAt) return true;
-    await sleep(250);
-  }
-
-  try {
-    process.kill(pid, "SIGKILL");
-  } catch {
-    return !isProcessAlive(pid);
-  }
-
-  const killDeadline = Date.now() + killGraceMs;
-  while (Date.now() < killDeadline) {
-    const liveStartedAt = readProcessStartedAt(pid);
-    if (!liveStartedAt) return true;
-    if (startedAt && liveStartedAt !== startedAt) return true;
-    await sleep(250);
-  }
-
-  const finalStartedAt = readProcessStartedAt(pid);
-  if (!finalStartedAt) return true;
-  return startedAt ? finalStartedAt !== startedAt : false;
+  return terminateTrackedProcess(pid, startedAt, options);
 }
 
-
 async function secureWriteText(path, content) {
   const tmpPath = `${path}.${process.pid}.${Date.now()}.tmp`;
   await writeFile(tmpPath, content, { encoding: "utf8", mode: 0o600 });
@@ -186,25 +134,7 @@ function listQueuedJobs() {
     .filter((name) => name.startsWith("oracle-"))
     .map((name) => readAnyJob(name.slice("oracle-".length)))
     .filter((job) => job?.status === "queued")
-    .sort((left, right) => {
-      const leftKey = left?.queuedAt || left?.createdAt || "";
-      const rightKey = right?.queuedAt || right?.createdAt || "";
-      return leftKey.localeCompare(rightKey) || String(left?.createdAt || "").localeCompare(String(right?.createdAt || "")) || String(left?.id || "").localeCompare(String(right?.id || ""));
-    });
-}
-
-function isActiveJobStatus(status) {
-  return ["preparing", "submitted", "waiting"].includes(String(status || ""));
-}
-
-function jobBlocksAdmission(job) {
-  return isActiveJobStatus(job?.status) || job?.cleanupPending === true || (Array.isArray(job?.cleanupWarnings) && job.cleanupWarnings.length > 0);
-}
-
-function hasDurableWorkerHandoff(job) {
-  if (!job || job.status === "queued") return false;
-  if (job.workerPid) return true;
-  return false;
+    .sort(compareQueuedOracleJobs);
 }
 
 async function mutateAnyJob(targetJobId, mutator) {
@@ -217,12 +147,6 @@ async function mutateAnyJob(targetJobId, mutator) {
   });
 }
 
-async function writeAnyJob(targetJobId, job) {
-  await withLock(ORACLE_STATE_DIR, "job", targetJobId, { processPid: process.pid, action: "writeJob", targetJobId }, async () => {
-    await secureWriteText(getAnyJobPath(targetJobId), `${JSON.stringify(job, null, 2)}\n`);
-  });
-}
-
 async function writeJobUnlocked(job) {
   await secureWriteText(jobPath, `${JSON.stringify(job, null, 2)}\n`);
 }
@@ -243,14 +167,6 @@ async function mutateJob(mutator) {
   });
 }
 
-function phasePatch(phase, patch = undefined, at = new Date().toISOString()) {
-  return {
-    ...(patch || {}),
-    phase,
-    phaseAt: at,
-  };
-}
-
 async function heartbeat(patch = undefined, options = {}) {
   const now = Date.now();
   const force = options.force === true;
@@ -342,7 +258,7 @@ async function cloneSeedProfileToRuntime(job) {
     await rm(job.runtimeProfileDir, { recursive: true, force: true }).catch(() => undefined);
     await ensurePrivateDir(dirname(job.runtimeProfileDir));
     const cloneArgs = job.config.browser.cloneStrategy === "apfs-clone" ? ["-cR", seedDir, job.runtimeProfileDir] : ["-R", seedDir, job.runtimeProfileDir];
-    await spawnCommand("cp", cloneArgs);
+    await spawnCommand("cp", cloneArgs, { timeoutMs: PROFILE_CLONE_TIMEOUT_MS });
   }, 10 * 60 * 1000);
 
   return seedGeneration;
@@ -400,102 +316,77 @@ async function tryAcquireRuntimeLeaseForJob(job, createdAt) {
   if (liveLeases.length >= job.config.browser.maxConcurrentJobs) {
     return false;
   }
-  await createLease(ORACLE_STATE_DIR, "runtime", job.runtimeId, {
-    jobId: job.id,
-    runtimeId: job.runtimeId,
-    runtimeSessionName: job.runtimeSessionName,
-    runtimeProfileDir: job.runtimeProfileDir,
-    projectId: job.projectId,
-    sessionId: job.sessionId,
-    createdAt,
-  });
+  await createLease(ORACLE_STATE_DIR, "runtime", job.runtimeId, buildRuntimeLeaseMetadata(job, createdAt));
   return true;
 }
 
 async function tryAcquireConversationLeaseForJob(job, createdAt) {
-  if (!job.conversationId) return true;
-  const existing = await readLeaseMetadata(ORACLE_STATE_DIR, "conversation", job.conversationId);
+  const metadata = buildConversationLeaseMetadata(job, createdAt);
+  if (!metadata) return true;
+  const existing = await readLeaseMetadata(ORACLE_STATE_DIR, "conversation", metadata.conversationId);
   if (existing?.jobId === job.id) return true;
   if (existing && existing.jobId !== job.id) {
     if (!jobBlocksAdmission(readAnyJob(existing.jobId))) {
-      await releaseLease(ORACLE_STATE_DIR, "conversation", job.conversationId).catch(() => undefined);
+      await releaseLease(ORACLE_STATE_DIR, "conversation", metadata.conversationId).catch(() => undefined);
     } else {
       return false;
     }
   }
-  await createLease(ORACLE_STATE_DIR, "conversation", job.conversationId, {
-    jobId: job.id,
-    conversationId: job.conversationId,
-    projectId: job.projectId,
-    sessionId: job.sessionId,
-    createdAt,
-  });
+  await createLease(ORACLE_STATE_DIR, "conversation", metadata.conversationId, metadata);
   return true;
 }
 
 async function spawnDetachedWorker(targetJobId) {
-  const child = spawn(process.execPath, [WORKER_SCRIPT_PATH, targetJobId], {
-    detached: true,
-    stdio: "ignore",
-  });
-  child.unref();
+  const child = await spawnDetachedNodeProcess(WORKER_SCRIPT_PATH, [targetJobId]);
   return {
     pid: child.pid,
     workerNonce: randomUUID(),
-    workerStartedAt: await waitForProcessStartedAt(child.pid),
+    workerStartedAt: child.startedAt,
   };
 }
 
 async function failQueuedPromotion(targetJobId, message, at = new Date().toISOString()) {
   await mutateAnyJob(targetJobId, (latest) => {
     if (["complete", "failed", "cancelled"].includes(String(latest.status || ""))) return latest;
-    return {
-      ...latest,
-      ...phasePatch("failed", {
-        status: "failed",
-        completedAt: at,
+    return transitionOracleJobPhase(latest, "failed", {
+      at,
+      source: "oracle:worker-cleanup-promotion",
+      message: `Queued promotion failed: ${message}`,
+      patch: {
         heartbeatAt: at,
         error: message,
-      }, at),
-    };
+      },
+    });
   }).catch(() => undefined);
 }
 
 async function promoteQueuedJobsAfterCleanup() {
   await withLock(ORACLE_STATE_DIR, "admission", "global", { processPid: process.pid, source: "worker_cleanup_promoter", jobId }, async () => {
-    for (const queuedJob of listQueuedJobs()) {
-      const current = readAnyJob(queuedJob.id);
-      if (!current || current.status !== "queued") continue;
-
-      let spawnedWorker;
-      const promotedAt = new Date().toISOString();
-      if (!existsSync(current.archivePath)) {
-        await failQueuedPromotion(current.id, `Queued oracle archive is missing: ${current.archivePath}`, promotedAt);
-        continue;
-      }
-      const runtimeLeaseAcquired = await tryAcquireRuntimeLeaseForJob(current, promotedAt);
-      if (!runtimeLeaseAcquired) break;
-
-      const conversationLeaseAcquired = await tryAcquireConversationLeaseForJob(current, promotedAt);
-      if (!conversationLeaseAcquired) {
-        await releaseLease(ORACLE_STATE_DIR, "runtime", current.runtimeId).catch(() => undefined);
-        continue;
-      }
-
-      try {
-        await mutateAnyJob(current.id, (latest) => {
+    await runQueuedJobPromotionPass({
+      listQueuedJobs,
+      refreshJob: (targetJobId) => readAnyJob(targetJobId),
+      readLatestJob: (targetJobId) => readAnyJob(targetJobId),
+      acquireRuntimeLease: async (job, at) => tryAcquireRuntimeLeaseForJob(job, at),
+      acquireConversationLease: async (job, at) => tryAcquireConversationLeaseForJob(job, at),
+      releaseRuntimeLease: async (job) => {
+        await releaseLease(ORACLE_STATE_DIR, "runtime", job.runtimeId);
+      },
+      markSubmitted: async (job, at) => {
+        await mutateAnyJob(job.id, (latest) => {
           if (latest.status !== "queued") throw new Error(`Queued job ${latest.id} changed state during cleanup promotion (${latest.status})`);
-          return {
-            ...latest,
-            ...phasePatch("submitted", {
-              status: "submitted",
-              submittedAt: latest.submittedAt || promotedAt,
-            }, promotedAt),
-          };
+          return transitionOracleJobPhase(latest, "submitted", {
+            at,
+            source: "oracle:worker-cleanup-promotion",
+            message: "Queued job admitted after runtime cleanup released capacity.",
+            patch: {
+              submittedAt: latest.submittedAt || at,
+            },
+          });
         });
-
-        spawnedWorker = await spawnDetachedWorker(current.id);
-        await mutateAnyJob(current.id, (latest) => {
+      },
+      spawnWorker: async (job) => spawnDetachedWorker(job.id),
+      persistWorker: async (job, spawnedWorker) => {
+        await mutateAnyJob(job.id, (latest) => {
           if (hasDurableWorkerHandoff(latest)) {
             return {
               ...latest,
@@ -511,44 +402,42 @@ async function promoteQueuedJobsAfterCleanup() {
             workerStartedAt: spawnedWorker.workerStartedAt,
           };
         });
-      } catch (error) {
-        const latest = readAnyJob(current.id);
-        if (hasDurableWorkerHandoff(latest)) {
-          await log(`Queued promotion handoff already durable for ${current.id}; leaving active job intact`).catch(() => undefined);
-          continue;
-        }
-        if (spawnedWorker) {
-          await terminateWorkerPid(spawnedWorker.pid, spawnedWorker.workerStartedAt).catch(() => undefined);
-        }
-        const failedAt = new Date().toISOString();
-        if (latest && !["complete", "failed", "cancelled"].includes(String(latest.status || ""))) {
-          await failQueuedPromotion(current.id, error instanceof Error ? error.message : String(error), failedAt);
-        }
+      },
+      hasDurableWorkerHandoff,
+      isTerminalJob: (job) => ["complete", "failed", "cancelled"].includes(String(job.status || "")),
+      failQueuedPromotion: async (job, message, at) => failQueuedPromotion(job.id, message, at),
+      terminateSpawnedWorker: async (spawnedWorker) => {
+        await terminateWorkerPid(spawnedWorker.pid, spawnedWorker.workerStartedAt);
+      },
+      cleanupAfterFailure: async ({ job, at, spawnedWorker }) => {
         if (spawnedWorker) {
           let cleanupWarnings = [];
           try {
-            cleanupWarnings = await cleanupRuntime(current);
+            cleanupWarnings = await cleanupRuntime(job);
           } catch (cleanupError) {
-            const message = `Cleanup-driven promotion teardown warning for ${current.id}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
+            const message = `Cleanup-driven promotion teardown warning for ${job.id}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
             cleanupWarnings = [message];
             await log(message).catch(() => undefined);
           }
           if (cleanupWarnings.length > 0) {
-            await mutateAnyJob(current.id, (job) => ({
-              ...job,
-              cleanupWarnings: [...(job.cleanupWarnings || []), ...cleanupWarnings],
-              lastCleanupAt: failedAt,
-              error: [job.error, ...cleanupWarnings].filter(Boolean).join("\n"),
+            await mutateAnyJob(job.id, (current) => applyOracleJobCleanupWarnings(current, cleanupWarnings, {
+              at,
+              source: "oracle:worker-cleanup-promotion",
+              message: `Cleanup-driven queued promotion teardown left ${cleanupWarnings.length} warning(s).`,
             })).catch(() => undefined);
-            await log(`Stopping queued cleanup promotion after ${current.id} because teardown left ${cleanupWarnings.length} warning(s)`).catch(() => undefined);
-            break;
+            await log(`Stopping queued cleanup promotion after ${job.id} because teardown left ${cleanupWarnings.length} warning(s)`).catch(() => undefined);
+            return "break";
           }
-        } else {
-          await releaseLease(ORACLE_STATE_DIR, "conversation", current.conversationId).catch(() => undefined);
-          await releaseLease(ORACLE_STATE_DIR, "runtime", current.runtimeId).catch(() => undefined);
+          return;
         }
-      }
-    }
+
+        await releaseLease(ORACLE_STATE_DIR, "conversation", job.conversationId).catch(() => undefined);
+        await releaseLease(ORACLE_STATE_DIR, "runtime", job.runtimeId).catch(() => undefined);
+      },
+      onDurableHandoff: async (job) => {
+        await log(`Queued promotion handoff already durable for ${job.id}; leaving active job intact`).catch(() => undefined);
+      },
+    });
   }).catch(async (error) => {
     await log(`Queued cleanup promotion warning: ${error instanceof Error ? error.message : String(error)}`).catch(() => undefined);
   });
@@ -671,14 +560,7 @@ async function currentUrl(job) {
 }
 
 function stripQuery(url) {
-  try {
-    const parsed = new URL(url);
-    parsed.hash = "";
-    parsed.search = "";
-    return parsed.toString();
-  } catch {
-    return url;
-  }
+  return stripUrlQueryAndHash(url);
 }
 
 async function snapshotText(job) {
@@ -1319,46 +1201,17 @@ async function assistantMessages(job) {
   return result.messages.map((message) => ({ text: typeof message?.text === "string" ? message.text : "" }));
 }
 
-function assistantSnapshotSlice(snapshot, responseIndex) {
-  const lines = snapshot.split("\n");
-  const assistantHeadingIndices = lines.flatMap((line, index) => (line.includes('heading "ChatGPT said:"') ? [index] : []));
-  const startIndex = assistantHeadingIndices[responseIndex];
-  if (startIndex === undefined) return undefined;
-
-  const endCandidates = [];
-  const nextAssistantIndex = assistantHeadingIndices[responseIndex + 1];
-  if (nextAssistantIndex !== undefined) endCandidates.push(nextAssistantIndex);
-
-  const composerIndex = lines.findIndex(
-    (line, index) => index > startIndex && line.includes(`textbox "${CHATGPT_LABELS.composer}"`),
-  );
-  if (composerIndex !== -1) endCandidates.push(composerIndex);
-
-  const endIndex = endCandidates.length > 0 ? Math.min(...endCandidates) : undefined;
-  return lines.slice(startIndex, endIndex).join("\n");
-}
-
 async function waitForStableChatUrl(job, previousChatUrl) {
   const timeoutAt = Date.now() + 60_000;
-  let lastUrl = "";
-  let stableCount = 0;
+  /** @type {import("./chatgpt-flow-helpers.d.mts").OracleStableValueState | undefined} */
+  let stableState;
 
   while (Date.now() < timeoutAt) {
     await heartbeat();
-    const url = stripQuery(await currentUrl(job));
-    let isConversationUrl = false;
-    try {
-      isConversationUrl = /\/c\/[A-Za-z0-9-]+$/i.test(new URL(url).pathname);
-    } catch {
-      isConversationUrl = false;
-    }
-    const isKnownFollowUpUrl = previousChatUrl ? stripQuery(previousChatUrl) === url : false;
-
-    if (isConversationUrl || isKnownFollowUpUrl) {
-      if (url === lastUrl) stableCount += 1;
-      else stableCount = 1;
-      lastUrl = url;
-      if (stableCount >= 2) return url;
+    const candidateUrl = resolveStableConversationUrlCandidate(await currentUrl(job), previousChatUrl);
+    if (candidateUrl) {
+      stableState = nextStableValueState(stableState, candidateUrl);
+      if (stableState.stableCount >= 2) return candidateUrl;
     }
 
     await sleep(1000);
@@ -1459,7 +1312,7 @@ function preferredArtifactName(label, index) {
 
 async function collectArtifactCandidates(job, responseIndex, responseText = "") {
   const snapshot = await snapshotText(job);
-  const targetSlice = assistantSnapshotSlice(snapshot, responseIndex);
+  const targetSlice = assistantSnapshotSlice(snapshot, CHATGPT_LABELS.composer, responseIndex);
   if (!targetSlice) return { snapshot, targetSlice, candidates: [], suspiciousLabels: [] };
 
   const structural = await evalPage(
@@ -1762,19 +1615,44 @@ async function run() {
 
   try {
     await log(`Starting oracle worker for job ${currentJob.id}`);
-    await heartbeat(phasePatch("cloning_runtime", { status: "waiting" }), { force: true });
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "cloning_runtime", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Cloning the auth seed profile into the isolated runtime.",
+      patch: { heartbeatAt: new Date().toISOString() },
+    }));
     await closeBrowser(currentJob);
 
     const seedGeneration = await cloneSeedProfileToRuntime(currentJob);
-    currentJob = await mutateJob((job) => ({ ...job, ...phasePatch("launching_browser", { seedGeneration, heartbeatAt: new Date().toISOString() }) }));
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "launching_browser", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Launching the isolated oracle browser runtime.",
+      patch: { seedGeneration, heartbeatAt: new Date().toISOString() },
+    }));
 
     const targetUrl = currentJob.chatUrl || currentJob.config.browser.chatUrl;
     await launchBrowser(currentJob, targetUrl);
-    currentJob = await mutateJob((job) => ({ ...job, ...phasePatch("verifying_auth", { heartbeatAt: new Date().toISOString() }) }));
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "verifying_auth", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Verifying the imported ChatGPT auth session.",
+      patch: { heartbeatAt: new Date().toISOString() },
+    }));
     await waitForOracleReady(currentJob);
-    currentJob = await mutateJob((job) => ({ ...job, ...phasePatch("configuring_model", { heartbeatAt: new Date().toISOString() }) }));
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "configuring_model", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Configuring the requested ChatGPT model selection.",
+      patch: { heartbeatAt: new Date().toISOString() },
+    }));
     await configureModel(currentJob);
-    currentJob = await mutateJob((job) => ({ ...job, ...phasePatch("uploading_archive", { heartbeatAt: new Date().toISOString() }) }));
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "uploading_archive", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Uploading the oracle context archive.",
+      patch: { heartbeatAt: new Date().toISOString() },
+    }));
     await uploadArchive(currentJob);
     await setComposerText(currentJob, await readFile(currentJob.promptPath, "utf8"));
     const baselineAssistantCount = (await assistantMessages(currentJob)).length;
@@ -1785,30 +1663,44 @@ async function run() {
 
     const chatUrl = await waitForStableChatUrl(currentJob, currentJob.chatUrl);
     const conversationId = parseConversationId(chatUrl) || currentJob.conversationId;
-    currentJob = await mutateJob((job) => ({
-      ...job,
-      ...phasePatch("awaiting_response", { chatUrl, conversationId, heartbeatAt: new Date().toISOString() }),
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "awaiting_response", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Waiting for the assistant response to finish streaming.",
+      patch: { chatUrl, conversationId, heartbeatAt: new Date().toISOString() },
     }));
 
     const completion = await waitForChatCompletion(currentJob, baselineAssistantCount);
-    currentJob = await mutateJob((job) => ({ ...job, ...phasePatch("extracting_response", { heartbeatAt: new Date().toISOString() }) }));
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "extracting_response", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Extracting the completed response body.",
+      patch: { heartbeatAt: new Date().toISOString() },
+    }));
     await secureWriteText(currentJob.responsePath, `${completion.responseText.trim()}\n`);
-    currentJob = await mutateJob((job) => ({ ...job, ...phasePatch("downloading_artifacts", { heartbeatAt: new Date().toISOString() }) }));
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "downloading_artifacts", {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: "Downloading any response artifacts.",
+      patch: { heartbeatAt: new Date().toISOString() },
+    }));
     const artifacts = await downloadArtifacts(currentJob, completion.responseIndex, completion.responseText);
     const artifactFailureCount = artifacts.filter((artifact) => artifact.unconfirmed || artifact.error).length;
     const finalPhase = artifactFailureCount > 0 ? "complete_with_artifact_errors" : "complete";
 
-    await heartbeat(
-      phasePatch(finalPhase, {
-        status: "complete",
-        completedAt: new Date().toISOString(),
+    currentJob = await mutateJob((job) => transitionOracleJobPhase(job, finalPhase, {
+      at: new Date().toISOString(),
+      source: "oracle:worker",
+      message: artifactFailureCount > 0
+        ? `Job completed with ${artifactFailureCount} artifact issue(s).`
+        : "Job completed successfully.",
+      patch: {
         responsePath: currentJob.responsePath,
         responseFormat: "text/plain",
         artifactFailureCount,
         cleanupPending: true,
-      }),
-      { force: true },
-    );
+      },
+    }));
     const persistedJob = await readJob().catch(() => undefined);
     await log(`Persisted final status after completion write: ${persistedJob?.status || "unknown"}`);
     await log(`Job ${currentJob.id} complete (${finalPhase}, artifact failures=${artifactFailureCount})`);
@@ -1817,15 +1709,15 @@ async function run() {
       const message = error instanceof Error ? error.message : String(error);
       await captureDiagnostics(currentJob, "failure");
       await log(`Job failed: ${message}`);
-      await heartbeat(
-        phasePatch("failed", {
-          status: "failed",
-          completedAt: new Date().toISOString(),
+      currentJob = await mutateJob((job) => transitionOracleJobPhase(job, "failed", {
+        at: new Date().toISOString(),
+        source: "oracle:worker",
+        message: `Job failed: ${message}`,
+        patch: {
           error: message,
           cleanupPending: true,
-        }),
-        { force: true },
-      );
+        },
+      }));
       process.exitCode = 1;
     }
   } finally {
@@ -1838,17 +1730,17 @@ async function run() {
     }
     if (currentJob?.id) {
       const cleanupAt = new Date().toISOString();
-      await mutateJob((job) => ({
-        ...job,
-        cleanupPending: false,
-        ...(cleanupWarnings.length > 0
-          ? {
-              cleanupWarnings: [...(job.cleanupWarnings || []), ...cleanupWarnings],
-              lastCleanupAt: cleanupAt,
-              error: [job.error, ...cleanupWarnings].filter(Boolean).join("\n"),
-            }
-          : { lastCleanupAt: cleanupAt }),
-      })).catch(() => undefined);
+      await mutateJob((job) => cleanupWarnings.length > 0
+        ? applyOracleJobCleanupWarnings(job, cleanupWarnings, {
+          at: cleanupAt,
+          source: "oracle:worker",
+          message: `Runtime cleanup completed with ${cleanupWarnings.length} warning(s).`,
+        })
+        : clearOracleJobCleanupState(job, {
+          at: cleanupAt,
+          source: "oracle:worker",
+          message: "Runtime cleanup finished without warnings.",
+        })).catch(() => undefined);
     }
     if (cleanupWarnings.length === 0) {
       await promoteQueuedJobsAfterCleanup().catch(() => undefined);