pi-oracle 0.3.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +27 -8
- package/docs/ORACLE_DESIGN.md +14 -8
- package/docs/ORACLE_ISOLATED_PI_VALIDATION.md +276 -0
- package/extensions/oracle/index.ts +8 -1
- package/extensions/oracle/lib/commands.ts +25 -29
- package/extensions/oracle/lib/config.ts +56 -2
- package/extensions/oracle/lib/jobs.ts +134 -219
- package/extensions/oracle/lib/locks.ts +41 -209
- package/extensions/oracle/lib/poller.ts +38 -52
- package/extensions/oracle/lib/queue.ts +75 -112
- package/extensions/oracle/lib/runtime.ts +102 -19
- package/extensions/oracle/lib/tools.ts +663 -294
- package/extensions/oracle/shared/job-coordination-helpers.d.mts +84 -0
- package/extensions/oracle/shared/job-coordination-helpers.mjs +168 -0
- package/extensions/oracle/shared/job-lifecycle-helpers.d.mts +131 -0
- package/extensions/oracle/shared/job-lifecycle-helpers.mjs +390 -0
- package/extensions/oracle/shared/job-observability-helpers.d.mts +60 -0
- package/extensions/oracle/shared/job-observability-helpers.mjs +161 -0
- package/extensions/oracle/shared/process-helpers.d.mts +20 -0
- package/extensions/oracle/shared/process-helpers.mjs +128 -0
- package/extensions/oracle/shared/state-coordination-helpers.d.mts +43 -0
- package/extensions/oracle/shared/state-coordination-helpers.mjs +381 -0
- package/extensions/oracle/worker/artifact-heuristics.mjs +5 -0
- package/extensions/oracle/worker/auth-bootstrap.mjs +125 -134
- package/extensions/oracle/worker/auth-cookie-policy.mjs +5 -0
- package/extensions/oracle/worker/auth-flow-helpers.d.mts +41 -0
- package/extensions/oracle/worker/auth-flow-helpers.mjs +165 -0
- package/extensions/oracle/worker/chatgpt-flow-helpers.d.mts +13 -0
- package/extensions/oracle/worker/chatgpt-flow-helpers.mjs +85 -0
- package/extensions/oracle/worker/chatgpt-ui-helpers.mjs +93 -9
- package/extensions/oracle/worker/run-job.mjs +166 -274
- package/extensions/oracle/worker/state-locks.mjs +31 -216
- package/package.json +4 -3
- package/prompts/oracle.md +16 -10
|
@@ -1,35 +1,42 @@
|
|
|
1
|
+
// Purpose: Manage durable oracle job state, lifecycle transitions, reconciliation, and worker orchestration from the extension side.
|
|
2
|
+
// Responsibilities: Persist job metadata, detect stale workers, coordinate notification/wake-up state, and spawn or terminate worker processes safely.
|
|
3
|
+
// Scope: Extension-facing job management only; low-level shared process and state primitives live in extensions/oracle/shared.
|
|
4
|
+
// Usage: Imported by oracle commands, tools, queue logic, poller flows, and runtime cleanup/reconciliation paths.
|
|
5
|
+
// Invariants/Assumptions: Job mutations happen under per-job locks, worker identity checks defend against PID reuse, and persisted jobs remain the source of truth.
|
|
1
6
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
3
|
-
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
7
|
+
import { existsSync, readdirSync, readFileSync, realpathSync } from "node:fs";
|
|
4
8
|
import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
5
|
-
import { join, resolve } from "node:path";
|
|
9
|
+
import { isAbsolute, join, relative as relativePath, resolve, sep } from "node:path";
|
|
6
10
|
import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
11
|
+
import {
|
|
12
|
+
ACTIVE_ORACLE_JOB_STATUSES,
|
|
13
|
+
applyOracleJobCleanupWarnings,
|
|
14
|
+
claimOracleJobNotification,
|
|
15
|
+
clearOracleJobCleanupState,
|
|
16
|
+
getOracleJobStatusForPhase,
|
|
17
|
+
markOracleJobCreated,
|
|
18
|
+
markOracleJobNotified,
|
|
19
|
+
markOracleJobWakeupSettled,
|
|
20
|
+
noteOracleJobWakeupRequested,
|
|
21
|
+
OPEN_ORACLE_JOB_STATUSES,
|
|
22
|
+
recordOracleJobNotificationTarget,
|
|
23
|
+
releaseOracleJobNotificationClaim,
|
|
24
|
+
TERMINAL_ORACLE_JOB_STATUSES,
|
|
25
|
+
transitionOracleJobPhase,
|
|
26
|
+
} from "../shared/job-lifecycle-helpers.mjs";
|
|
27
|
+
import type { OracleJobLifecycleEvent as SharedOracleJobLifecycleEvent, OracleJobPhase as SharedOracleJobPhase, OracleJobStatus as SharedOracleJobStatus } from "../shared/job-lifecycle-helpers.mjs";
|
|
28
|
+
import { hasDurableWorkerHandoff as sharedHasDurableWorkerHandoff } from "../shared/job-coordination-helpers.mjs";
|
|
29
|
+
import { isTrackedProcessAlive, readProcessStartedAt, spawnDetachedNodeProcess, terminateTrackedProcess } from "../shared/process-helpers.mjs";
|
|
7
30
|
import type { OracleConfig, OracleResolvedSelection } from "./config.js";
|
|
8
31
|
import { withJobLock, withLock } from "./locks.js";
|
|
9
32
|
import { cleanupRuntimeArtifacts, getProjectId, getSessionId, parseConversationId, requirePersistedSessionFile, type OracleCleanupReport } from "./runtime.js";
|
|
10
33
|
|
|
11
|
-
export type OracleJobStatus =
|
|
12
|
-
export type OracleJobPhase =
|
|
13
|
-
| "queued"
|
|
14
|
-
| "submitted"
|
|
15
|
-
| "cloning_runtime"
|
|
16
|
-
| "launching_browser"
|
|
17
|
-
| "verifying_auth"
|
|
18
|
-
| "configuring_model"
|
|
19
|
-
| "uploading_archive"
|
|
20
|
-
| "awaiting_response"
|
|
21
|
-
| "extracting_response"
|
|
22
|
-
| "downloading_artifacts"
|
|
23
|
-
| "complete"
|
|
24
|
-
| "complete_with_artifact_errors"
|
|
25
|
-
| "failed"
|
|
26
|
-
| "cancelled";
|
|
34
|
+
export type OracleJobStatus = SharedOracleJobStatus;
|
|
35
|
+
export type OracleJobPhase = SharedOracleJobPhase;
|
|
27
36
|
|
|
28
37
|
export type OracleWakeupSettlementSource = "oracle_read" | "oracle_status";
|
|
29
38
|
|
|
30
|
-
export
|
|
31
|
-
export const OPEN_ORACLE_JOB_STATUSES: OracleJobStatus[] = ["queued", ...ACTIVE_ORACLE_JOB_STATUSES];
|
|
32
|
-
export const TERMINAL_ORACLE_JOB_STATUSES: OracleJobStatus[] = ["complete", "failed", "cancelled"];
|
|
39
|
+
export { ACTIVE_ORACLE_JOB_STATUSES, OPEN_ORACLE_JOB_STATUSES, TERMINAL_ORACLE_JOB_STATUSES };
|
|
33
40
|
export const ORACLE_MISSING_WORKER_GRACE_MS = 30_000;
|
|
34
41
|
export const ORACLE_STALE_HEARTBEAT_MS = 3 * 60 * 1000;
|
|
35
42
|
export const ORACLE_NOTIFICATION_CLAIM_TTL_MS = 60_000;
|
|
@@ -65,9 +72,7 @@ export function hasRetainedPreSubmitArchive(job: Pick<OracleJob, "submittedAt" |
|
|
|
65
72
|
export function hasDurableWorkerHandoff(
|
|
66
73
|
job: Pick<OracleJob, "status" | "phase" | "workerPid" | "workerStartedAt" | "heartbeatAt">,
|
|
67
74
|
): boolean {
|
|
68
|
-
|
|
69
|
-
if (job.workerPid) return true;
|
|
70
|
-
return false;
|
|
75
|
+
return sharedHasDurableWorkerHandoff(job);
|
|
71
76
|
}
|
|
72
77
|
|
|
73
78
|
export function hasPersistedOriginSession(
|
|
@@ -76,30 +81,8 @@ export function hasPersistedOriginSession(
|
|
|
76
81
|
return typeof job.originSessionFile === "string" && job.originSessionFile.length > 0 && job.sessionId === job.originSessionFile;
|
|
77
82
|
}
|
|
78
83
|
|
|
79
|
-
function readProcessStartedAt(pid: number | undefined): string | undefined {
|
|
80
|
-
if (!pid || pid <= 0) return undefined;
|
|
81
|
-
try {
|
|
82
|
-
const startedAt = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8" }).trim();
|
|
83
|
-
return startedAt || undefined;
|
|
84
|
-
} catch {
|
|
85
|
-
return undefined;
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
async function waitForProcessStartedAt(pid: number | undefined, timeoutMs = 2_000): Promise<string | undefined> {
|
|
90
|
-
const deadline = Date.now() + timeoutMs;
|
|
91
|
-
while (Date.now() < deadline) {
|
|
92
|
-
const startedAt = readProcessStartedAt(pid);
|
|
93
|
-
if (startedAt) return startedAt;
|
|
94
|
-
await sleep(100);
|
|
95
|
-
}
|
|
96
|
-
return readProcessStartedAt(pid);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
84
|
export function isWorkerProcessAlive(pid: number | undefined, startedAt?: string): boolean {
|
|
100
|
-
|
|
101
|
-
if (!currentStartedAt) return false;
|
|
102
|
-
return startedAt ? currentStartedAt === startedAt : true;
|
|
85
|
+
return isTrackedProcessAlive(pid, startedAt);
|
|
103
86
|
}
|
|
104
87
|
|
|
105
88
|
export interface OracleArtifactRecord {
|
|
@@ -178,6 +161,7 @@ export interface OracleJob {
|
|
|
178
161
|
cleanupWarnings?: string[];
|
|
179
162
|
lastCleanupAt?: string;
|
|
180
163
|
cleanupPending?: boolean;
|
|
164
|
+
lifecycleEvents?: SharedOracleJobLifecycleEvent[];
|
|
181
165
|
}
|
|
182
166
|
|
|
183
167
|
export interface OracleSubmitInput {
|
|
@@ -267,13 +251,7 @@ export async function updateJob(id: string, mutate: (job: OracleJob) => OracleJo
|
|
|
267
251
|
export async function appendCleanupWarnings(jobId: string, warnings: string[], at = new Date().toISOString()): Promise<OracleJob | undefined> {
|
|
268
252
|
if (warnings.length === 0) return readJob(jobId);
|
|
269
253
|
try {
|
|
270
|
-
return await updateJob(jobId, (job) => ({
|
|
271
|
-
...job,
|
|
272
|
-
cleanupPending: false,
|
|
273
|
-
cleanupWarnings: Array.from(new Set([...(job.cleanupWarnings || []), ...warnings])),
|
|
274
|
-
lastCleanupAt: at,
|
|
275
|
-
error: [job.error, ...warnings].filter(Boolean).join("\n"),
|
|
276
|
-
}));
|
|
254
|
+
return await updateJob(jobId, (job) => applyOracleJobCleanupWarnings(job, warnings, { at, source: "oracle:jobs" }));
|
|
277
255
|
} catch {
|
|
278
256
|
return readJob(jobId);
|
|
279
257
|
}
|
|
@@ -281,21 +259,12 @@ export async function appendCleanupWarnings(jobId: string, warnings: string[], a
|
|
|
281
259
|
|
|
282
260
|
export async function clearCleanupPending(jobId: string, at = new Date().toISOString()): Promise<OracleJob | undefined> {
|
|
283
261
|
try {
|
|
284
|
-
return await updateJob(jobId, (job) => ({
|
|
285
|
-
...job,
|
|
286
|
-
cleanupPending: false,
|
|
287
|
-
cleanupWarnings: undefined,
|
|
288
|
-
lastCleanupAt: at,
|
|
289
|
-
}));
|
|
262
|
+
return await updateJob(jobId, (job) => clearOracleJobCleanupState(job, { at, source: "oracle:jobs" }));
|
|
290
263
|
} catch {
|
|
291
264
|
return readJob(jobId);
|
|
292
265
|
}
|
|
293
266
|
}
|
|
294
267
|
|
|
295
|
-
function sleep(ms: number): Promise<void> {
|
|
296
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
297
|
-
}
|
|
298
|
-
|
|
299
268
|
function parseTimestamp(value: string | undefined): number | undefined {
|
|
300
269
|
if (!value) return undefined;
|
|
301
270
|
const parsed = Date.parse(value);
|
|
@@ -316,6 +285,16 @@ function notificationClaimIsLive(job: Pick<OracleJob, "notifyClaimedAt" | "notif
|
|
|
316
285
|
return now - claimedAtMs < ORACLE_NOTIFICATION_CLAIM_TTL_MS;
|
|
317
286
|
}
|
|
318
287
|
|
|
288
|
+
function getWakeupRetentionGraceDeadline(job: Pick<OracleJob, "wakeupLastRequestedAt">, now = Date.now()): { retryAt: string; remainingMs: number } | undefined {
|
|
289
|
+
const lastRequestedAtMs = parseTimestamp(job.wakeupLastRequestedAt);
|
|
290
|
+
if (lastRequestedAtMs === undefined) return undefined;
|
|
291
|
+
const retryAtMs = lastRequestedAtMs + ORACLE_WAKEUP_POST_SEND_RETENTION_MS;
|
|
292
|
+
return {
|
|
293
|
+
retryAt: new Date(retryAtMs).toISOString(),
|
|
294
|
+
remainingMs: Math.max(0, retryAtMs - now),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
319
298
|
function wakeupRetentionGraceIsActive(job: Pick<OracleJob, "wakeupLastRequestedAt">, now = Date.now()): boolean {
|
|
320
299
|
const lastRequestedAtMs = parseTimestamp(job.wakeupLastRequestedAt);
|
|
321
300
|
if (lastRequestedAtMs === undefined) return false;
|
|
@@ -335,13 +314,14 @@ export function shouldRequestWakeup(job: Pick<OracleJob, "wakeupAttemptCount" |
|
|
|
335
314
|
return now - lastRequestedAtMs >= getWakeupRetryDelayMs(attempts);
|
|
336
315
|
}
|
|
337
316
|
|
|
338
|
-
export function withJobPhase<T extends Pick<OracleJob, "phase" | "phaseAt">>(
|
|
317
|
+
export function withJobPhase<T extends Pick<OracleJob, "phase" | "phaseAt" | "status">>(
|
|
339
318
|
phase: OracleJobPhase,
|
|
340
319
|
patch?: Omit<Partial<OracleJob>, "phase" | "phaseAt">,
|
|
341
320
|
at = new Date().toISOString(),
|
|
342
321
|
): Partial<OracleJob> {
|
|
343
322
|
return {
|
|
344
323
|
...(patch || {}),
|
|
324
|
+
status: (patch?.status as OracleJobStatus | undefined) ?? getOracleJobStatusForPhase(phase),
|
|
345
325
|
phase,
|
|
346
326
|
phaseAt: at,
|
|
347
327
|
};
|
|
@@ -356,39 +336,7 @@ export async function terminateWorkerPid(
|
|
|
356
336
|
startedAt?: string,
|
|
357
337
|
options?: { termGraceMs?: number; killGraceMs?: number },
|
|
358
338
|
): Promise<boolean> {
|
|
359
|
-
|
|
360
|
-
const currentStartedAt = readProcessStartedAt(pid);
|
|
361
|
-
if (!currentStartedAt) return true;
|
|
362
|
-
if (startedAt && currentStartedAt !== startedAt) return false;
|
|
363
|
-
|
|
364
|
-
const termGraceMs = options?.termGraceMs ?? 5000;
|
|
365
|
-
const killGraceMs = options?.killGraceMs ?? 2000;
|
|
366
|
-
|
|
367
|
-
try {
|
|
368
|
-
process.kill(pid, "SIGTERM");
|
|
369
|
-
} catch {
|
|
370
|
-
return !isWorkerProcessAlive(pid, startedAt);
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
const termDeadline = Date.now() + termGraceMs;
|
|
374
|
-
while (Date.now() < termDeadline) {
|
|
375
|
-
if (!isWorkerProcessAlive(pid, startedAt)) return true;
|
|
376
|
-
await sleep(250);
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
try {
|
|
380
|
-
process.kill(pid, "SIGKILL");
|
|
381
|
-
} catch {
|
|
382
|
-
return !isWorkerProcessAlive(pid, startedAt);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
const killDeadline = Date.now() + killGraceMs;
|
|
386
|
-
while (Date.now() < killDeadline) {
|
|
387
|
-
if (!isWorkerProcessAlive(pid, startedAt)) return true;
|
|
388
|
-
await sleep(250);
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
return !isWorkerProcessAlive(pid, startedAt);
|
|
339
|
+
return terminateTrackedProcess(pid, startedAt, options);
|
|
392
340
|
}
|
|
393
341
|
|
|
394
342
|
export function getStaleOracleJobReason(job: OracleJob, now = Date.now()): string | undefined {
|
|
@@ -526,12 +474,17 @@ export async function removeTerminalOracleJob(job: OracleJob): Promise<{ removed
|
|
|
526
474
|
},
|
|
527
475
|
};
|
|
528
476
|
}
|
|
529
|
-
|
|
477
|
+
const nowMs = Date.now();
|
|
478
|
+
if (wakeupRetentionGraceIsActive(current, nowMs)) {
|
|
479
|
+
const graceDeadline = getWakeupRetentionGraceDeadline(current, nowMs);
|
|
480
|
+
const retryHint = graceDeadline
|
|
481
|
+
? ` Retry after ${graceDeadline.retryAt} (${Math.ceil(graceDeadline.remainingMs / 1000)}s remaining).`
|
|
482
|
+
: "";
|
|
530
483
|
return {
|
|
531
484
|
removed: false,
|
|
532
485
|
cleanupReport: {
|
|
533
486
|
attempted: [],
|
|
534
|
-
warnings: [`Refusing to remove terminal oracle job ${current.id} because its wake-up delivery is still within the post-send retention grace window
|
|
487
|
+
warnings: [`Refusing to remove terminal oracle job ${current.id} because its wake-up delivery is still within the post-send retention grace window.${retryHint}`],
|
|
535
488
|
},
|
|
536
489
|
};
|
|
537
490
|
}
|
|
@@ -547,13 +500,11 @@ export async function removeTerminalOracleJob(job: OracleJob): Promise<{ removed
|
|
|
547
500
|
|
|
548
501
|
const cleanupReport = await cleanupJobResources(current);
|
|
549
502
|
if (cleanupReport.warnings.length > 0) {
|
|
550
|
-
await writeJobUnlocked({
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
error: [current.error, ...cleanupReport.warnings].filter(Boolean).join("\n"),
|
|
556
|
-
});
|
|
503
|
+
await writeJobUnlocked(applyOracleJobCleanupWarnings(current, cleanupReport.warnings, {
|
|
504
|
+
at: new Date().toISOString(),
|
|
505
|
+
source: "oracle:cleanup",
|
|
506
|
+
message: "Terminal job cleanup completed with warnings during removal.",
|
|
507
|
+
}));
|
|
557
508
|
return { removed: false, cleanupReport };
|
|
558
509
|
}
|
|
559
510
|
await rm(getJobDir(current.id), { recursive: true, force: true });
|
|
@@ -644,20 +595,19 @@ export async function reconcileStaleOracleJobs(): Promise<OracleJob[]> {
|
|
|
644
595
|
? ` Terminated stale worker PID ${current.workerPid}.`
|
|
645
596
|
: ` Failed to terminate stale worker PID ${current.workerPid}.`
|
|
646
597
|
: "";
|
|
647
|
-
repairedJob = {
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
598
|
+
repairedJob = transitionOracleJobPhase(current, "failed", {
|
|
599
|
+
at: recoveredAt,
|
|
600
|
+
source: "oracle:reconcile",
|
|
601
|
+
message: `Recovered stale job: ${currentStaleReason}.${suffix}`.trim(),
|
|
602
|
+
clearNotificationClaim: true,
|
|
603
|
+
patch: {
|
|
652
604
|
heartbeatAt: recoveredAt,
|
|
653
|
-
notifyClaimedAt: undefined,
|
|
654
|
-
notifyClaimedBy: undefined,
|
|
655
605
|
cleanupPending: terminated,
|
|
656
606
|
error: current.error
|
|
657
607
|
? `${current.error}\nRecovered stale job: ${currentStaleReason}.${suffix}`.trim()
|
|
658
608
|
: `Recovered stale job: ${currentStaleReason}.${suffix}`.trim(),
|
|
659
|
-
},
|
|
660
|
-
};
|
|
609
|
+
},
|
|
610
|
+
});
|
|
661
611
|
await writeJobUnlocked(repairedJob);
|
|
662
612
|
});
|
|
663
613
|
|
|
@@ -710,11 +660,7 @@ export async function tryClaimNotification(jobId: string, claimedBy: string, now
|
|
|
710
660
|
Date.now() - claimedAtMs < ORACLE_NOTIFICATION_CLAIM_TTL_MS;
|
|
711
661
|
if (claimIsLive) return undefined;
|
|
712
662
|
|
|
713
|
-
const next
|
|
714
|
-
...current,
|
|
715
|
-
notifyClaimedBy: claimedBy,
|
|
716
|
-
notifyClaimedAt: now,
|
|
717
|
-
};
|
|
663
|
+
const next = claimOracleJobNotification(current, claimedBy, now);
|
|
718
664
|
await writeJobUnlocked(next);
|
|
719
665
|
return next;
|
|
720
666
|
});
|
|
@@ -732,11 +678,12 @@ export async function recordNotificationTarget(
|
|
|
732
678
|
if (!notificationClaimIsOwnedBy(current, claimedBy)) {
|
|
733
679
|
throw new Error(`Oracle notification claim is not owned by ${claimedBy}: ${jobId}`);
|
|
734
680
|
}
|
|
735
|
-
const next
|
|
736
|
-
|
|
681
|
+
const next = recordOracleJobNotificationTarget(current, {
|
|
682
|
+
at: new Date().toISOString(),
|
|
683
|
+
source: "oracle:poller",
|
|
737
684
|
notificationSessionKey: options.notificationSessionKey,
|
|
738
685
|
notificationSessionFile: options.notificationSessionFile,
|
|
739
|
-
};
|
|
686
|
+
});
|
|
740
687
|
await writeJobUnlocked(next);
|
|
741
688
|
return next;
|
|
742
689
|
});
|
|
@@ -755,18 +702,13 @@ export async function markJobNotified(
|
|
|
755
702
|
if (!notificationClaimIsOwnedBy(current, claimedBy)) {
|
|
756
703
|
throw new Error(`Oracle notification claim is not owned by ${claimedBy}: ${jobId}`);
|
|
757
704
|
}
|
|
758
|
-
const next
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
notificationEntryId: options?.notificationEntryId
|
|
762
|
-
notificationSessionKey: options?.notificationSessionKey
|
|
763
|
-
notificationSessionFile: options?.notificationSessionFile
|
|
764
|
-
|
|
765
|
-
wakeupLastRequestedAt: undefined,
|
|
766
|
-
wakeupSettledAt: undefined,
|
|
767
|
-
notifyClaimedAt: undefined,
|
|
768
|
-
notifyClaimedBy: undefined,
|
|
769
|
-
};
|
|
705
|
+
const next = markOracleJobNotified(current, {
|
|
706
|
+
at,
|
|
707
|
+
source: "oracle:poller",
|
|
708
|
+
notificationEntryId: options?.notificationEntryId,
|
|
709
|
+
notificationSessionKey: options?.notificationSessionKey,
|
|
710
|
+
notificationSessionFile: options?.notificationSessionFile,
|
|
711
|
+
});
|
|
770
712
|
await writeJobUnlocked(next);
|
|
771
713
|
return next;
|
|
772
714
|
});
|
|
@@ -777,11 +719,7 @@ export async function releaseNotificationClaim(jobId: string, claimedBy: string)
|
|
|
777
719
|
const current = readJob(jobId);
|
|
778
720
|
if (!current) return undefined;
|
|
779
721
|
if (current.notifyClaimedBy && current.notifyClaimedBy !== claimedBy) return current;
|
|
780
|
-
const next
|
|
781
|
-
...current,
|
|
782
|
-
notifyClaimedAt: undefined,
|
|
783
|
-
notifyClaimedBy: undefined,
|
|
784
|
-
};
|
|
722
|
+
const next = releaseOracleJobNotificationClaim(current);
|
|
785
723
|
await writeJobUnlocked(next);
|
|
786
724
|
return next;
|
|
787
725
|
});
|
|
@@ -789,11 +727,7 @@ export async function releaseNotificationClaim(jobId: string, claimedBy: string)
|
|
|
789
727
|
|
|
790
728
|
export async function noteWakeupRequested(jobId: string, at = new Date().toISOString()): Promise<OracleJob | undefined> {
|
|
791
729
|
try {
|
|
792
|
-
return await updateJob(jobId, (job) => ({
|
|
793
|
-
...job,
|
|
794
|
-
wakeupAttemptCount: (job.wakeupAttemptCount ?? 0) + 1,
|
|
795
|
-
wakeupLastRequestedAt: at,
|
|
796
|
-
}));
|
|
730
|
+
return await updateJob(jobId, (job) => noteOracleJobWakeupRequested(job, { at, source: "oracle:poller" }));
|
|
797
731
|
} catch {
|
|
798
732
|
return readJob(jobId);
|
|
799
733
|
}
|
|
@@ -819,37 +753,13 @@ export async function markWakeupSettled(
|
|
|
819
753
|
const sessionKey = getWakeupSessionKey(options.sessionFile, options.cwd);
|
|
820
754
|
|
|
821
755
|
try {
|
|
822
|
-
return await updateJob(jobId, (job) => {
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
wakeupSettledSessionKey: job.wakeupSettledSessionKey ?? sessionKey,
|
|
830
|
-
wakeupSettledBeforeFirstAttempt: job.wakeupSettledBeforeFirstAttempt ?? beforeFirstAttempt,
|
|
831
|
-
};
|
|
832
|
-
}
|
|
833
|
-
|
|
834
|
-
if (beforeFirstAttempt && !options.allowBeforeFirstAttempt) {
|
|
835
|
-
return {
|
|
836
|
-
...job,
|
|
837
|
-
wakeupObservedAt: job.wakeupObservedAt ?? at,
|
|
838
|
-
wakeupObservedSource: job.wakeupObservedSource ?? options.source,
|
|
839
|
-
wakeupObservedSessionFile: job.wakeupObservedSessionFile ?? options.sessionFile,
|
|
840
|
-
wakeupObservedSessionKey: job.wakeupObservedSessionKey ?? sessionKey,
|
|
841
|
-
};
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
return {
|
|
845
|
-
...job,
|
|
846
|
-
wakeupSettledAt: at,
|
|
847
|
-
wakeupSettledSource: options.source,
|
|
848
|
-
wakeupSettledSessionFile: options.sessionFile,
|
|
849
|
-
wakeupSettledSessionKey: sessionKey,
|
|
850
|
-
wakeupSettledBeforeFirstAttempt: beforeFirstAttempt,
|
|
851
|
-
};
|
|
852
|
-
});
|
|
756
|
+
return await updateJob(jobId, (job) => markOracleJobWakeupSettled(job, {
|
|
757
|
+
source: options.source,
|
|
758
|
+
at,
|
|
759
|
+
sessionFile: options.sessionFile,
|
|
760
|
+
sessionKey,
|
|
761
|
+
allowBeforeFirstAttempt: options.allowBeforeFirstAttempt,
|
|
762
|
+
}));
|
|
853
763
|
} catch {
|
|
854
764
|
return readJob(jobId);
|
|
855
765
|
}
|
|
@@ -863,26 +773,24 @@ export async function cancelOracleJob(id: string, reason = "Cancelled by user"):
|
|
|
863
773
|
|
|
864
774
|
const now = new Date().toISOString();
|
|
865
775
|
if (current.status === "queued") {
|
|
866
|
-
const cancelled = await updateJob(id, (job) => ({
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
776
|
+
const cancelled = await updateJob(id, (job) => transitionOracleJobPhase(job, "cancelled", {
|
|
777
|
+
at: now,
|
|
778
|
+
source: "oracle:cancel",
|
|
779
|
+
message: `Job cancelled: ${reason}`,
|
|
780
|
+
clearNotificationClaim: true,
|
|
781
|
+
patch: {
|
|
871
782
|
heartbeatAt: now,
|
|
872
|
-
notifyClaimedAt: undefined,
|
|
873
|
-
notifyClaimedBy: undefined,
|
|
874
783
|
error: reason,
|
|
875
|
-
},
|
|
784
|
+
},
|
|
876
785
|
}));
|
|
877
786
|
|
|
878
787
|
const cleanupReport = await cleanupJobResources(cancelled);
|
|
879
788
|
if (cleanupReport.warnings.length === 0) return cancelled;
|
|
880
789
|
|
|
881
|
-
return updateJob(id, (job) => ({
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
error: [job.error, ...cleanupReport.warnings].filter(Boolean).join("\n"),
|
|
790
|
+
return updateJob(id, (job) => applyOracleJobCleanupWarnings(job, cleanupReport.warnings, {
|
|
791
|
+
at: now,
|
|
792
|
+
source: "oracle:cancel",
|
|
793
|
+
message: "Queued job cleanup completed with warnings after cancellation.",
|
|
886
794
|
}));
|
|
887
795
|
}
|
|
888
796
|
|
|
@@ -891,18 +799,19 @@ export async function cancelOracleJob(id: string, reason = "Cancelled by user"):
|
|
|
891
799
|
const cancelled = await updateJob(id, (job) => {
|
|
892
800
|
if (isTerminalOracleJob(job)) return job;
|
|
893
801
|
transitioned = true;
|
|
894
|
-
return {
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
802
|
+
return transitionOracleJobPhase(job, terminated ? "cancelled" : "failed", {
|
|
803
|
+
at: now,
|
|
804
|
+
source: "oracle:cancel",
|
|
805
|
+
message: terminated
|
|
806
|
+
? `Job cancelled: ${reason}`
|
|
807
|
+
: `Job cancellation failed because worker PID ${job.workerPid ?? "unknown"} did not exit.`,
|
|
808
|
+
clearNotificationClaim: true,
|
|
809
|
+
patch: {
|
|
899
810
|
heartbeatAt: now,
|
|
900
|
-
notifyClaimedAt: undefined,
|
|
901
|
-
notifyClaimedBy: undefined,
|
|
902
811
|
cleanupPending: terminated,
|
|
903
812
|
error: terminated ? reason : `${reason}; worker PID ${job.workerPid ?? "unknown"} did not exit`,
|
|
904
|
-
},
|
|
905
|
-
};
|
|
813
|
+
},
|
|
814
|
+
});
|
|
906
815
|
});
|
|
907
816
|
if (!transitioned) return cancelled;
|
|
908
817
|
|
|
@@ -910,11 +819,10 @@ export async function cancelOracleJob(id: string, reason = "Cancelled by user"):
|
|
|
910
819
|
const cleanupWarnings = [
|
|
911
820
|
`Oracle runtime cleanup is blocked because worker PID ${current.workerPid ?? "unknown"} could not be terminated safely.`,
|
|
912
821
|
];
|
|
913
|
-
return updateJob(id, (job) => ({
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
error: [job.error, ...cleanupWarnings].filter(Boolean).join("\n"),
|
|
822
|
+
return updateJob(id, (job) => applyOracleJobCleanupWarnings(job, cleanupWarnings, {
|
|
823
|
+
at: now,
|
|
824
|
+
source: "oracle:cancel",
|
|
825
|
+
message: "Runtime cleanup remained blocked after cancellation.",
|
|
918
826
|
}));
|
|
919
827
|
}
|
|
920
828
|
|
|
@@ -924,12 +832,10 @@ export async function cancelOracleJob(id: string, reason = "Cancelled by user"):
|
|
|
924
832
|
return finalized ?? cancelled;
|
|
925
833
|
}
|
|
926
834
|
|
|
927
|
-
return updateJob(id, (job) => ({
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
lastCleanupAt: now,
|
|
932
|
-
error: [job.error, ...cleanupReport.warnings].filter(Boolean).join("\n"),
|
|
835
|
+
return updateJob(id, (job) => applyOracleJobCleanupWarnings(job, cleanupReport.warnings, {
|
|
836
|
+
at: now,
|
|
837
|
+
source: "oracle:cancel",
|
|
838
|
+
message: "Runtime cleanup completed with warnings after cancellation.",
|
|
933
839
|
}));
|
|
934
840
|
});
|
|
935
841
|
}
|
|
@@ -967,7 +873,7 @@ export async function createJob(
|
|
|
967
873
|
|
|
968
874
|
const createdAt = options?.createdAt ?? new Date().toISOString();
|
|
969
875
|
const initialState = options?.initialState ?? "submitted";
|
|
970
|
-
const job
|
|
876
|
+
const job = markOracleJobCreated({
|
|
971
877
|
id,
|
|
972
878
|
status: initialState,
|
|
973
879
|
phase: initialState,
|
|
@@ -999,17 +905,27 @@ export async function createJob(
|
|
|
999
905
|
runtimeProfileDir: runtime.runtimeProfileDir,
|
|
1000
906
|
seedGeneration: runtime.seedGeneration,
|
|
1001
907
|
config,
|
|
1002
|
-
}
|
|
908
|
+
} satisfies OracleJob, {
|
|
909
|
+
at: createdAt,
|
|
910
|
+
source: "oracle:create",
|
|
911
|
+
message: initialState === "queued" ? "Job created and queued for later admission." : "Job created and admitted for worker launch.",
|
|
912
|
+
});
|
|
1003
913
|
|
|
1004
914
|
await writeJob(job);
|
|
1005
915
|
return job;
|
|
1006
916
|
}
|
|
1007
917
|
|
|
918
|
+
function isPathInsideDirectory(rootPath: string, candidatePath: string): boolean {
|
|
919
|
+
const boundary = relativePath(rootPath, candidatePath);
|
|
920
|
+
return boundary === "" || (!boundary.startsWith(`..${sep}`) && boundary !== ".." && !isAbsolute(boundary));
|
|
921
|
+
}
|
|
922
|
+
|
|
1008
923
|
export function resolveArchiveInputs(cwd: string, files: string[]): { absolute: string; relative: string }[] {
|
|
1009
924
|
if (files.length === 0) {
|
|
1010
925
|
throw new Error("oracle_submit requires at least one file or directory to archive");
|
|
1011
926
|
}
|
|
1012
927
|
|
|
928
|
+
const realCwd = realpathSync(cwd);
|
|
1013
929
|
return files.map((file) => {
|
|
1014
930
|
const absolute = resolve(cwd, file);
|
|
1015
931
|
const relative = absolute.startsWith(`${cwd}/`) ? absolute.slice(cwd.length + 1) : absolute === cwd ? "." : "";
|
|
@@ -1019,6 +935,9 @@ export function resolveArchiveInputs(cwd: string, files: string[]): { absolute:
|
|
|
1019
935
|
if (!existsSync(absolute)) {
|
|
1020
936
|
throw new Error(`Archive input does not exist: ${file}`);
|
|
1021
937
|
}
|
|
938
|
+
if (!isPathInsideDirectory(realCwd, realpathSync(absolute))) {
|
|
939
|
+
throw new Error(`Archive input must resolve inside the project cwd without symlink escapes: ${file}`);
|
|
940
|
+
}
|
|
1022
941
|
return { absolute, relative };
|
|
1023
942
|
});
|
|
1024
943
|
}
|
|
@@ -1028,14 +947,10 @@ export async function spawnWorker(
|
|
|
1028
947
|
jobId: string,
|
|
1029
948
|
): Promise<{ pid: number | undefined; nonce: string; startedAt: string | undefined }> {
|
|
1030
949
|
const nonce = randomUUID();
|
|
1031
|
-
const child =
|
|
1032
|
-
detached: true,
|
|
1033
|
-
stdio: "ignore",
|
|
1034
|
-
});
|
|
1035
|
-
child.unref();
|
|
950
|
+
const child = await spawnDetachedNodeProcess(workerPath, [jobId, nonce]);
|
|
1036
951
|
return {
|
|
1037
952
|
pid: child.pid,
|
|
1038
953
|
nonce,
|
|
1039
|
-
startedAt:
|
|
954
|
+
startedAt: child.startedAt,
|
|
1040
955
|
};
|
|
1041
956
|
}
|