pi-worker-graph 0.1.0-dev.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +455 -0
- package/SECURITY.md +54 -0
- package/dist/config.d.ts +25 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +181 -0
- package/dist/config.js.map +1 -0
- package/dist/context.d.ts +22 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +81 -0
- package/dist/context.js.map +1 -0
- package/dist/coordination.d.ts +19 -0
- package/dist/coordination.d.ts.map +1 -0
- package/dist/coordination.js +267 -0
- package/dist/coordination.js.map +1 -0
- package/dist/execution-failure.d.ts +42 -0
- package/dist/execution-failure.d.ts.map +1 -0
- package/dist/execution-failure.js +90 -0
- package/dist/execution-failure.js.map +1 -0
- package/dist/extension.d.ts +8 -0
- package/dist/extension.d.ts.map +1 -0
- package/dist/extension.js +641 -0
- package/dist/extension.js.map +1 -0
- package/dist/graph.d.ts +44 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/graph.js +292 -0
- package/dist/graph.js.map +1 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/json.d.ts +9 -0
- package/dist/json.d.ts.map +1 -0
- package/dist/json.js +66 -0
- package/dist/json.js.map +1 -0
- package/dist/orchestrator.d.ts +15 -0
- package/dist/orchestrator.d.ts.map +1 -0
- package/dist/orchestrator.js +473 -0
- package/dist/orchestrator.js.map +1 -0
- package/dist/output.d.ts +61 -0
- package/dist/output.d.ts.map +1 -0
- package/dist/output.js +248 -0
- package/dist/output.js.map +1 -0
- package/dist/pi-subprocess.d.ts +92 -0
- package/dist/pi-subprocess.d.ts.map +1 -0
- package/dist/pi-subprocess.js +897 -0
- package/dist/pi-subprocess.js.map +1 -0
- package/dist/run.d.ts +89 -0
- package/dist/run.d.ts.map +1 -0
- package/dist/run.js +562 -0
- package/dist/run.js.map +1 -0
- package/dist/store.d.ts +331 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +1993 -0
- package/dist/store.js.map +1 -0
- package/dist/usage.d.ts +35 -0
- package/dist/usage.d.ts.map +1 -0
- package/dist/usage.js +88 -0
- package/dist/usage.js.map +1 -0
- package/docs/DECISIONS.md +221 -0
- package/docs/DESIGN.md +392 -0
- package/docs/NEXT.md +229 -0
- package/docs/PLAN.md +203 -0
- package/docs/worker-graph.example.json +12 -0
- package/extensions/index.ts +1 -0
- package/extensions/tsconfig.json +11 -0
- package/package.json +66 -0
package/dist/store.js
ADDED
|
@@ -0,0 +1,1993 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { chmod, link, mkdir, open, readdir, rename, rm, } from "node:fs/promises";
|
|
3
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
|
+
import { normalizeGraph, setNodeStatus, settleBlocked } from "./graph.js";
|
|
5
|
+
import { isJsonValue, isRecord } from "./json.js";
|
|
6
|
+
import { NODE_OUTPUT_LIMITS, parseNodeArtifact, parseNodeDiagnostics, parseNodeOutput, } from "./output.js";
|
|
7
|
+
import { parseTaskUsage } from "./usage.js";
|
|
8
|
+
export const RUN_STORE_MAX_RECORD_BYTES = 1024 * 1024;
|
|
9
|
+
export const RUN_STORE_DEFAULT_MAX_RUNS = 64;
|
|
10
|
+
export const RUN_STORE_MAX_RUNS = 256;
|
|
11
|
+
export const RUN_COORDINATION_MAX_TEXT_BYTES = 16 * 1024;
|
|
12
|
+
/** Bound for one path or symbol, matching the report path bound. */
|
|
13
|
+
export const RUN_COORDINATION_MAX_ITEM_BYTES = 4 * 1024;
|
|
14
|
+
export const RUN_COORDINATION_MAX_ITEMS = 16;
|
|
15
|
+
/** Retained coordination records per run. */
|
|
16
|
+
export const RUN_COORDINATION_MAX_RECORDS = 256;
|
|
17
|
+
/** Bound for one requested coordination page. */
|
|
18
|
+
export const RUN_COORDINATION_MAX_READ = 256;
|
|
19
|
+
/**
|
|
20
|
+
* Serialized bound for one coordination record, checked when a record is
|
|
21
|
+
* published and again when it is read, so the field bounds above cannot
|
|
22
|
+
* combine into a record that alone exceeds a page.
|
|
23
|
+
*/
|
|
24
|
+
export const RUN_COORDINATION_MAX_RECORD_BYTES = 32 * 1024;
|
|
25
|
+
/**
|
|
26
|
+
* Serialized bound for one returned coordination page. A page stops before the
|
|
27
|
+
* first record that would exceed it and reports a cursor, and no single record
|
|
28
|
+
* can reach it, so a returned page never exceeds this.
|
|
29
|
+
*/
|
|
30
|
+
export const RUN_COORDINATION_MAX_PAGE_BYTES = 64 * 1024;
|
|
31
|
+
/**
|
|
32
|
+
* Length of a coordination record identifier, which is also a read cursor.
|
|
33
|
+
* Identifiers are fixed-width so that sorting them sorts the sequence, and the
|
|
34
|
+
* child tool schemas bound their cursor parameter with this.
|
|
35
|
+
*/
|
|
36
|
+
export const RUN_COORDINATION_ID_LENGTH = 6;
|
|
37
|
+
const MUTATION_LOCK_WAIT_MS = 250;
|
|
38
|
+
const MUTATION_LOCK_POLL_MS = 5;
|
|
39
|
+
const SCHEMA_VERSION = 1;
|
|
40
|
+
const DIRECTORY_MODE = 0o700;
|
|
41
|
+
const FILE_MODE = 0o600;
|
|
42
|
+
const RUN_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
43
|
+
const RUN_SLOT_DIRECTORY = "slots";
|
|
44
|
+
const RUN_SLOT_RECORD = "slot.json";
|
|
45
|
+
const RUN_SLOT_PATTERN = /^(0|[1-9][0-9]*)\.json$/;
|
|
46
|
+
const RUN_MUTATION_LOCK = "mutation.lock";
|
|
47
|
+
const RUN_COORDINATION_DIRECTORY = "coordination";
|
|
48
|
+
const RUN_ARTIFACT_DIRECTORY = "artifacts";
|
|
49
|
+
const RECORD_SUFFIX = ".json";
|
|
50
|
+
const COORDINATION_ID_PATTERN = new RegExp(`^[0-9]{${RUN_COORDINATION_ID_LENGTH}}$`);
|
|
51
|
+
const MAX_COORDINATION_SEQUENCE = 10 ** RUN_COORDINATION_ID_LENGTH - 1;
|
|
52
|
+
const COORDINATION_PUBLISH_ATTEMPTS = 16;
|
|
53
|
+
const TASK_KEY_PATTERN = /^task-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
54
|
+
const NODE_STATUSES = new Set([
|
|
55
|
+
"pending",
|
|
56
|
+
"running",
|
|
57
|
+
"succeeded",
|
|
58
|
+
"failed",
|
|
59
|
+
"aborted",
|
|
60
|
+
"blocked",
|
|
61
|
+
]);
|
|
62
|
+
const OUTPUT_STATUSES = new Set([
|
|
63
|
+
"succeeded",
|
|
64
|
+
"failed",
|
|
65
|
+
"aborted",
|
|
66
|
+
]);
|
|
67
|
+
const NODE_OUTPUT_RECORD_FIELDS = new Set([
|
|
68
|
+
"schemaVersion",
|
|
69
|
+
"kind",
|
|
70
|
+
"runId",
|
|
71
|
+
"taskId",
|
|
72
|
+
"taskKey",
|
|
73
|
+
"attempt",
|
|
74
|
+
"status",
|
|
75
|
+
"completedAt",
|
|
76
|
+
"output",
|
|
77
|
+
"artifact",
|
|
78
|
+
"usage",
|
|
79
|
+
"diagnostics",
|
|
80
|
+
]);
|
|
81
|
+
const NODE_ARTIFACT_RECORD_FIELDS = new Set([
|
|
82
|
+
"schemaVersion",
|
|
83
|
+
"kind",
|
|
84
|
+
"runId",
|
|
85
|
+
"taskId",
|
|
86
|
+
"taskKey",
|
|
87
|
+
"attempt",
|
|
88
|
+
"publishedAt",
|
|
89
|
+
"text",
|
|
90
|
+
]);
|
|
91
|
+
const RUN_OWNER_RECORD_FIELDS = new Set([
|
|
92
|
+
"schemaVersion",
|
|
93
|
+
"kind",
|
|
94
|
+
"runId",
|
|
95
|
+
"ownerId",
|
|
96
|
+
"acquiredAt",
|
|
97
|
+
]);
|
|
98
|
+
const RUN_MUTATION_LOCK_FIELDS = new Set([
|
|
99
|
+
"schemaVersion",
|
|
100
|
+
"kind",
|
|
101
|
+
"runId",
|
|
102
|
+
"taskId",
|
|
103
|
+
]);
|
|
104
|
+
const RUN_EVENT_RECORD_FIELDS = new Set([
|
|
105
|
+
"schemaVersion",
|
|
106
|
+
"kind",
|
|
107
|
+
"eventId",
|
|
108
|
+
"runId",
|
|
109
|
+
"taskId",
|
|
110
|
+
"eventKind",
|
|
111
|
+
"timestamp",
|
|
112
|
+
"message",
|
|
113
|
+
"paths",
|
|
114
|
+
"symbols",
|
|
115
|
+
"recipients",
|
|
116
|
+
]);
|
|
117
|
+
const RUN_MESSAGE_RECORD_FIELDS = new Set([
|
|
118
|
+
"schemaVersion",
|
|
119
|
+
"kind",
|
|
120
|
+
"messageId",
|
|
121
|
+
"runId",
|
|
122
|
+
"senderTaskId",
|
|
123
|
+
"recipientTaskId",
|
|
124
|
+
"timestamp",
|
|
125
|
+
"message",
|
|
126
|
+
]);
|
|
127
|
+
const RUN_EVENT_KINDS = new Set([
|
|
128
|
+
"decision",
|
|
129
|
+
"interface",
|
|
130
|
+
"risk",
|
|
131
|
+
"conflict",
|
|
132
|
+
"handoff",
|
|
133
|
+
"progress",
|
|
134
|
+
]);
|
|
135
|
+
export class RunStoreError extends Error {
|
|
136
|
+
code;
|
|
137
|
+
recordPath;
|
|
138
|
+
constructor(code, message, recordPath) {
|
|
139
|
+
super(message);
|
|
140
|
+
this.name = "RunStoreError";
|
|
141
|
+
this.code = code;
|
|
142
|
+
this.recordPath = recordPath;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
function errorCode(error) {
|
|
146
|
+
if (typeof error === "object" &&
|
|
147
|
+
error !== null &&
|
|
148
|
+
"code" in error &&
|
|
149
|
+
typeof error.code === "string") {
|
|
150
|
+
return error.code;
|
|
151
|
+
}
|
|
152
|
+
return undefined;
|
|
153
|
+
}
|
|
154
|
+
function stateRootPath(stateRoot) {
|
|
155
|
+
if (stateRoot.trim().length === 0) {
|
|
156
|
+
throw new RunStoreError("invalid_argument", "The run store state root must not be empty");
|
|
157
|
+
}
|
|
158
|
+
const root = resolve(stateRoot);
|
|
159
|
+
if (dirname(root) === root) {
|
|
160
|
+
throw new RunStoreError("invalid_argument", "The run store state root must not be the filesystem root");
|
|
161
|
+
}
|
|
162
|
+
return root;
|
|
163
|
+
}
|
|
164
|
+
async function publishedRunIds(runs) {
|
|
165
|
+
const entries = await readdir(runs, { withFileTypes: true });
|
|
166
|
+
return new Set(entries
|
|
167
|
+
.filter((entry) => entry.isDirectory() && RUN_ID_PATTERN.test(entry.name))
|
|
168
|
+
.map((entry) => entry.name));
|
|
169
|
+
}
|
|
170
|
+
async function slotOwner(path) {
|
|
171
|
+
try {
|
|
172
|
+
const record = await readJson(path);
|
|
173
|
+
if (!isRecord(record) || typeof record.runId !== "string")
|
|
174
|
+
return undefined;
|
|
175
|
+
// The recorded owner is compared against directory names, so it must be a
|
|
176
|
+
// run ID and not a traversal written into the slot by anything else.
|
|
177
|
+
return RUN_ID_PATTERN.test(record.runId) ? record.runId : undefined;
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
async function readSlotClaims(slots) {
|
|
184
|
+
let entries;
|
|
185
|
+
try {
|
|
186
|
+
entries = await readdir(slots, { withFileTypes: true, encoding: "utf8" });
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
// No directory is no claims. A creator makes it before reading, so this
|
|
190
|
+
// is only reached by a reader looking at a store that holds no capacity.
|
|
191
|
+
if (errorCode(error) === "ENOENT")
|
|
192
|
+
return [];
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
return Promise.all(entries
|
|
196
|
+
.filter((entry) => entry.isFile() && RUN_SLOT_PATTERN.test(entry.name))
|
|
197
|
+
.map(async (entry) => {
|
|
198
|
+
const index = Number.parseInt(entry.name, 10);
|
|
199
|
+
const owner = await slotOwner(join(slots, entry.name));
|
|
200
|
+
return owner === undefined ? { index } : { index, owner };
|
|
201
|
+
}));
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Capacity is a fixed set of atomically claimed slot files, so the limit is
|
|
205
|
+
* structural rather than counted: at most `maximumRuns` slots can exist, so at
|
|
206
|
+
* most `maximumRuns` runs can publish. Concurrent creators are arbitrated by
|
|
207
|
+
* the filesystem — the lowest index whose link succeeds is that creator's — so
|
|
208
|
+
* an available slot is always claimed by exactly one of them rather than
|
|
209
|
+
* refused to all of them.
|
|
210
|
+
*
|
|
211
|
+
* Runs are read before slots. A creator links its slot before renaming its run
|
|
212
|
+
* into place, so every published run's slot is visible here and an in-flight
|
|
213
|
+
* creation can never be mistaken for a missing one. A published run that is
|
|
214
|
+
* genuinely unaccounted for means the two disagree, and the store then admits
|
|
215
|
+
* nobody rather than letting each creator claim the same free capacity.
|
|
216
|
+
*
|
|
217
|
+
* The structure only bounds what it can see: a run holding a slot at or above
|
|
218
|
+
* the current limit occupies none of the indices a claim searches, so lowering
|
|
219
|
+
* the limit below an assigned index would leave that run's capacity claimable
|
|
220
|
+
* a second time. Such a run is treated the same way as a missing slot — the
|
|
221
|
+
* store admits nobody until the numbering is reconciled.
|
|
222
|
+
*/
|
|
223
|
+
async function claimRunSlot(runs, runId, maximumRuns) {
|
|
224
|
+
const slots = join(runs, RUN_SLOT_DIRECTORY);
|
|
225
|
+
await mkdir(slots, { recursive: true, mode: DIRECTORY_MODE });
|
|
226
|
+
await chmod(slots, DIRECTORY_MODE);
|
|
227
|
+
const published = await publishedRunIds(runs);
|
|
228
|
+
const claims = await readSlotClaims(slots);
|
|
229
|
+
const owners = new Set(claims.flatMap((claim) => claim.owner ?? []));
|
|
230
|
+
const bounded = new Set(claims.flatMap((claim) => claim.index < maximumRuns ? (claim.owner ?? []) : []));
|
|
231
|
+
const beyondLimit = [...published].filter((id) => owners.has(id) && !bounded.has(id)).length;
|
|
232
|
+
if (beyondLimit > 0) {
|
|
233
|
+
throw new RunStoreError("retention_limit", `${beyondLimit} published run(s) hold a capacity slot at or above the current maximum of ${maximumRuns}; the run store admits no new work until the slot files in ${JSON.stringify(slots)} are renumbered below that maximum`, slots);
|
|
234
|
+
}
|
|
235
|
+
const unaccounted = [...published].filter((id) => !owners.has(id)).length;
|
|
236
|
+
if (unaccounted > 0) {
|
|
237
|
+
throw new RunStoreError("retention_limit", `${unaccounted} published run(s) have no capacity slot; the run store admits no new work until the run directories in ${JSON.stringify(runs)} and their slots agree`, slots);
|
|
238
|
+
}
|
|
239
|
+
if (published.size >= maximumRuns) {
|
|
240
|
+
throw new RunStoreError("retention_limit", `The run store retains ${published.size} of at most ${maximumRuns} runs; remove old run state before starting another graph`, runs);
|
|
241
|
+
}
|
|
242
|
+
const claimed = await linkFirstFreeSlot(slots, runId, maximumRuns);
|
|
243
|
+
if (claimed !== undefined)
|
|
244
|
+
return claimed;
|
|
245
|
+
const reclaimable = claims.filter((claim) => claim.index < maximumRuns &&
|
|
246
|
+
(claim.owner === undefined || !published.has(claim.owner))).length;
|
|
247
|
+
throw new RunStoreError("retention_limit", `All ${maximumRuns} run capacity slots are claimed; ${reclaimable} of them belong to runs that were never published and can be deleted from ${JSON.stringify(slots)}`, slots);
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* The claim is a hard link to a record that is already complete on disk, so a
|
|
251
|
+
* crash can never leave a slot that holds capacity without naming its owner.
|
|
252
|
+
*/
|
|
253
|
+
async function linkFirstFreeSlot(slots, runId, maximumRuns) {
|
|
254
|
+
const temporaryPath = await writeTemporaryFile(join(slots, RUN_SLOT_RECORD), {
|
|
255
|
+
schemaVersion: SCHEMA_VERSION,
|
|
256
|
+
kind: "run-slot",
|
|
257
|
+
runId,
|
|
258
|
+
});
|
|
259
|
+
try {
|
|
260
|
+
for (let index = 0; index < maximumRuns; index += 1) {
|
|
261
|
+
const path = join(slots, `${index}.json`);
|
|
262
|
+
try {
|
|
263
|
+
await link(temporaryPath, path);
|
|
264
|
+
return path;
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
if (errorCode(error) !== "EEXIST")
|
|
268
|
+
throw error;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
return undefined;
|
|
272
|
+
}
|
|
273
|
+
finally {
|
|
274
|
+
await rm(temporaryPath, { force: true });
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
async function releaseRunSlot(path, runId) {
|
|
278
|
+
if ((await slotOwner(path)) !== runId)
|
|
279
|
+
return;
|
|
280
|
+
await rm(path, { force: true });
|
|
281
|
+
}
|
|
282
|
+
function assertRunId(runId) {
|
|
283
|
+
if (!RUN_ID_PATTERN.test(runId)) {
|
|
284
|
+
throw new RunStoreError("invalid_identifier", `Invalid run ID ${JSON.stringify(runId)}`);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function runsPath(stateRoot) {
|
|
288
|
+
return join(stateRootPath(stateRoot), "runs");
|
|
289
|
+
}
|
|
290
|
+
function runPath(stateRoot, runId) {
|
|
291
|
+
assertRunId(runId);
|
|
292
|
+
return join(runsPath(stateRoot), runId);
|
|
293
|
+
}
|
|
294
|
+
function ownerPath(stateRoot, runId) {
|
|
295
|
+
return join(runPath(stateRoot, runId), "owner.json");
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Events and messages of a run share one journal directory, so a single
|
|
299
|
+
* exclusive create both claims a record's sequence number and stores the
|
|
300
|
+
* record. The directory is created with the run, so publishing is only ever
|
|
301
|
+
* one atomic link into an existing directory.
|
|
302
|
+
*/
|
|
303
|
+
function coordinationDirectory(stateRoot, runId) {
|
|
304
|
+
return join(runPath(stateRoot, runId), RUN_COORDINATION_DIRECTORY);
|
|
305
|
+
}
|
|
306
|
+
function recordId(fileName) {
|
|
307
|
+
return fileName.slice(0, -RECORD_SUFFIX.length);
|
|
308
|
+
}
|
|
309
|
+
function isTimestamp(value) {
|
|
310
|
+
return (typeof value === "string" &&
|
|
311
|
+
value.length > 0 &&
|
|
312
|
+
!Number.isNaN(Date.parse(value)));
|
|
313
|
+
}
|
|
314
|
+
function serializedRecord(value, recordPath) {
|
|
315
|
+
let serialized;
|
|
316
|
+
try {
|
|
317
|
+
serialized = `${JSON.stringify(value)}\n`;
|
|
318
|
+
}
|
|
319
|
+
catch {
|
|
320
|
+
throw new RunStoreError("invalid_record", "Record is not JSON serializable", recordPath);
|
|
321
|
+
}
|
|
322
|
+
if (Buffer.byteLength(serialized) > RUN_STORE_MAX_RECORD_BYTES) {
|
|
323
|
+
throw new RunStoreError("record_too_large", `Record exceeds the ${RUN_STORE_MAX_RECORD_BYTES} byte limit`, recordPath);
|
|
324
|
+
}
|
|
325
|
+
return serialized;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* `durable` flushes the record before it is linked into place, which is what
|
|
329
|
+
* every stored record needs. The mutation lock is the exception: it is not
|
|
330
|
+
* state to recover, and flushing it would only lengthen the critical section
|
|
331
|
+
* that the orchestrator and its workers contend for.
|
|
332
|
+
*/
|
|
333
|
+
async function writeNewFile(path, value, durable = true) {
|
|
334
|
+
const contents = serializedRecord(value, path);
|
|
335
|
+
const handle = await open(path, "wx", FILE_MODE);
|
|
336
|
+
try {
|
|
337
|
+
await handle.writeFile(contents, "utf8");
|
|
338
|
+
if (durable)
|
|
339
|
+
await handle.sync();
|
|
340
|
+
}
|
|
341
|
+
finally {
|
|
342
|
+
await handle.close();
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
async function writeTemporaryFile(finalPath, value, durable = true) {
|
|
346
|
+
const temporaryPath = join(dirname(finalPath), `.${basename(finalPath)}.${randomUUID()}.tmp`);
|
|
347
|
+
try {
|
|
348
|
+
await writeNewFile(temporaryPath, value, durable);
|
|
349
|
+
return temporaryPath;
|
|
350
|
+
}
|
|
351
|
+
catch (error) {
|
|
352
|
+
await rm(temporaryPath, { force: true });
|
|
353
|
+
throw error;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
async function replaceRecord(path, value) {
|
|
357
|
+
const temporaryPath = await writeTemporaryFile(path, value);
|
|
358
|
+
try {
|
|
359
|
+
await rename(temporaryPath, path);
|
|
360
|
+
}
|
|
361
|
+
finally {
|
|
362
|
+
await rm(temporaryPath, { force: true });
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
async function publishRecord(path, value, durable = true) {
|
|
366
|
+
const temporaryPath = await writeTemporaryFile(path, value, durable);
|
|
367
|
+
try {
|
|
368
|
+
await link(temporaryPath, path);
|
|
369
|
+
}
|
|
370
|
+
catch (error) {
|
|
371
|
+
if (errorCode(error) === "EEXIST") {
|
|
372
|
+
throw new RunStoreError("record_exists", `Immutable record already exists at ${JSON.stringify(path)}`, path);
|
|
373
|
+
}
|
|
374
|
+
throw error;
|
|
375
|
+
}
|
|
376
|
+
finally {
|
|
377
|
+
await rm(temporaryPath, { force: true });
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
async function readJson(path) {
|
|
381
|
+
let handle;
|
|
382
|
+
try {
|
|
383
|
+
handle = await open(path, "r");
|
|
384
|
+
}
|
|
385
|
+
catch (error) {
|
|
386
|
+
if (errorCode(error) === "ENOENT") {
|
|
387
|
+
throw new RunStoreError("not_found", `Record not found at ${JSON.stringify(path)}`, path);
|
|
388
|
+
}
|
|
389
|
+
throw error;
|
|
390
|
+
}
|
|
391
|
+
try {
|
|
392
|
+
const metadata = await handle.stat();
|
|
393
|
+
if (metadata.size > RUN_STORE_MAX_RECORD_BYTES) {
|
|
394
|
+
throw new RunStoreError("record_too_large", `Record exceeds the ${RUN_STORE_MAX_RECORD_BYTES} byte limit`, path);
|
|
395
|
+
}
|
|
396
|
+
const buffer = Buffer.alloc(RUN_STORE_MAX_RECORD_BYTES + 1);
|
|
397
|
+
let length = 0;
|
|
398
|
+
while (length <= RUN_STORE_MAX_RECORD_BYTES) {
|
|
399
|
+
const { bytesRead } = await handle.read(buffer, length, buffer.length - length, length);
|
|
400
|
+
if (bytesRead === 0)
|
|
401
|
+
break;
|
|
402
|
+
length += bytesRead;
|
|
403
|
+
}
|
|
404
|
+
if (length > RUN_STORE_MAX_RECORD_BYTES) {
|
|
405
|
+
throw new RunStoreError("record_too_large", `Record exceeds the ${RUN_STORE_MAX_RECORD_BYTES} byte limit`, path);
|
|
406
|
+
}
|
|
407
|
+
try {
|
|
408
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, length));
|
|
409
|
+
return JSON.parse(text);
|
|
410
|
+
}
|
|
411
|
+
catch {
|
|
412
|
+
throw new RunStoreError("malformed_record", `Record at ${JSON.stringify(path)} is not valid UTF-8 JSON`, path);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
finally {
|
|
416
|
+
await handle.close();
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
function invalidRecord(path, detail) {
|
|
420
|
+
throw new RunStoreError("invalid_record", `Invalid record at ${JSON.stringify(path)}: ${detail}`, path);
|
|
421
|
+
}
|
|
422
|
+
function validateRunOwnership(value, path) {
|
|
423
|
+
if (!isRecord(value) ||
|
|
424
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
425
|
+
value.kind !== "run-owner" ||
|
|
426
|
+
typeof value.runId !== "string" ||
|
|
427
|
+
!RUN_ID_PATTERN.test(value.runId) ||
|
|
428
|
+
typeof value.ownerId !== "string" ||
|
|
429
|
+
!RUN_ID_PATTERN.test(value.ownerId) ||
|
|
430
|
+
!isTimestamp(value.acquiredAt) ||
|
|
431
|
+
Object.keys(value).some((field) => !RUN_OWNER_RECORD_FIELDS.has(field))) {
|
|
432
|
+
return invalidRecord(path, "run ownership does not match schema version 1");
|
|
433
|
+
}
|
|
434
|
+
return { runId: value.runId, ownerId: value.ownerId };
|
|
435
|
+
}
|
|
436
|
+
function validateRunMutationLock(value, path) {
|
|
437
|
+
if (!isRecord(value) ||
|
|
438
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
439
|
+
value.kind !== "run-mutation-lock" ||
|
|
440
|
+
typeof value.runId !== "string" ||
|
|
441
|
+
!RUN_ID_PATTERN.test(value.runId) ||
|
|
442
|
+
(value.taskId !== undefined && typeof value.taskId !== "string") ||
|
|
443
|
+
Object.keys(value).some((field) => !RUN_MUTATION_LOCK_FIELDS.has(field))) {
|
|
444
|
+
return invalidRecord(path, "run mutation lock does not match schema version 1");
|
|
445
|
+
}
|
|
446
|
+
return {
|
|
447
|
+
schemaVersion: SCHEMA_VERSION,
|
|
448
|
+
kind: "run-mutation-lock",
|
|
449
|
+
runId: value.runId,
|
|
450
|
+
...(value.taskId === undefined ? {} : { taskId: value.taskId }),
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
function parseOwnershipCapability(value) {
|
|
454
|
+
try {
|
|
455
|
+
if (!isRecord(value))
|
|
456
|
+
throw new Error("invalid ownership capability");
|
|
457
|
+
const prototype = Object.getPrototypeOf(value);
|
|
458
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
459
|
+
throw new Error("invalid ownership capability");
|
|
460
|
+
}
|
|
461
|
+
const keys = Reflect.ownKeys(value);
|
|
462
|
+
if (keys.length !== 2 ||
|
|
463
|
+
keys.some((key) => key !== "runId" && key !== "ownerId")) {
|
|
464
|
+
throw new Error("invalid ownership capability");
|
|
465
|
+
}
|
|
466
|
+
const runIdDescriptor = Object.getOwnPropertyDescriptor(value, "runId");
|
|
467
|
+
const ownerIdDescriptor = Object.getOwnPropertyDescriptor(value, "ownerId");
|
|
468
|
+
if (runIdDescriptor === undefined ||
|
|
469
|
+
!runIdDescriptor.enumerable ||
|
|
470
|
+
!("value" in runIdDescriptor) ||
|
|
471
|
+
ownerIdDescriptor === undefined ||
|
|
472
|
+
!ownerIdDescriptor.enumerable ||
|
|
473
|
+
!("value" in ownerIdDescriptor) ||
|
|
474
|
+
typeof runIdDescriptor.value !== "string" ||
|
|
475
|
+
!RUN_ID_PATTERN.test(runIdDescriptor.value) ||
|
|
476
|
+
typeof ownerIdDescriptor.value !== "string" ||
|
|
477
|
+
!RUN_ID_PATTERN.test(ownerIdDescriptor.value)) {
|
|
478
|
+
throw new Error("invalid ownership capability");
|
|
479
|
+
}
|
|
480
|
+
return Object.freeze({
|
|
481
|
+
runId: runIdDescriptor.value,
|
|
482
|
+
ownerId: ownerIdDescriptor.value,
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
catch {
|
|
486
|
+
throw new RunStoreError("ownership", "Run ownership capability is invalid");
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* Takes the run mutation lock, naming `taskId` as its holder when a worker
|
|
491
|
+
* publication takes it rather than the orchestrator.
|
|
492
|
+
*
|
|
493
|
+
* The lock is claimed by hard-linking a record that is already complete on
|
|
494
|
+
* disk, so the lock never exists without naming who holds it and a waiter
|
|
495
|
+
* always has something to attribute contention to. Waiting out the deadline
|
|
496
|
+
* fails with `locked`, which is contention and not an ownership conflict: only
|
|
497
|
+
* a caller that can show the named holder has finished may recover the lock.
|
|
498
|
+
*/
|
|
499
|
+
async function acquireRunMutationLock(stateRoot, runId, taskId) {
|
|
500
|
+
const path = join(runPath(stateRoot, runId), RUN_MUTATION_LOCK);
|
|
501
|
+
const record = {
|
|
502
|
+
schemaVersion: SCHEMA_VERSION,
|
|
503
|
+
kind: "run-mutation-lock",
|
|
504
|
+
runId,
|
|
505
|
+
...(taskId === undefined ? {} : { taskId }),
|
|
506
|
+
};
|
|
507
|
+
const deadline = Date.now() + MUTATION_LOCK_WAIT_MS;
|
|
508
|
+
for (;;) {
|
|
509
|
+
try {
|
|
510
|
+
await publishRecord(path, record, false);
|
|
511
|
+
return async () => {
|
|
512
|
+
await rm(path, { force: true });
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
catch (error) {
|
|
516
|
+
if (error instanceof RunStoreError && error.code === "record_exists") {
|
|
517
|
+
if (Date.now() >= deadline) {
|
|
518
|
+
throw new RunStoreError("locked", `Run ${JSON.stringify(runId)} has a mutation in flight`, path);
|
|
519
|
+
}
|
|
520
|
+
await new Promise((resolve) => setTimeout(resolve, MUTATION_LOCK_POLL_MS));
|
|
521
|
+
continue;
|
|
522
|
+
}
|
|
523
|
+
if (errorCode(error) === "ENOENT") {
|
|
524
|
+
throw new RunStoreError("not_found", `Run ${JSON.stringify(runId)} was not found`, path);
|
|
525
|
+
}
|
|
526
|
+
throw error;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
async function assertRunOwnership(stateRoot, runId, ownership) {
|
|
531
|
+
const capability = parseOwnershipCapability(ownership);
|
|
532
|
+
const path = ownerPath(stateRoot, runId);
|
|
533
|
+
const releaseMutationLock = await acquireRunMutationLock(stateRoot, runId);
|
|
534
|
+
try {
|
|
535
|
+
if (capability.runId !== runId) {
|
|
536
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is owned by another orchestrator`, path);
|
|
537
|
+
}
|
|
538
|
+
const record = validateRunOwnership(await readJson(path), path);
|
|
539
|
+
if (record.runId !== runId || record.ownerId !== capability.ownerId) {
|
|
540
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is owned by another orchestrator`, path);
|
|
541
|
+
}
|
|
542
|
+
return releaseMutationLock;
|
|
543
|
+
}
|
|
544
|
+
catch (error) {
|
|
545
|
+
await releaseMutationLock();
|
|
546
|
+
if (error instanceof RunStoreError && error.code === "not_found") {
|
|
547
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is not owned by an orchestrator`, path);
|
|
548
|
+
}
|
|
549
|
+
throw error;
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
function validStringArray(value, maximumItemBytes) {
|
|
553
|
+
return (Array.isArray(value) &&
|
|
554
|
+
value.length <= RUN_COORDINATION_MAX_ITEMS &&
|
|
555
|
+
value.every((item) => typeof item === "string" &&
|
|
556
|
+
item.trim().length > 0 &&
|
|
557
|
+
Buffer.byteLength(item) <= maximumItemBytes));
|
|
558
|
+
}
|
|
559
|
+
function validateRunEvent(value, path) {
|
|
560
|
+
if (!isRecord(value) ||
|
|
561
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
562
|
+
value.kind !== "run-event" ||
|
|
563
|
+
typeof value.eventId !== "string" ||
|
|
564
|
+
!COORDINATION_ID_PATTERN.test(value.eventId) ||
|
|
565
|
+
typeof value.runId !== "string" ||
|
|
566
|
+
!RUN_ID_PATTERN.test(value.runId) ||
|
|
567
|
+
typeof value.taskId !== "string" ||
|
|
568
|
+
value.taskId.length === 0 ||
|
|
569
|
+
Buffer.byteLength(value.taskId) > RUN_COORDINATION_MAX_TEXT_BYTES ||
|
|
570
|
+
typeof value.eventKind !== "string" ||
|
|
571
|
+
!RUN_EVENT_KINDS.has(value.eventKind) ||
|
|
572
|
+
!isTimestamp(value.timestamp) ||
|
|
573
|
+
typeof value.message !== "string" ||
|
|
574
|
+
value.message.length === 0 ||
|
|
575
|
+
Buffer.byteLength(value.message) > RUN_COORDINATION_MAX_TEXT_BYTES ||
|
|
576
|
+
(value.paths !== undefined &&
|
|
577
|
+
!validStringArray(value.paths, RUN_COORDINATION_MAX_ITEM_BYTES)) ||
|
|
578
|
+
(value.symbols !== undefined &&
|
|
579
|
+
!validStringArray(value.symbols, RUN_COORDINATION_MAX_ITEM_BYTES)) ||
|
|
580
|
+
(value.recipients !== undefined &&
|
|
581
|
+
!validStringArray(value.recipients, RUN_COORDINATION_MAX_TEXT_BYTES)) ||
|
|
582
|
+
Object.keys(value).some((field) => !RUN_EVENT_RECORD_FIELDS.has(field))) {
|
|
583
|
+
return invalidRecord(path, "run event does not match schema version 1");
|
|
584
|
+
}
|
|
585
|
+
return value;
|
|
586
|
+
}
|
|
587
|
+
function validateRunMessage(value, path) {
|
|
588
|
+
if (!isRecord(value) ||
|
|
589
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
590
|
+
value.kind !== "run-message" ||
|
|
591
|
+
typeof value.messageId !== "string" ||
|
|
592
|
+
!COORDINATION_ID_PATTERN.test(value.messageId) ||
|
|
593
|
+
typeof value.runId !== "string" ||
|
|
594
|
+
!RUN_ID_PATTERN.test(value.runId) ||
|
|
595
|
+
typeof value.senderTaskId !== "string" ||
|
|
596
|
+
value.senderTaskId.length === 0 ||
|
|
597
|
+
Buffer.byteLength(value.senderTaskId) > RUN_COORDINATION_MAX_TEXT_BYTES ||
|
|
598
|
+
typeof value.recipientTaskId !== "string" ||
|
|
599
|
+
value.recipientTaskId.length === 0 ||
|
|
600
|
+
Buffer.byteLength(value.recipientTaskId) >
|
|
601
|
+
RUN_COORDINATION_MAX_TEXT_BYTES ||
|
|
602
|
+
!isTimestamp(value.timestamp) ||
|
|
603
|
+
typeof value.message !== "string" ||
|
|
604
|
+
value.message.length === 0 ||
|
|
605
|
+
Buffer.byteLength(value.message) > RUN_COORDINATION_MAX_TEXT_BYTES ||
|
|
606
|
+
Object.keys(value).some((field) => !RUN_MESSAGE_RECORD_FIELDS.has(field))) {
|
|
607
|
+
return invalidRecord(path, "run message does not match schema version 1");
|
|
608
|
+
}
|
|
609
|
+
return value;
|
|
610
|
+
}
|
|
611
|
+
function coordinationText(value, field, maximumBytes = RUN_COORDINATION_MAX_TEXT_BYTES) {
|
|
612
|
+
if (typeof value !== "string" ||
|
|
613
|
+
value.trim().length === 0 ||
|
|
614
|
+
Buffer.byteLength(value) > maximumBytes) {
|
|
615
|
+
throw new RunStoreError("invalid_argument", `Coordination ${field} is invalid or oversized`);
|
|
616
|
+
}
|
|
617
|
+
return value;
|
|
618
|
+
}
|
|
619
|
+
function coordinationList(value, field, maximumItemBytes) {
|
|
620
|
+
if (value === undefined)
|
|
621
|
+
return undefined;
|
|
622
|
+
if (!validStringArray(value, maximumItemBytes)) {
|
|
623
|
+
throw new RunStoreError("invalid_argument", `Coordination ${field} is invalid or oversized`);
|
|
624
|
+
}
|
|
625
|
+
return Object.freeze([...value]);
|
|
626
|
+
}
|
|
627
|
+
function coordinationLimit(value) {
|
|
628
|
+
if (value === undefined)
|
|
629
|
+
return RUN_COORDINATION_MAX_READ;
|
|
630
|
+
if (typeof value !== "number" ||
|
|
631
|
+
!Number.isInteger(value) ||
|
|
632
|
+
value <= 0 ||
|
|
633
|
+
value > RUN_COORDINATION_MAX_READ) {
|
|
634
|
+
throw new RunStoreError("invalid_argument", "Coordination read limit is invalid");
|
|
635
|
+
}
|
|
636
|
+
return value;
|
|
637
|
+
}
|
|
638
|
+
function coordinationCursor(value) {
|
|
639
|
+
if (value === undefined)
|
|
640
|
+
return undefined;
|
|
641
|
+
if (typeof value !== "string" || !COORDINATION_ID_PATTERN.test(value)) {
|
|
642
|
+
throw new RunStoreError("invalid_argument", "Coordination cursor is invalid");
|
|
643
|
+
}
|
|
644
|
+
return value;
|
|
645
|
+
}
|
|
646
|
+
function validateManifest(value, path) {
|
|
647
|
+
if (!isRecord(value) ||
|
|
648
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
649
|
+
value.kind !== "run-manifest" ||
|
|
650
|
+
typeof value.runId !== "string" ||
|
|
651
|
+
!RUN_ID_PATTERN.test(value.runId) ||
|
|
652
|
+
!isTimestamp(value.createdAt) ||
|
|
653
|
+
!isRecord(value.graph) ||
|
|
654
|
+
!Array.isArray(value.graph.tasks)) {
|
|
655
|
+
return invalidRecord(path, "manifest envelope does not match schema version 1");
|
|
656
|
+
}
|
|
657
|
+
const tasks = [];
|
|
658
|
+
const taskKeys = new Set();
|
|
659
|
+
for (const candidate of value.graph.tasks) {
|
|
660
|
+
if (!isRecord(candidate) ||
|
|
661
|
+
typeof candidate.id !== "string" ||
|
|
662
|
+
typeof candidate.key !== "string" ||
|
|
663
|
+
!TASK_KEY_PATTERN.test(candidate.key) ||
|
|
664
|
+
!Array.isArray(candidate.needs) ||
|
|
665
|
+
!candidate.needs.every((dependency) => typeof dependency === "string") ||
|
|
666
|
+
("payload" in candidate && !isJsonValue(candidate.payload))) {
|
|
667
|
+
return invalidRecord(path, "manifest contains an invalid task");
|
|
668
|
+
}
|
|
669
|
+
if (taskKeys.has(candidate.key)) {
|
|
670
|
+
return invalidRecord(path, "manifest contains duplicate task storage keys");
|
|
671
|
+
}
|
|
672
|
+
taskKeys.add(candidate.key);
|
|
673
|
+
tasks.push(candidate);
|
|
674
|
+
}
|
|
675
|
+
const concurrency = value.graph.concurrency;
|
|
676
|
+
if (concurrency !== undefined &&
|
|
677
|
+
(!Number.isInteger(concurrency) || concurrency <= 0)) {
|
|
678
|
+
return invalidRecord(path, "manifest contains invalid concurrency");
|
|
679
|
+
}
|
|
680
|
+
try {
|
|
681
|
+
const normalized = normalizeGraph({
|
|
682
|
+
tasks: tasks.map((task) => ({
|
|
683
|
+
id: task.id,
|
|
684
|
+
needs: task.needs,
|
|
685
|
+
...(task.payload === undefined ? {} : { payload: task.payload }),
|
|
686
|
+
})),
|
|
687
|
+
...(concurrency === undefined
|
|
688
|
+
? {}
|
|
689
|
+
: { concurrency: concurrency }),
|
|
690
|
+
});
|
|
691
|
+
for (const [index, task] of tasks.entries()) {
|
|
692
|
+
const normalizedTask = normalized.tasks[index];
|
|
693
|
+
if (normalizedTask?.id !== task.id ||
|
|
694
|
+
JSON.stringify(normalizedTask.needs) !== JSON.stringify(task.needs)) {
|
|
695
|
+
return invalidRecord(path, "manifest graph is not normalized");
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
catch {
|
|
700
|
+
return invalidRecord(path, "manifest contains an invalid graph");
|
|
701
|
+
}
|
|
702
|
+
return value;
|
|
703
|
+
}
|
|
704
|
+
function validateNodeState(value, path) {
|
|
705
|
+
if (!isRecord(value) ||
|
|
706
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
707
|
+
value.kind !== "node-state" ||
|
|
708
|
+
typeof value.runId !== "string" ||
|
|
709
|
+
typeof value.taskId !== "string" ||
|
|
710
|
+
typeof value.taskKey !== "string" ||
|
|
711
|
+
!Number.isInteger(value.attempt) ||
|
|
712
|
+
value.attempt <= 0 ||
|
|
713
|
+
typeof value.status !== "string" ||
|
|
714
|
+
!NODE_STATUSES.has(value.status) ||
|
|
715
|
+
!isTimestamp(value.updatedAt)) {
|
|
716
|
+
return invalidRecord(path, "node state does not match schema version 1");
|
|
717
|
+
}
|
|
718
|
+
return value;
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
721
|
+
* Artifact text is bounded and non-blank but otherwise unconstrained: it is
|
|
722
|
+
* retained verbatim for a reader, never interpreted by the runtime. Absence
|
|
723
|
+
* is rejected here, because every caller has already decided the artifact is
|
|
724
|
+
* present.
|
|
725
|
+
*/
|
|
726
|
+
function parseArtifactText(value) {
|
|
727
|
+
let text;
|
|
728
|
+
try {
|
|
729
|
+
text = parseNodeArtifact(value);
|
|
730
|
+
}
|
|
731
|
+
catch {
|
|
732
|
+
text = undefined;
|
|
733
|
+
}
|
|
734
|
+
if (text === undefined) {
|
|
735
|
+
throw new RunStoreError("invalid_argument", `A text artifact must be non-blank and at most ${NODE_OUTPUT_LIMITS.maxArtifactBytes} bytes`);
|
|
736
|
+
}
|
|
737
|
+
return text;
|
|
738
|
+
}
|
|
739
|
+
function validArtifactReference(value) {
|
|
740
|
+
return (isRecord(value) &&
|
|
741
|
+
Object.keys(value).length === 1 &&
|
|
742
|
+
Number.isInteger(value.bytes) &&
|
|
743
|
+
value.bytes > 0 &&
|
|
744
|
+
value.bytes <= NODE_OUTPUT_LIMITS.maxArtifactBytes);
|
|
745
|
+
}
|
|
746
|
+
function validateNodeArtifact(value, path) {
|
|
747
|
+
if (!isRecord(value) ||
|
|
748
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
749
|
+
value.kind !== "node-artifact" ||
|
|
750
|
+
typeof value.runId !== "string" ||
|
|
751
|
+
typeof value.taskId !== "string" ||
|
|
752
|
+
typeof value.taskKey !== "string" ||
|
|
753
|
+
!Number.isInteger(value.attempt) ||
|
|
754
|
+
value.attempt <= 0 ||
|
|
755
|
+
!isTimestamp(value.publishedAt) ||
|
|
756
|
+
Object.keys(value).some((field) => !NODE_ARTIFACT_RECORD_FIELDS.has(field))) {
|
|
757
|
+
return invalidRecord(path, "node artifact does not match schema version 1");
|
|
758
|
+
}
|
|
759
|
+
let text;
|
|
760
|
+
try {
|
|
761
|
+
text = parseArtifactText(value.text);
|
|
762
|
+
}
|
|
763
|
+
catch {
|
|
764
|
+
return invalidRecord(path, "node artifact text is invalid or oversized");
|
|
765
|
+
}
|
|
766
|
+
return {
|
|
767
|
+
schemaVersion: SCHEMA_VERSION,
|
|
768
|
+
kind: "node-artifact",
|
|
769
|
+
runId: value.runId,
|
|
770
|
+
taskId: value.taskId,
|
|
771
|
+
taskKey: value.taskKey,
|
|
772
|
+
attempt: value.attempt,
|
|
773
|
+
publishedAt: value.publishedAt,
|
|
774
|
+
text,
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
function validateNodeOutputRecord(value, path) {
|
|
778
|
+
if (!isRecord(value) ||
|
|
779
|
+
value.schemaVersion !== SCHEMA_VERSION ||
|
|
780
|
+
value.kind !== "node-output" ||
|
|
781
|
+
typeof value.runId !== "string" ||
|
|
782
|
+
typeof value.taskId !== "string" ||
|
|
783
|
+
typeof value.taskKey !== "string" ||
|
|
784
|
+
!Number.isInteger(value.attempt) ||
|
|
785
|
+
value.attempt <= 0 ||
|
|
786
|
+
typeof value.status !== "string" ||
|
|
787
|
+
!OUTPUT_STATUSES.has(value.status) ||
|
|
788
|
+
!isTimestamp(value.completedAt) ||
|
|
789
|
+
Object.keys(value).some((field) => !NODE_OUTPUT_RECORD_FIELDS.has(field))) {
|
|
790
|
+
return invalidRecord(path, "node output does not match schema version 1");
|
|
791
|
+
}
|
|
792
|
+
if ("artifact" in value && !validArtifactReference(value.artifact)) {
|
|
793
|
+
return invalidRecord(path, "node output has an invalid artifact reference");
|
|
794
|
+
}
|
|
795
|
+
let usage;
|
|
796
|
+
try {
|
|
797
|
+
usage = parseTaskUsage(value.usage);
|
|
798
|
+
}
|
|
799
|
+
catch {
|
|
800
|
+
return invalidRecord(path, "node output has invalid usage");
|
|
801
|
+
}
|
|
802
|
+
let diagnostics;
|
|
803
|
+
try {
|
|
804
|
+
diagnostics = parseNodeDiagnostics(value.diagnostics);
|
|
805
|
+
}
|
|
806
|
+
catch {
|
|
807
|
+
return invalidRecord(path, "node output has invalid diagnostics");
|
|
808
|
+
}
|
|
809
|
+
const base = {
|
|
810
|
+
schemaVersion: SCHEMA_VERSION,
|
|
811
|
+
kind: "node-output",
|
|
812
|
+
runId: value.runId,
|
|
813
|
+
taskId: value.taskId,
|
|
814
|
+
taskKey: value.taskKey,
|
|
815
|
+
attempt: value.attempt,
|
|
816
|
+
completedAt: value.completedAt,
|
|
817
|
+
...(value.artifact === undefined
|
|
818
|
+
? {}
|
|
819
|
+
: {
|
|
820
|
+
artifact: { bytes: value.artifact.bytes },
|
|
821
|
+
}),
|
|
822
|
+
...(usage === undefined ? {} : { usage }),
|
|
823
|
+
...(diagnostics === undefined ? {} : { diagnostics }),
|
|
824
|
+
};
|
|
825
|
+
if (value.status === "succeeded") {
|
|
826
|
+
if (!("output" in value)) {
|
|
827
|
+
return invalidRecord(path, "succeeded node output is missing its report");
|
|
828
|
+
}
|
|
829
|
+
let output;
|
|
830
|
+
try {
|
|
831
|
+
output = parseNodeOutput(value.output);
|
|
832
|
+
}
|
|
833
|
+
catch {
|
|
834
|
+
return invalidRecord(path, "succeeded node output has an invalid report");
|
|
835
|
+
}
|
|
836
|
+
if (output.blockers.length > 0) {
|
|
837
|
+
return invalidRecord(path, "succeeded node output report contains blockers");
|
|
838
|
+
}
|
|
839
|
+
return { ...base, status: value.status, output };
|
|
840
|
+
}
|
|
841
|
+
if (value.status === "failed") {
|
|
842
|
+
if (!("output" in value))
|
|
843
|
+
return { ...base, status: value.status };
|
|
844
|
+
try {
|
|
845
|
+
return {
|
|
846
|
+
...base,
|
|
847
|
+
status: value.status,
|
|
848
|
+
output: parseNodeOutput(value.output),
|
|
849
|
+
};
|
|
850
|
+
}
|
|
851
|
+
catch {
|
|
852
|
+
return invalidRecord(path, "failed node output has an invalid report");
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
if ("output" in value) {
|
|
856
|
+
return invalidRecord(path, "aborted node output contains a report");
|
|
857
|
+
}
|
|
858
|
+
return { ...base, status: "aborted" };
|
|
859
|
+
}
|
|
860
|
+
function findTask(manifest, taskId) {
|
|
861
|
+
const task = manifest.graph.tasks.find((candidate) => candidate.id === taskId);
|
|
862
|
+
if (!task) {
|
|
863
|
+
throw new RunStoreError("unknown_task", `Run ${JSON.stringify(manifest.runId)} has no task ${JSON.stringify(taskId)}`);
|
|
864
|
+
}
|
|
865
|
+
return task;
|
|
866
|
+
}
|
|
867
|
+
function assertIdentity(path, manifest, task, record) {
|
|
868
|
+
if (record.runId !== manifest.runId ||
|
|
869
|
+
record.taskId !== task.id ||
|
|
870
|
+
record.taskKey !== task.key) {
|
|
871
|
+
invalidRecord(path, "record identity does not match its run and task");
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
async function readNodeStateRecord(stateRoot, manifest, task) {
|
|
875
|
+
const path = join(runPath(stateRoot, manifest.runId), "nodes", `${task.key}.json`);
|
|
876
|
+
const record = validateNodeState(await readJson(path), path);
|
|
877
|
+
assertIdentity(path, manifest, task, record);
|
|
878
|
+
return record;
|
|
879
|
+
}
|
|
880
|
+
async function readNodeOutputRecord(stateRoot, manifest, task) {
|
|
881
|
+
const path = join(runPath(stateRoot, manifest.runId), "outputs", `${task.key}.json`);
|
|
882
|
+
const record = validateNodeOutputRecord(await readJson(path), path);
|
|
883
|
+
assertIdentity(path, manifest, task, record);
|
|
884
|
+
return record;
|
|
885
|
+
}
|
|
886
|
+
function artifactPath(stateRoot, runId, taskKey) {
|
|
887
|
+
return join(runPath(stateRoot, runId), RUN_ARTIFACT_DIRECTORY, `${taskKey}.json`);
|
|
888
|
+
}
|
|
889
|
+
async function readNodeArtifactRecord(stateRoot, manifest, task) {
|
|
890
|
+
const path = artifactPath(stateRoot, manifest.runId, task.key);
|
|
891
|
+
const record = validateNodeArtifact(await readJson(path), path);
|
|
892
|
+
assertIdentity(path, manifest, task, record);
|
|
893
|
+
return record;
|
|
894
|
+
}
|
|
895
|
+
function graphFromManifest(manifest) {
|
|
896
|
+
return normalizeGraph({
|
|
897
|
+
tasks: manifest.graph.tasks.map((task) => ({
|
|
898
|
+
id: task.id,
|
|
899
|
+
needs: task.needs,
|
|
900
|
+
...(task.payload === undefined ? {} : { payload: task.payload }),
|
|
901
|
+
})),
|
|
902
|
+
...(manifest.graph.concurrency === undefined
|
|
903
|
+
? {}
|
|
904
|
+
: { concurrency: manifest.graph.concurrency }),
|
|
905
|
+
});
|
|
906
|
+
}
|
|
907
|
+
export async function createRun(stateRoot, graph, maximumRuns = RUN_STORE_DEFAULT_MAX_RUNS) {
|
|
908
|
+
const root = stateRootPath(stateRoot);
|
|
909
|
+
if (!Number.isInteger(maximumRuns) ||
|
|
910
|
+
maximumRuns <= 0 ||
|
|
911
|
+
maximumRuns > RUN_STORE_MAX_RUNS) {
|
|
912
|
+
throw new RunStoreError("invalid_argument", `The maximum retained run count must be between 1 and ${RUN_STORE_MAX_RUNS}`);
|
|
913
|
+
}
|
|
914
|
+
const validatedGraph = normalizeGraph({
|
|
915
|
+
tasks: graph.tasks,
|
|
916
|
+
...(graph.concurrency === undefined
|
|
917
|
+
? {}
|
|
918
|
+
: { concurrency: graph.concurrency }),
|
|
919
|
+
});
|
|
920
|
+
const runId = randomUUID();
|
|
921
|
+
const createdAt = new Date().toISOString();
|
|
922
|
+
const tasks = validatedGraph.tasks.map((task) => {
|
|
923
|
+
if (task.payload !== undefined && !isJsonValue(task.payload)) {
|
|
924
|
+
throw new RunStoreError("invalid_argument", `Payload for task ${JSON.stringify(task.id)} is not a JSON value`);
|
|
925
|
+
}
|
|
926
|
+
return {
|
|
927
|
+
id: task.id,
|
|
928
|
+
key: `task-${randomUUID()}`,
|
|
929
|
+
needs: [...task.needs],
|
|
930
|
+
...(task.payload === undefined
|
|
931
|
+
? {}
|
|
932
|
+
: { payload: task.payload }),
|
|
933
|
+
};
|
|
934
|
+
});
|
|
935
|
+
const manifest = {
|
|
936
|
+
schemaVersion: SCHEMA_VERSION,
|
|
937
|
+
kind: "run-manifest",
|
|
938
|
+
runId,
|
|
939
|
+
createdAt,
|
|
940
|
+
graph: {
|
|
941
|
+
tasks,
|
|
942
|
+
...(validatedGraph.concurrency === undefined
|
|
943
|
+
? {}
|
|
944
|
+
: { concurrency: validatedGraph.concurrency }),
|
|
945
|
+
},
|
|
946
|
+
};
|
|
947
|
+
serializedRecord(manifest);
|
|
948
|
+
const runs = join(root, "runs");
|
|
949
|
+
await mkdir(runs, { recursive: true, mode: DIRECTORY_MODE });
|
|
950
|
+
await chmod(runs, DIRECTORY_MODE);
|
|
951
|
+
const finalRunPath = join(runs, runId);
|
|
952
|
+
const temporaryRunPath = join(runs, `.${runId}.${randomUUID()}.tmp`);
|
|
953
|
+
let slotPath;
|
|
954
|
+
let published = false;
|
|
955
|
+
try {
|
|
956
|
+
await mkdir(temporaryRunPath, { mode: DIRECTORY_MODE });
|
|
957
|
+
for (const directory of [
|
|
958
|
+
"nodes",
|
|
959
|
+
"outputs",
|
|
960
|
+
RUN_ARTIFACT_DIRECTORY,
|
|
961
|
+
RUN_COORDINATION_DIRECTORY,
|
|
962
|
+
]) {
|
|
963
|
+
await mkdir(join(temporaryRunPath, directory), { mode: DIRECTORY_MODE });
|
|
964
|
+
}
|
|
965
|
+
await writeNewFile(join(temporaryRunPath, "run.json"), manifest);
|
|
966
|
+
for (const task of tasks) {
|
|
967
|
+
const state = {
|
|
968
|
+
schemaVersion: SCHEMA_VERSION,
|
|
969
|
+
kind: "node-state",
|
|
970
|
+
runId,
|
|
971
|
+
taskId: task.id,
|
|
972
|
+
taskKey: task.key,
|
|
973
|
+
attempt: 1,
|
|
974
|
+
status: "pending",
|
|
975
|
+
updatedAt: createdAt,
|
|
976
|
+
};
|
|
977
|
+
await writeNewFile(join(temporaryRunPath, "nodes", `${task.key}.json`), state);
|
|
978
|
+
}
|
|
979
|
+
// The slot is claimed as late as possible, so an interrupted creation can
|
|
980
|
+
// only strand one between this call and the rename that publishes the run.
|
|
981
|
+
slotPath = await claimRunSlot(runs, runId, maximumRuns);
|
|
982
|
+
await rename(temporaryRunPath, finalRunPath);
|
|
983
|
+
published = true;
|
|
984
|
+
}
|
|
985
|
+
finally {
|
|
986
|
+
await rm(temporaryRunPath, { recursive: true, force: true });
|
|
987
|
+
if (!published && slotPath !== undefined) {
|
|
988
|
+
await releaseRunSlot(slotPath, runId);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
return manifest;
|
|
992
|
+
}
|
|
993
|
+
export async function acquireRunOwnership(stateRoot, runId) {
|
|
994
|
+
// Validate the run before creating an owner record, so ownership can never
|
|
995
|
+
// accidentally create a new addressable run.
|
|
996
|
+
await readRun(stateRoot, runId);
|
|
997
|
+
const path = ownerPath(stateRoot, runId);
|
|
998
|
+
const ownership = Object.freeze({
|
|
999
|
+
runId,
|
|
1000
|
+
ownerId: randomUUID(),
|
|
1001
|
+
});
|
|
1002
|
+
const releaseMutationLock = await acquireRunMutationLock(stateRoot, runId);
|
|
1003
|
+
try {
|
|
1004
|
+
try {
|
|
1005
|
+
await writeNewFile(path, {
|
|
1006
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1007
|
+
kind: "run-owner",
|
|
1008
|
+
runId,
|
|
1009
|
+
ownerId: ownership.ownerId,
|
|
1010
|
+
acquiredAt: new Date().toISOString(),
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
catch (error) {
|
|
1014
|
+
if (errorCode(error) === "EEXIST") {
|
|
1015
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is already owned by another orchestrator`, path);
|
|
1016
|
+
}
|
|
1017
|
+
throw error;
|
|
1018
|
+
}
|
|
1019
|
+
return ownership;
|
|
1020
|
+
}
|
|
1021
|
+
finally {
|
|
1022
|
+
await releaseMutationLock();
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Names every run and every capacity slot the store retains.
|
|
1027
|
+
*
|
|
1028
|
+
* Cleanup is explicit and by name, so an operator has to be able to see what
|
|
1029
|
+
* holds capacity before choosing what to give up. Reads are per run, which is
|
|
1030
|
+
* why this is a maintenance call and not something the run lifecycle uses.
|
|
1031
|
+
*/
|
|
1032
|
+
export async function listRetainedRuns(stateRoot) {
|
|
1033
|
+
const runs = runsPath(stateRoot);
|
|
1034
|
+
let published;
|
|
1035
|
+
try {
|
|
1036
|
+
published = await publishedRunIds(runs);
|
|
1037
|
+
}
|
|
1038
|
+
catch (error) {
|
|
1039
|
+
if (errorCode(error) === "ENOENT")
|
|
1040
|
+
return [];
|
|
1041
|
+
throw error;
|
|
1042
|
+
}
|
|
1043
|
+
const claims = await readSlotClaims(join(runs, RUN_SLOT_DIRECTORY));
|
|
1044
|
+
const slotted = new Set(claims.flatMap((claim) => claim.owner ?? []));
|
|
1045
|
+
const entries = await Promise.all([
|
|
1046
|
+
...claims.map((claim) => retainedRun(stateRoot, claim.owner, claim.index)),
|
|
1047
|
+
...[...published]
|
|
1048
|
+
.filter((runId) => !slotted.has(runId))
|
|
1049
|
+
.map((runId) => retainedRun(stateRoot, runId, undefined)),
|
|
1050
|
+
]);
|
|
1051
|
+
// Oldest first, so the entries an operator is most likely to give up come
|
|
1052
|
+
// first; entries with no run directory sort last, by slot.
|
|
1053
|
+
return Object.freeze(entries.sort((left, right) => (left.createdAt ?? "\uffff").localeCompare(right.createdAt ?? "\uffff") ||
|
|
1054
|
+
(left.runId ?? "").localeCompare(right.runId ?? "") ||
|
|
1055
|
+
(left.slot ?? 0) - (right.slot ?? 0)));
|
|
1056
|
+
}
|
|
1057
|
+
async function retainedRun(stateRoot, runId, slot) {
|
|
1058
|
+
if (runId === undefined)
|
|
1059
|
+
return { owned: false, ...(slot === undefined ? {} : { slot }) };
|
|
1060
|
+
let createdAt;
|
|
1061
|
+
try {
|
|
1062
|
+
createdAt = (await readRun(stateRoot, runId)).createdAt;
|
|
1063
|
+
}
|
|
1064
|
+
catch (error) {
|
|
1065
|
+
// This listing is what an operator reads when the store is already in
|
|
1066
|
+
// trouble, so one run it cannot read must not hide the rest. The entry
|
|
1067
|
+
// still names the run, which is all `deleteRun` needs. A slot naming a run
|
|
1068
|
+
// that was never published, or one already removed, reads the same way.
|
|
1069
|
+
if (!(error instanceof RunStoreError))
|
|
1070
|
+
throw error;
|
|
1071
|
+
}
|
|
1072
|
+
return {
|
|
1073
|
+
runId,
|
|
1074
|
+
...(slot === undefined ? {} : { slot }),
|
|
1075
|
+
...(createdAt === undefined ? {} : { createdAt }),
|
|
1076
|
+
// Read the way `deleteRun` reads it, so the listing never reports a run as
|
|
1077
|
+
// free that deletion then refuses.
|
|
1078
|
+
owned: await runHasOwner(stateRoot, runId),
|
|
1079
|
+
};
|
|
1080
|
+
}
|
|
1081
|
+
/**
|
|
1082
|
+
* Only a missing owner record proves a run is unowned. Anything else in its
|
|
1083
|
+
* place is treated as an owner, so cleanup fails closed rather than deleting
|
|
1084
|
+
* state an orchestrator may still be writing.
|
|
1085
|
+
*/
|
|
1086
|
+
async function runHasOwner(stateRoot, runId) {
|
|
1087
|
+
try {
|
|
1088
|
+
await readJson(ownerPath(stateRoot, runId));
|
|
1089
|
+
}
|
|
1090
|
+
catch (error) {
|
|
1091
|
+
return !(error instanceof RunStoreError && error.code === "not_found");
|
|
1092
|
+
}
|
|
1093
|
+
return true;
|
|
1094
|
+
}
|
|
1095
|
+
async function findRunSlot(slots, runId) {
|
|
1096
|
+
let entries;
|
|
1097
|
+
try {
|
|
1098
|
+
entries = await readdir(slots, { withFileTypes: true, encoding: "utf8" });
|
|
1099
|
+
}
|
|
1100
|
+
catch (error) {
|
|
1101
|
+
if (errorCode(error) === "ENOENT")
|
|
1102
|
+
return undefined;
|
|
1103
|
+
throw error;
|
|
1104
|
+
}
|
|
1105
|
+
for (const entry of entries) {
|
|
1106
|
+
if (!entry.isFile() || !RUN_SLOT_PATTERN.test(entry.name))
|
|
1107
|
+
continue;
|
|
1108
|
+
const path = join(slots, entry.name);
|
|
1109
|
+
if ((await slotOwner(path)) === runId)
|
|
1110
|
+
return path;
|
|
1111
|
+
}
|
|
1112
|
+
return undefined;
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Removes one run's directory and its capacity slot, and reports whether
|
|
1116
|
+
* anything was there to remove.
|
|
1117
|
+
*
|
|
1118
|
+
* The run is named rather than selected by age or count: nothing here may
|
|
1119
|
+
* decide on an operator's behalf which diagnostic state is worth losing. A run
|
|
1120
|
+
* an orchestrator still holds is refused, and the mutation lock is held across
|
|
1121
|
+
* that check so ownership cannot be acquired between it and the removal.
|
|
1122
|
+
*
|
|
1123
|
+
* The directory goes first and the slot second. An interruption then strands a
|
|
1124
|
+
* slot, which the store already reports as reclaimable capacity, instead of
|
|
1125
|
+
* leaving a published run whose slot is missing — the disagreement that stops
|
|
1126
|
+
* the store admitting any work at all.
|
|
1127
|
+
*/
|
|
1128
|
+
export async function deleteRun(stateRoot, runId) {
|
|
1129
|
+
assertRunId(runId);
|
|
1130
|
+
const runs = runsPath(stateRoot);
|
|
1131
|
+
const slot = await findRunSlot(join(runs, RUN_SLOT_DIRECTORY), runId);
|
|
1132
|
+
let removed = false;
|
|
1133
|
+
let releaseMutationLock;
|
|
1134
|
+
try {
|
|
1135
|
+
releaseMutationLock = await acquireRunMutationLock(stateRoot, runId);
|
|
1136
|
+
}
|
|
1137
|
+
catch (error) {
|
|
1138
|
+
// With no run directory there is only a stranded slot left to release.
|
|
1139
|
+
if (!(error instanceof RunStoreError) || error.code !== "not_found") {
|
|
1140
|
+
throw error;
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
if (releaseMutationLock !== undefined) {
|
|
1144
|
+
try {
|
|
1145
|
+
if (await runHasOwner(stateRoot, runId)) {
|
|
1146
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is owned by an orchestrator`, ownerPath(stateRoot, runId));
|
|
1147
|
+
}
|
|
1148
|
+
await rm(join(runs, runId), { recursive: true, force: true });
|
|
1149
|
+
removed = true;
|
|
1150
|
+
}
|
|
1151
|
+
finally {
|
|
1152
|
+
// The lock lived inside the directory, so this is now a no-op.
|
|
1153
|
+
await releaseMutationLock();
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
if (slot !== undefined) {
|
|
1157
|
+
await releaseRunSlot(slot, runId);
|
|
1158
|
+
removed = true;
|
|
1159
|
+
}
|
|
1160
|
+
return removed;
|
|
1161
|
+
}
|
|
1162
|
+
/**
|
|
1163
|
+
* Removes a mutation lock left behind by a task executor, and reports whether
|
|
1164
|
+
* one was removed. Only the run's owner may recover a lock; ordinary callers
|
|
1165
|
+
* must use the lock release returned by the mutation operation instead.
|
|
1166
|
+
*
|
|
1167
|
+
* `isHolderFinished` decides whether a lock that is still in place is stale.
|
|
1168
|
+
* The graph runner passes it while workers may still be running, and answers
|
|
1169
|
+
* only for a task of this run whose lifecycle has ended, so contention with a
|
|
1170
|
+
* live worker is left alone rather than resolved by taking its lock away. A
|
|
1171
|
+
* lock naming no task, or one this state root cannot attribute, is never
|
|
1172
|
+
* recovered on that path. Omitting the predicate recovers unconditionally and
|
|
1173
|
+
* is correct only once every task promise has settled.
|
|
1174
|
+
*/
|
|
1175
|
+
export async function recoverRunMutationLock(stateRoot, ownership, isHolderFinished) {
|
|
1176
|
+
const capability = parseOwnershipCapability(ownership);
|
|
1177
|
+
const runId = capability.runId;
|
|
1178
|
+
assertRunId(runId);
|
|
1179
|
+
const owner = ownerPath(stateRoot, runId);
|
|
1180
|
+
try {
|
|
1181
|
+
const record = validateRunOwnership(await readJson(owner), owner);
|
|
1182
|
+
if (record.runId !== runId || record.ownerId !== capability.ownerId) {
|
|
1183
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is owned by another orchestrator`, owner);
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
catch (error) {
|
|
1187
|
+
if (error instanceof RunStoreError && error.code === "not_found") {
|
|
1188
|
+
return false;
|
|
1189
|
+
}
|
|
1190
|
+
throw error;
|
|
1191
|
+
}
|
|
1192
|
+
const path = join(runPath(stateRoot, runId), RUN_MUTATION_LOCK);
|
|
1193
|
+
if (isHolderFinished !== undefined) {
|
|
1194
|
+
let holder;
|
|
1195
|
+
try {
|
|
1196
|
+
holder = validateRunMutationLock(await readJson(path), path);
|
|
1197
|
+
}
|
|
1198
|
+
catch (error) {
|
|
1199
|
+
// The lock was released while it was being read, so there is nothing to
|
|
1200
|
+
// recover and the caller should simply retry its mutation.
|
|
1201
|
+
if (error instanceof RunStoreError && error.code === "not_found") {
|
|
1202
|
+
return false;
|
|
1203
|
+
}
|
|
1204
|
+
// Anything else in the lock's place is reported as the invalid record it
|
|
1205
|
+
// is, rather than escaping as a bare filesystem error.
|
|
1206
|
+
if (!(error instanceof RunStoreError)) {
|
|
1207
|
+
return invalidRecord(path, "run mutation lock could not be read");
|
|
1208
|
+
}
|
|
1209
|
+
throw error;
|
|
1210
|
+
}
|
|
1211
|
+
if (holder.runId !== runId) {
|
|
1212
|
+
invalidRecord(path, "run mutation lock does not match its run");
|
|
1213
|
+
}
|
|
1214
|
+
if (holder.taskId === undefined || !isHolderFinished(holder.taskId)) {
|
|
1215
|
+
return false;
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
await rm(path, { recursive: true, force: true });
|
|
1219
|
+
return true;
|
|
1220
|
+
}
|
|
1221
|
+
export async function releaseRunOwnership(stateRoot, ownership) {
|
|
1222
|
+
const capability = parseOwnershipCapability(ownership);
|
|
1223
|
+
const runId = capability.runId;
|
|
1224
|
+
assertRunId(runId);
|
|
1225
|
+
const path = ownerPath(stateRoot, runId);
|
|
1226
|
+
let releaseMutationLock;
|
|
1227
|
+
try {
|
|
1228
|
+
releaseMutationLock = await acquireRunMutationLock(stateRoot, runId);
|
|
1229
|
+
}
|
|
1230
|
+
catch (error) {
|
|
1231
|
+
if (errorCode(error) === "ENOENT" ||
|
|
1232
|
+
(error instanceof RunStoreError && error.code === "not_found")) {
|
|
1233
|
+
return;
|
|
1234
|
+
}
|
|
1235
|
+
throw error;
|
|
1236
|
+
}
|
|
1237
|
+
try {
|
|
1238
|
+
let record;
|
|
1239
|
+
try {
|
|
1240
|
+
record = validateRunOwnership(await readJson(path), path);
|
|
1241
|
+
}
|
|
1242
|
+
catch (error) {
|
|
1243
|
+
if (error instanceof RunStoreError && error.code === "not_found")
|
|
1244
|
+
return;
|
|
1245
|
+
throw error;
|
|
1246
|
+
}
|
|
1247
|
+
if (record.runId !== runId || record.ownerId !== capability.ownerId) {
|
|
1248
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is owned by another orchestrator`, path);
|
|
1249
|
+
}
|
|
1250
|
+
try {
|
|
1251
|
+
await rm(path);
|
|
1252
|
+
}
|
|
1253
|
+
catch (error) {
|
|
1254
|
+
if (errorCode(error) !== "ENOENT")
|
|
1255
|
+
throw error;
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
finally {
|
|
1259
|
+
await releaseMutationLock();
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
export async function readRun(stateRoot, runId) {
|
|
1263
|
+
const path = join(runPath(stateRoot, runId), "run.json");
|
|
1264
|
+
const manifest = validateManifest(await readJson(path), path);
|
|
1265
|
+
if (manifest.runId !== runId) {
|
|
1266
|
+
invalidRecord(path, "manifest run ID does not match its directory");
|
|
1267
|
+
}
|
|
1268
|
+
return manifest;
|
|
1269
|
+
}
|
|
1270
|
+
export async function readNodeState(stateRoot, runId, taskId) {
|
|
1271
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1272
|
+
return readNodeStateRecord(stateRoot, manifest, findTask(manifest, taskId));
|
|
1273
|
+
}
|
|
1274
|
+
/**
|
|
1275
|
+
* Validates and persists one graph transition. Every mutation requires the
|
|
1276
|
+
* caller to hold the run's exclusive ownership record.
|
|
1277
|
+
*/
|
|
1278
|
+
export async function writeNodeState(stateRoot, runId, taskId, status, ownership) {
|
|
1279
|
+
const releaseMutationLock = await assertRunOwnership(stateRoot, runId, ownership);
|
|
1280
|
+
try {
|
|
1281
|
+
return await writeNodeStateUnlocked(stateRoot, runId, taskId, status);
|
|
1282
|
+
}
|
|
1283
|
+
finally {
|
|
1284
|
+
await releaseMutationLock();
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
async function writeNodeStateUnlocked(stateRoot, runId, taskId, status) {
|
|
1288
|
+
if (!NODE_STATUSES.has(status)) {
|
|
1289
|
+
throw new RunStoreError("invalid_argument", `Invalid node status ${JSON.stringify(status)}`);
|
|
1290
|
+
}
|
|
1291
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1292
|
+
const task = findTask(manifest, taskId);
|
|
1293
|
+
const states = await Promise.all(manifest.graph.tasks.map((candidate) => readNodeStateRecord(stateRoot, manifest, candidate)));
|
|
1294
|
+
const current = states.find((candidate) => candidate.taskId === task.id);
|
|
1295
|
+
if (!current) {
|
|
1296
|
+
throw new RunStoreError("invalid_record", `Run is missing node state for task ${JSON.stringify(task.id)}`);
|
|
1297
|
+
}
|
|
1298
|
+
if (current.status === status)
|
|
1299
|
+
return current;
|
|
1300
|
+
const graph = graphFromManifest(manifest);
|
|
1301
|
+
const graphState = new Map(states.map((candidate) => [candidate.taskId, candidate.status]));
|
|
1302
|
+
try {
|
|
1303
|
+
if (status === "blocked") {
|
|
1304
|
+
if (settleBlocked(graph, graphState).get(task.id) !== "blocked") {
|
|
1305
|
+
throw new Error(`Task ${JSON.stringify(task.id)} has no blocking prerequisite`);
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
else {
|
|
1309
|
+
setNodeStatus(graph, graphState, task.id, status);
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
catch (error) {
|
|
1313
|
+
throw new RunStoreError("invalid_record", error instanceof Error ? error.message : "Invalid node transition");
|
|
1314
|
+
}
|
|
1315
|
+
if (OUTPUT_STATUSES.has(status)) {
|
|
1316
|
+
let output;
|
|
1317
|
+
try {
|
|
1318
|
+
output = await readNodeOutputRecord(stateRoot, manifest, task);
|
|
1319
|
+
}
|
|
1320
|
+
catch (error) {
|
|
1321
|
+
if (error instanceof RunStoreError && error.code === "not_found") {
|
|
1322
|
+
throw new RunStoreError("invalid_record", `Terminal state ${JSON.stringify(status)} requires a published output`);
|
|
1323
|
+
}
|
|
1324
|
+
throw error;
|
|
1325
|
+
}
|
|
1326
|
+
if (output.status !== status || output.attempt !== current.attempt) {
|
|
1327
|
+
throw new RunStoreError("invalid_record", "Terminal state does not match the published output");
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
const path = join(runPath(stateRoot, runId), "nodes", `${task.key}.json`);
|
|
1331
|
+
const record = {
|
|
1332
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1333
|
+
kind: "node-state",
|
|
1334
|
+
runId,
|
|
1335
|
+
taskId: task.id,
|
|
1336
|
+
taskKey: task.key,
|
|
1337
|
+
attempt: current.attempt,
|
|
1338
|
+
status,
|
|
1339
|
+
updatedAt: new Date().toISOString(),
|
|
1340
|
+
};
|
|
1341
|
+
await replaceRecord(path, record);
|
|
1342
|
+
return record;
|
|
1343
|
+
}
|
|
1344
|
+
export async function publishNodeOutput(stateRoot, runId, input, ownership) {
|
|
1345
|
+
const releaseMutationLock = await assertRunOwnership(stateRoot, runId, ownership);
|
|
1346
|
+
try {
|
|
1347
|
+
return await publishNodeOutputUnlocked(stateRoot, runId, input);
|
|
1348
|
+
}
|
|
1349
|
+
finally {
|
|
1350
|
+
await releaseMutationLock();
|
|
1351
|
+
}
|
|
1352
|
+
}
|
|
1353
|
+
async function publishNodeOutputUnlocked(stateRoot, runId, input) {
|
|
1354
|
+
if (!OUTPUT_STATUSES.has(input.status)) {
|
|
1355
|
+
throw new RunStoreError("invalid_argument", `Invalid node output status ${JSON.stringify(input.status)}`);
|
|
1356
|
+
}
|
|
1357
|
+
let output;
|
|
1358
|
+
if (input.status === "aborted" && "output" in input) {
|
|
1359
|
+
throw new RunStoreError("invalid_argument", `Aborted task ${JSON.stringify(input.taskId)} must not publish a node report`);
|
|
1360
|
+
}
|
|
1361
|
+
if (input.status === "succeeded" || "output" in input) {
|
|
1362
|
+
try {
|
|
1363
|
+
output = parseNodeOutput(input.output);
|
|
1364
|
+
}
|
|
1365
|
+
catch {
|
|
1366
|
+
throw new RunStoreError("invalid_argument", `Output for task ${JSON.stringify(input.taskId)} is not a valid node report`);
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
if (input.status === "succeeded" && output?.blockers.length) {
|
|
1370
|
+
throw new RunStoreError("invalid_argument", `Succeeded task ${JSON.stringify(input.taskId)} must not report blockers`);
|
|
1371
|
+
}
|
|
1372
|
+
// An aborted task published nothing, so it has no long-form text to retain.
|
|
1373
|
+
// Every other status may, whether or not it also produced a report.
|
|
1374
|
+
let artifact;
|
|
1375
|
+
if (input.artifact !== undefined) {
|
|
1376
|
+
if (input.status === "aborted") {
|
|
1377
|
+
throw new RunStoreError("invalid_argument", `Aborted task ${JSON.stringify(input.taskId)} must not publish a text artifact`);
|
|
1378
|
+
}
|
|
1379
|
+
try {
|
|
1380
|
+
artifact = parseArtifactText(input.artifact);
|
|
1381
|
+
}
|
|
1382
|
+
catch {
|
|
1383
|
+
throw new RunStoreError("invalid_argument", `Text artifact for task ${JSON.stringify(input.taskId)} is invalid or oversized`);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
let usage;
|
|
1387
|
+
try {
|
|
1388
|
+
usage = parseTaskUsage(input.usage);
|
|
1389
|
+
}
|
|
1390
|
+
catch {
|
|
1391
|
+
throw new RunStoreError("invalid_argument", `Usage for task ${JSON.stringify(input.taskId)} is invalid or out of range`);
|
|
1392
|
+
}
|
|
1393
|
+
let diagnostics;
|
|
1394
|
+
try {
|
|
1395
|
+
diagnostics = parseNodeDiagnostics(input.diagnostics);
|
|
1396
|
+
}
|
|
1397
|
+
catch {
|
|
1398
|
+
throw new RunStoreError("invalid_argument", `Diagnostics for task ${JSON.stringify(input.taskId)} are invalid or oversized`);
|
|
1399
|
+
}
|
|
1400
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1401
|
+
const task = findTask(manifest, input.taskId);
|
|
1402
|
+
const state = await readNodeStateRecord(stateRoot, manifest, task);
|
|
1403
|
+
const canPublish = state.status === "running" ||
|
|
1404
|
+
(state.status === "pending" && input.status !== "succeeded") ||
|
|
1405
|
+
state.status === input.status;
|
|
1406
|
+
if (!canPublish) {
|
|
1407
|
+
throw new RunStoreError("invalid_record", `Output status ${JSON.stringify(input.status)} conflicts with node status ${JSON.stringify(state.status)}`);
|
|
1408
|
+
}
|
|
1409
|
+
// The artifact is published first, so an interruption can only strand an
|
|
1410
|
+
// artifact no output claims. Publishing the output first would leave a
|
|
1411
|
+
// reference to a file that does not exist.
|
|
1412
|
+
const publishedAt = new Date().toISOString();
|
|
1413
|
+
if (artifact !== undefined) {
|
|
1414
|
+
const artifactRecord = {
|
|
1415
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1416
|
+
kind: "node-artifact",
|
|
1417
|
+
runId,
|
|
1418
|
+
taskId: task.id,
|
|
1419
|
+
taskKey: task.key,
|
|
1420
|
+
attempt: state.attempt,
|
|
1421
|
+
publishedAt,
|
|
1422
|
+
text: artifact,
|
|
1423
|
+
};
|
|
1424
|
+
await publishRecord(artifactPath(stateRoot, runId, task.key), artifactRecord);
|
|
1425
|
+
}
|
|
1426
|
+
const path = join(runPath(stateRoot, runId), "outputs", `${task.key}.json`);
|
|
1427
|
+
const baseRecord = {
|
|
1428
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1429
|
+
kind: "node-output",
|
|
1430
|
+
runId,
|
|
1431
|
+
taskId: task.id,
|
|
1432
|
+
taskKey: task.key,
|
|
1433
|
+
attempt: state.attempt,
|
|
1434
|
+
completedAt: publishedAt,
|
|
1435
|
+
...(artifact === undefined
|
|
1436
|
+
? {}
|
|
1437
|
+
: { artifact: { bytes: Buffer.byteLength(artifact, "utf8") } }),
|
|
1438
|
+
...(usage === undefined ? {} : { usage }),
|
|
1439
|
+
...(diagnostics === undefined ? {} : { diagnostics }),
|
|
1440
|
+
};
|
|
1441
|
+
let record;
|
|
1442
|
+
if (input.status === "succeeded") {
|
|
1443
|
+
if (output === undefined) {
|
|
1444
|
+
throw new Error("Validated node report is unexpectedly missing");
|
|
1445
|
+
}
|
|
1446
|
+
record = { ...baseRecord, status: input.status, output };
|
|
1447
|
+
}
|
|
1448
|
+
else if (input.status === "failed") {
|
|
1449
|
+
record = {
|
|
1450
|
+
...baseRecord,
|
|
1451
|
+
status: input.status,
|
|
1452
|
+
...(output === undefined ? {} : { output }),
|
|
1453
|
+
};
|
|
1454
|
+
}
|
|
1455
|
+
else {
|
|
1456
|
+
record = { ...baseRecord, status: input.status };
|
|
1457
|
+
}
|
|
1458
|
+
await publishRecord(path, record);
|
|
1459
|
+
return record;
|
|
1460
|
+
}
|
|
1461
|
+
function parsePublishRunEvent(value) {
|
|
1462
|
+
try {
|
|
1463
|
+
if (!isRecord(value) ||
|
|
1464
|
+
Object.keys(value).some((field) => !new Set([
|
|
1465
|
+
"taskId",
|
|
1466
|
+
"eventKind",
|
|
1467
|
+
"message",
|
|
1468
|
+
"paths",
|
|
1469
|
+
"symbols",
|
|
1470
|
+
"recipients",
|
|
1471
|
+
]).has(field))) {
|
|
1472
|
+
throw new Error("invalid event");
|
|
1473
|
+
}
|
|
1474
|
+
const eventKind = value.eventKind;
|
|
1475
|
+
if (typeof eventKind !== "string" ||
|
|
1476
|
+
!RUN_EVENT_KINDS.has(eventKind)) {
|
|
1477
|
+
throw new Error("invalid event kind");
|
|
1478
|
+
}
|
|
1479
|
+
const taskId = coordinationText(value.taskId, "task ID");
|
|
1480
|
+
const message = coordinationText(value.message, "message");
|
|
1481
|
+
const paths = coordinationList(value.paths, "paths", RUN_COORDINATION_MAX_ITEM_BYTES);
|
|
1482
|
+
const symbols = coordinationList(value.symbols, "symbols", RUN_COORDINATION_MAX_ITEM_BYTES);
|
|
1483
|
+
const recipients = coordinationList(value.recipients, "recipients", RUN_COORDINATION_MAX_TEXT_BYTES);
|
|
1484
|
+
return {
|
|
1485
|
+
taskId,
|
|
1486
|
+
eventKind: eventKind,
|
|
1487
|
+
message,
|
|
1488
|
+
...(paths === undefined ? {} : { paths }),
|
|
1489
|
+
...(symbols === undefined ? {} : { symbols }),
|
|
1490
|
+
...(recipients === undefined ? {} : { recipients }),
|
|
1491
|
+
};
|
|
1492
|
+
}
|
|
1493
|
+
catch {
|
|
1494
|
+
throw new RunStoreError("invalid_argument", "Run event is invalid or oversized");
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
function parsePublishRunMessage(value) {
|
|
1498
|
+
try {
|
|
1499
|
+
if (!isRecord(value) ||
|
|
1500
|
+
Object.keys(value).some((field) => !new Set(["senderTaskId", "recipientTaskId", "message"]).has(field))) {
|
|
1501
|
+
throw new Error("invalid message");
|
|
1502
|
+
}
|
|
1503
|
+
return {
|
|
1504
|
+
senderTaskId: coordinationText(value.senderTaskId, "sender task ID"),
|
|
1505
|
+
recipientTaskId: coordinationText(value.recipientTaskId, "recipient task ID"),
|
|
1506
|
+
message: coordinationText(value.message, "message"),
|
|
1507
|
+
};
|
|
1508
|
+
}
|
|
1509
|
+
catch {
|
|
1510
|
+
throw new RunStoreError("invalid_argument", "Run message is invalid or oversized");
|
|
1511
|
+
}
|
|
1512
|
+
}
|
|
1513
|
+
function parseRunEventQuery(value) {
|
|
1514
|
+
try {
|
|
1515
|
+
if (value === undefined)
|
|
1516
|
+
return {};
|
|
1517
|
+
if (!isRecord(value) ||
|
|
1518
|
+
Object.keys(value).some((field) => !new Set([
|
|
1519
|
+
"cursor",
|
|
1520
|
+
"eventKind",
|
|
1521
|
+
"recipient",
|
|
1522
|
+
"path",
|
|
1523
|
+
"symbol",
|
|
1524
|
+
"limit",
|
|
1525
|
+
]).has(field))) {
|
|
1526
|
+
throw new Error("invalid event query");
|
|
1527
|
+
}
|
|
1528
|
+
const cursor = coordinationCursor(value.cursor);
|
|
1529
|
+
const eventKind = value.eventKind;
|
|
1530
|
+
if (eventKind !== undefined &&
|
|
1531
|
+
(typeof eventKind !== "string" ||
|
|
1532
|
+
!RUN_EVENT_KINDS.has(eventKind))) {
|
|
1533
|
+
throw new Error("invalid event kind");
|
|
1534
|
+
}
|
|
1535
|
+
const recipient = value.recipient === undefined
|
|
1536
|
+
? undefined
|
|
1537
|
+
: coordinationText(value.recipient, "recipient");
|
|
1538
|
+
const path = value.path === undefined
|
|
1539
|
+
? undefined
|
|
1540
|
+
: coordinationText(value.path, "path", RUN_COORDINATION_MAX_ITEM_BYTES);
|
|
1541
|
+
const symbol = value.symbol === undefined
|
|
1542
|
+
? undefined
|
|
1543
|
+
: coordinationText(value.symbol, "symbol", RUN_COORDINATION_MAX_ITEM_BYTES);
|
|
1544
|
+
const limit = coordinationLimit(value.limit);
|
|
1545
|
+
return {
|
|
1546
|
+
...(cursor === undefined ? {} : { cursor }),
|
|
1547
|
+
...(eventKind === undefined
|
|
1548
|
+
? {}
|
|
1549
|
+
: { eventKind: eventKind }),
|
|
1550
|
+
...(recipient === undefined ? {} : { recipient }),
|
|
1551
|
+
...(path === undefined ? {} : { path }),
|
|
1552
|
+
...(symbol === undefined ? {} : { symbol }),
|
|
1553
|
+
limit,
|
|
1554
|
+
};
|
|
1555
|
+
}
|
|
1556
|
+
catch {
|
|
1557
|
+
throw new RunStoreError("invalid_argument", "Run event query is invalid");
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
function parseRunMessageQuery(value) {
|
|
1561
|
+
try {
|
|
1562
|
+
if (value === undefined)
|
|
1563
|
+
return {};
|
|
1564
|
+
if (!isRecord(value) ||
|
|
1565
|
+
Object.keys(value).some((field) => !new Set(["cursor", "limit"]).has(field))) {
|
|
1566
|
+
throw new Error("invalid message query");
|
|
1567
|
+
}
|
|
1568
|
+
const cursor = coordinationCursor(value.cursor);
|
|
1569
|
+
const limit = coordinationLimit(value.limit);
|
|
1570
|
+
return { ...(cursor === undefined ? {} : { cursor }), limit };
|
|
1571
|
+
}
|
|
1572
|
+
catch {
|
|
1573
|
+
throw new RunStoreError("invalid_argument", "Run message query is invalid");
|
|
1574
|
+
}
|
|
1575
|
+
}
|
|
1576
|
+
async function coordinationFiles(directory) {
|
|
1577
|
+
let entries;
|
|
1578
|
+
try {
|
|
1579
|
+
entries = await readdir(directory, {
|
|
1580
|
+
withFileTypes: true,
|
|
1581
|
+
encoding: "utf8",
|
|
1582
|
+
});
|
|
1583
|
+
}
|
|
1584
|
+
catch (error) {
|
|
1585
|
+
if (errorCode(error) === "ENOENT") {
|
|
1586
|
+
throw new RunStoreError("not_found", `Coordination directory not found at ${JSON.stringify(directory)}`, directory);
|
|
1587
|
+
}
|
|
1588
|
+
throw error;
|
|
1589
|
+
}
|
|
1590
|
+
// Identifiers are fixed-width, so sorting the names sorts the sequence.
|
|
1591
|
+
const files = entries
|
|
1592
|
+
.filter((entry) => entry.isFile() &&
|
|
1593
|
+
entry.name.endsWith(RECORD_SUFFIX) &&
|
|
1594
|
+
COORDINATION_ID_PATTERN.test(recordId(entry.name)))
|
|
1595
|
+
.map((entry) => entry.name)
|
|
1596
|
+
.sort();
|
|
1597
|
+
if (files.length > RUN_COORDINATION_MAX_RECORDS) {
|
|
1598
|
+
throw new RunStoreError("record_too_large", `Coordination records exceed the retained limit of ${RUN_COORDINATION_MAX_RECORDS}`, directory);
|
|
1599
|
+
}
|
|
1600
|
+
return files;
|
|
1601
|
+
}
|
|
1602
|
+
/**
|
|
1603
|
+
* Coordination records are published by workers, which never hold the
|
|
1604
|
+
* orchestrator's ownership capability: that capability authorizes node state
|
|
1605
|
+
* and output mutations for the whole run. Requiring an owner record rather than
|
|
1606
|
+
* the capability itself keeps an unowned or finished run immutable while
|
|
1607
|
+
* leaving worker writes unprivileged.
|
|
1608
|
+
*
|
|
1609
|
+
* Records are therefore attributed to a task rather than authenticated. Every
|
|
1610
|
+
* worker of a run shares this state root, exactly as they share the checkout.
|
|
1611
|
+
*/
|
|
1612
|
+
async function withActiveRunPublication(stateRoot, runId, holderTaskId, publish) {
|
|
1613
|
+
// The lock names its holder and the graph runner acts on that name, so the
|
|
1614
|
+
// task is resolved against the manifest before it is written and a worker
|
|
1615
|
+
// cannot label its lock with another task. A published run's manifest is
|
|
1616
|
+
// immutable, so it is read once and outside the lock; only the active-owner
|
|
1617
|
+
// check has to be synchronized with ownership release.
|
|
1618
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1619
|
+
const holder = findTask(manifest, holderTaskId);
|
|
1620
|
+
const releaseMutationLock = await acquireRunMutationLock(stateRoot, runId, holder.id);
|
|
1621
|
+
try {
|
|
1622
|
+
await assertActiveRunOwner(stateRoot, runId);
|
|
1623
|
+
return await publish(manifest, holder);
|
|
1624
|
+
}
|
|
1625
|
+
finally {
|
|
1626
|
+
await releaseMutationLock();
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
async function assertActiveRunOwner(stateRoot, runId) {
|
|
1630
|
+
const path = ownerPath(stateRoot, runId);
|
|
1631
|
+
let owner;
|
|
1632
|
+
try {
|
|
1633
|
+
owner = validateRunOwnership(await readJson(path), path);
|
|
1634
|
+
}
|
|
1635
|
+
catch (error) {
|
|
1636
|
+
if (error instanceof RunStoreError && error.code === "not_found") {
|
|
1637
|
+
throw new RunStoreError("ownership", `Run ${JSON.stringify(runId)} is not owned by an orchestrator`, path);
|
|
1638
|
+
}
|
|
1639
|
+
throw error;
|
|
1640
|
+
}
|
|
1641
|
+
if (owner.runId !== runId) {
|
|
1642
|
+
invalidRecord(path, "run ownership does not match its run");
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1645
|
+
/**
|
|
1646
|
+
* Coordination records are ordered by one run-global sequence, so a cursor
|
|
1647
|
+
* names a position that no later record can precede and a reader polling with
|
|
1648
|
+
* one can never skip a record.
|
|
1649
|
+
*
|
|
1650
|
+
* The caller holds the run mutation lock while checking ownership and invoking
|
|
1651
|
+
* this function, so publishers of one run do not race each other. Claiming a
|
|
1652
|
+
* sequence number and publishing the record it names are still one exclusive
|
|
1653
|
+
* create in the journal, so a failed attempt leaves no identifier behind and
|
|
1654
|
+
* a writer that ignored the lock cannot take an identifier already used. The
|
|
1655
|
+
* lock additionally prevents ownership release from landing between the
|
|
1656
|
+
* active-run check and publication.
|
|
1657
|
+
*/
|
|
1658
|
+
async function publishCoordinationRecord(stateRoot, runId, build) {
|
|
1659
|
+
const directory = coordinationDirectory(stateRoot, runId);
|
|
1660
|
+
for (let attempt = 0; attempt < COORDINATION_PUBLISH_ATTEMPTS; attempt += 1) {
|
|
1661
|
+
const published = await coordinationFiles(directory);
|
|
1662
|
+
const previous = published.at(-1);
|
|
1663
|
+
const sequence = previous === undefined ? 1 : Number.parseInt(recordId(previous), 10) + 1;
|
|
1664
|
+
if (published.length >= RUN_COORDINATION_MAX_RECORDS ||
|
|
1665
|
+
sequence > MAX_COORDINATION_SEQUENCE) {
|
|
1666
|
+
throw new RunStoreError("retention_limit", `Run ${JSON.stringify(runId)} retains the maximum of ${RUN_COORDINATION_MAX_RECORDS} coordination records`, directory);
|
|
1667
|
+
}
|
|
1668
|
+
const id = String(sequence).padStart(RUN_COORDINATION_ID_LENGTH, "0");
|
|
1669
|
+
const record = build(id);
|
|
1670
|
+
// Bounded as a whole rather than field by field: the guarantee a reader
|
|
1671
|
+
// depends on is that one record always fits in one page.
|
|
1672
|
+
const bytes = Buffer.byteLength(JSON.stringify(record));
|
|
1673
|
+
if (bytes > RUN_COORDINATION_MAX_RECORD_BYTES) {
|
|
1674
|
+
throw new RunStoreError("record_too_large", `Coordination record of ${bytes} bytes exceeds the bound of ${RUN_COORDINATION_MAX_RECORD_BYTES} bytes`, directory);
|
|
1675
|
+
}
|
|
1676
|
+
try {
|
|
1677
|
+
await publishRecord(join(directory, `${id}${RECORD_SUFFIX}`), record);
|
|
1678
|
+
return record;
|
|
1679
|
+
}
|
|
1680
|
+
catch (error) {
|
|
1681
|
+
if (!(error instanceof RunStoreError) || error.code !== "record_exists") {
|
|
1682
|
+
throw error;
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
// Publication is serialized by the run mutation lock, so the identifier the
|
|
1687
|
+
// journal reports as free is normally free. Exhausting the attempts means
|
|
1688
|
+
// something outside that lock is writing into the journal.
|
|
1689
|
+
throw new RunStoreError("record_exists", `Run ${JSON.stringify(runId)} could not claim a coordination identifier`, directory);
|
|
1690
|
+
}
|
|
1691
|
+
/**
|
|
1692
|
+
* Names every record after `cursor`, in sequence order, without reading any of
|
|
1693
|
+
* them. Both kinds share the journal and filtered records still have to be
|
|
1694
|
+
* read, so a page cannot be bounded here, but a cursor prunes the common
|
|
1695
|
+
* polling case to the new records alone.
|
|
1696
|
+
*/
|
|
1697
|
+
async function coordinationEntries(stateRoot, runId, cursor) {
|
|
1698
|
+
const directory = coordinationDirectory(stateRoot, runId);
|
|
1699
|
+
return (await coordinationFiles(directory))
|
|
1700
|
+
.map((file) => ({ id: recordId(file), path: join(directory, file) }))
|
|
1701
|
+
.filter((entry) => cursor === undefined || entry.id > cursor);
|
|
1702
|
+
}
|
|
1703
|
+
/**
|
|
1704
|
+
* Reads entries in sequence order until the requested count or the page byte
|
|
1705
|
+
* bound is reached, and reports a cursor for the rest. A following page can be
|
|
1706
|
+
* empty, because a cursor is also returned when what remains behind it is only
|
|
1707
|
+
* entries this reader is never given.
|
|
1708
|
+
*
|
|
1709
|
+
* Both kinds share one journal, so a reader passes over entries it will never
|
|
1710
|
+
* return: the other kind, and another task's mail. Those are behind the cursor
|
|
1711
|
+
* once they have been examined, so a poller reads each of them once rather
|
|
1712
|
+
* than re-reading the whole journal on every call. An entry examined but held
|
|
1713
|
+
* back — the one that overflowed the page — stays ahead of the cursor. A
|
|
1714
|
+
* cursor therefore belongs to the query that produced it: reused under a
|
|
1715
|
+
* different filter or a different recipient, it starts past records that
|
|
1716
|
+
* filter would have matched.
|
|
1717
|
+
*
|
|
1718
|
+
* Every record is bounded when it is published, so a page always has room for
|
|
1719
|
+
* its first record and the byte bound holds without an exception for it.
|
|
1720
|
+
*/
|
|
1721
|
+
async function readCoordinationPage(entries, limit, select) {
|
|
1722
|
+
const records = [];
|
|
1723
|
+
// Account for the JSON array brackets and separators as well as records.
|
|
1724
|
+
let bytes = 0;
|
|
1725
|
+
let cursor;
|
|
1726
|
+
let examined;
|
|
1727
|
+
// Reached only with at least one record in the page, so the cursor is always
|
|
1728
|
+
// the identifier of a record the caller has now seen.
|
|
1729
|
+
const truncated = () => cursor === undefined ? { records } : { records, nextCursor: cursor };
|
|
1730
|
+
for (const entry of entries) {
|
|
1731
|
+
if (records.length >= limit)
|
|
1732
|
+
return truncated();
|
|
1733
|
+
const record = await select(entry);
|
|
1734
|
+
if (record === undefined) {
|
|
1735
|
+
examined = entry.id;
|
|
1736
|
+
continue;
|
|
1737
|
+
}
|
|
1738
|
+
const size = Buffer.byteLength(JSON.stringify(record));
|
|
1739
|
+
if (size > RUN_COORDINATION_MAX_RECORD_BYTES) {
|
|
1740
|
+
return invalidRecord(entry.path, "coordination record exceeds its serialized bound");
|
|
1741
|
+
}
|
|
1742
|
+
const pageBytes = bytes + size + (records.length === 0 ? 2 : 1);
|
|
1743
|
+
if (pageBytes > RUN_COORDINATION_MAX_PAGE_BYTES)
|
|
1744
|
+
return truncated();
|
|
1745
|
+
records.push(record);
|
|
1746
|
+
bytes = pageBytes;
|
|
1747
|
+
cursor = entry.id;
|
|
1748
|
+
examined = entry.id;
|
|
1749
|
+
}
|
|
1750
|
+
// Nothing was held back, so the position runs to the last entry examined.
|
|
1751
|
+
return examined === undefined || examined === cursor
|
|
1752
|
+
? { records }
|
|
1753
|
+
: { records, nextCursor: examined };
|
|
1754
|
+
}
|
|
1755
|
+
/**
|
|
1756
|
+
* Both kinds share the journal, so a reader has to look before validating.
|
|
1757
|
+
* Anything else in there is a tampered record, not a record to skip.
|
|
1758
|
+
*/
|
|
1759
|
+
function coordinationRecordKind(value, path) {
|
|
1760
|
+
if (!isRecord(value) ||
|
|
1761
|
+
(value.kind !== "run-event" && value.kind !== "run-message")) {
|
|
1762
|
+
return invalidRecord(path, "coordination record is neither an event nor a message");
|
|
1763
|
+
}
|
|
1764
|
+
return value.kind;
|
|
1765
|
+
}
|
|
1766
|
+
/**
|
|
1767
|
+
* A record names its own task rather than inheriting one from its path, so a
|
|
1768
|
+
* reader checks that the task is still one of the run's before returning it.
|
|
1769
|
+
*/
|
|
1770
|
+
function assertCoordinationTask(manifest, taskId, path) {
|
|
1771
|
+
if (!manifest.graph.tasks.some((task) => task.id === taskId)) {
|
|
1772
|
+
invalidRecord(path, "coordination record names a task outside its run");
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
export async function publishRunEvent(stateRoot, runId, input) {
|
|
1776
|
+
const event = parsePublishRunEvent(input);
|
|
1777
|
+
return withActiveRunPublication(stateRoot, runId, event.taskId, async (manifest, task) => {
|
|
1778
|
+
for (const recipient of event.recipients ?? []) {
|
|
1779
|
+
findTask(manifest, recipient);
|
|
1780
|
+
}
|
|
1781
|
+
const record = await publishCoordinationRecord(stateRoot, runId, (eventId) => ({
|
|
1782
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1783
|
+
kind: "run-event",
|
|
1784
|
+
eventId,
|
|
1785
|
+
runId,
|
|
1786
|
+
taskId: task.id,
|
|
1787
|
+
eventKind: event.eventKind,
|
|
1788
|
+
timestamp: new Date().toISOString(),
|
|
1789
|
+
message: event.message,
|
|
1790
|
+
...(event.paths === undefined ? {} : { paths: event.paths }),
|
|
1791
|
+
...(event.symbols === undefined ? {} : { symbols: event.symbols }),
|
|
1792
|
+
...(event.recipients === undefined
|
|
1793
|
+
? {}
|
|
1794
|
+
: { recipients: event.recipients }),
|
|
1795
|
+
}));
|
|
1796
|
+
return Object.freeze(record);
|
|
1797
|
+
});
|
|
1798
|
+
}
|
|
1799
|
+
export async function readRunEvents(stateRoot, runId, options) {
|
|
1800
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1801
|
+
const query = parseRunEventQuery(options);
|
|
1802
|
+
const entries = await coordinationEntries(stateRoot, manifest.runId, query.cursor);
|
|
1803
|
+
const page = await readCoordinationPage(entries, query.limit ?? RUN_COORDINATION_MAX_READ, async (entry) => {
|
|
1804
|
+
const value = await readJson(entry.path);
|
|
1805
|
+
if (coordinationRecordKind(value, entry.path) !== "run-event") {
|
|
1806
|
+
return undefined;
|
|
1807
|
+
}
|
|
1808
|
+
const record = validateRunEvent(value, entry.path);
|
|
1809
|
+
if (record.eventId !== entry.id || record.runId !== manifest.runId) {
|
|
1810
|
+
return invalidRecord(entry.path, "run event identity does not match its path");
|
|
1811
|
+
}
|
|
1812
|
+
assertCoordinationTask(manifest, record.taskId, entry.path);
|
|
1813
|
+
const matches = (query.eventKind === undefined ||
|
|
1814
|
+
record.eventKind === query.eventKind) &&
|
|
1815
|
+
(query.recipient === undefined ||
|
|
1816
|
+
record.recipients?.includes(query.recipient) === true) &&
|
|
1817
|
+
(query.path === undefined ||
|
|
1818
|
+
record.paths?.includes(query.path) === true) &&
|
|
1819
|
+
(query.symbol === undefined ||
|
|
1820
|
+
record.symbols?.includes(query.symbol) === true);
|
|
1821
|
+
return matches ? record : undefined;
|
|
1822
|
+
});
|
|
1823
|
+
return page.nextCursor === undefined
|
|
1824
|
+
? { events: Object.freeze(page.records) }
|
|
1825
|
+
: { events: Object.freeze(page.records), nextCursor: page.nextCursor };
|
|
1826
|
+
}
|
|
1827
|
+
export async function sendRunMessage(stateRoot, runId, input) {
|
|
1828
|
+
const message = parsePublishRunMessage(input);
|
|
1829
|
+
return withActiveRunPublication(stateRoot, runId, message.senderTaskId, async (manifest, sender) => {
|
|
1830
|
+
const recipient = findTask(manifest, message.recipientTaskId);
|
|
1831
|
+
const record = await publishCoordinationRecord(stateRoot, runId, (messageId) => ({
|
|
1832
|
+
schemaVersion: SCHEMA_VERSION,
|
|
1833
|
+
kind: "run-message",
|
|
1834
|
+
messageId,
|
|
1835
|
+
runId,
|
|
1836
|
+
senderTaskId: sender.id,
|
|
1837
|
+
recipientTaskId: recipient.id,
|
|
1838
|
+
timestamp: new Date().toISOString(),
|
|
1839
|
+
message: message.message,
|
|
1840
|
+
}));
|
|
1841
|
+
return Object.freeze(record);
|
|
1842
|
+
});
|
|
1843
|
+
}
|
|
1844
|
+
export async function readRunMessages(stateRoot, runId, recipientTaskId, options) {
|
|
1845
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1846
|
+
const recipient = findTask(manifest, recipientTaskId);
|
|
1847
|
+
const query = parseRunMessageQuery(options);
|
|
1848
|
+
const entries = await coordinationEntries(stateRoot, manifest.runId, query.cursor);
|
|
1849
|
+
const page = await readCoordinationPage(entries, query.limit ?? RUN_COORDINATION_MAX_READ, async (entry) => {
|
|
1850
|
+
const value = await readJson(entry.path);
|
|
1851
|
+
if (coordinationRecordKind(value, entry.path) !== "run-message") {
|
|
1852
|
+
return undefined;
|
|
1853
|
+
}
|
|
1854
|
+
const record = validateRunMessage(value, entry.path);
|
|
1855
|
+
if (record.messageId !== entry.id || record.runId !== manifest.runId) {
|
|
1856
|
+
return invalidRecord(entry.path, "run message identity does not match its path");
|
|
1857
|
+
}
|
|
1858
|
+
assertCoordinationTask(manifest, record.senderTaskId, entry.path);
|
|
1859
|
+
assertCoordinationTask(manifest, record.recipientTaskId, entry.path);
|
|
1860
|
+
// An inbox is bound to the reading worker, so a record addressed to
|
|
1861
|
+
// another task is skipped rather than refused.
|
|
1862
|
+
return record.recipientTaskId === recipient.id ? record : undefined;
|
|
1863
|
+
});
|
|
1864
|
+
return page.nextCursor === undefined
|
|
1865
|
+
? { messages: Object.freeze(page.records) }
|
|
1866
|
+
: { messages: Object.freeze(page.records), nextCursor: page.nextCursor };
|
|
1867
|
+
}
|
|
1868
|
+
/**
|
|
1869
|
+
* Reads the retained text artifact of a task's current attempt.
|
|
1870
|
+
*
|
|
1871
|
+
* The artifact is only readable through the output that claims it, and only
|
|
1872
|
+
* while that output itself agrees with the node's current state: an artifact
|
|
1873
|
+
* left behind by an interrupted publication has no reference vouching for it,
|
|
1874
|
+
* and is reported as inconsistent rather than returned as if it were current.
|
|
1875
|
+
*/
|
|
1876
|
+
export async function readNodeArtifact(stateRoot, runId, taskId) {
|
|
1877
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1878
|
+
const task = findTask(manifest, taskId);
|
|
1879
|
+
const [record, output] = await Promise.all([
|
|
1880
|
+
readNodeArtifactRecord(stateRoot, manifest, task),
|
|
1881
|
+
readNodeOutput(stateRoot, runId, taskId),
|
|
1882
|
+
]);
|
|
1883
|
+
if (output.artifact === undefined ||
|
|
1884
|
+
record.attempt !== output.attempt ||
|
|
1885
|
+
output.artifact.bytes !== Buffer.byteLength(record.text, "utf8")) {
|
|
1886
|
+
throw new RunStoreError("invalid_record", "Node artifact does not match the published node output", artifactPath(stateRoot, runId, task.key));
|
|
1887
|
+
}
|
|
1888
|
+
return record;
|
|
1889
|
+
}
|
|
1890
|
+
function addUsage(total, usage) {
|
|
1891
|
+
return {
|
|
1892
|
+
turns: total.turns + usage.turns,
|
|
1893
|
+
input: total.input + usage.input,
|
|
1894
|
+
output: total.output + usage.output,
|
|
1895
|
+
cacheRead: total.cacheRead + usage.cacheRead,
|
|
1896
|
+
cacheWrite: total.cacheWrite + usage.cacheWrite,
|
|
1897
|
+
totalTokens: total.totalTokens + usage.totalTokens,
|
|
1898
|
+
cost: {
|
|
1899
|
+
input: total.cost.input + usage.cost.input,
|
|
1900
|
+
output: total.cost.output + usage.cost.output,
|
|
1901
|
+
cacheRead: total.cost.cacheRead + usage.cost.cacheRead,
|
|
1902
|
+
cacheWrite: total.cost.cacheWrite + usage.cost.cacheWrite,
|
|
1903
|
+
total: total.cost.total + usage.cost.total,
|
|
1904
|
+
},
|
|
1905
|
+
};
|
|
1906
|
+
}
|
|
1907
|
+
/**
|
|
1908
|
+
* Sums what one retained run spent, from the outputs it published.
|
|
1909
|
+
*
|
|
1910
|
+
* Derived rather than stored, so it is correct for a run that was interrupted
|
|
1911
|
+
* and needs no record kept in step with the outputs. A task with no readable
|
|
1912
|
+
* output contributes nothing and is named in `unaccounted`: an unknown total
|
|
1913
|
+
* must not read as a cheap one.
|
|
1914
|
+
*/
|
|
1915
|
+
export async function readRunUsage(stateRoot, runId) {
|
|
1916
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1917
|
+
const tasks = [];
|
|
1918
|
+
const unaccounted = [];
|
|
1919
|
+
let total = {
|
|
1920
|
+
turns: 0,
|
|
1921
|
+
input: 0,
|
|
1922
|
+
output: 0,
|
|
1923
|
+
cacheRead: 0,
|
|
1924
|
+
cacheWrite: 0,
|
|
1925
|
+
totalTokens: 0,
|
|
1926
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
1927
|
+
};
|
|
1928
|
+
for (const task of manifest.graph.tasks) {
|
|
1929
|
+
const state = await readNodeStateRecord(stateRoot, manifest, task);
|
|
1930
|
+
let usage;
|
|
1931
|
+
let published = true;
|
|
1932
|
+
try {
|
|
1933
|
+
const record = await readNodeOutputRecord(stateRoot, manifest, task);
|
|
1934
|
+
// The same agreement every other reader requires. A malformed or
|
|
1935
|
+
// inconsistent record is reported, not quietly counted as nothing:
|
|
1936
|
+
// only a task that published no output at all is an expected absence.
|
|
1937
|
+
assertOutputMatchesState(record, state);
|
|
1938
|
+
usage = record.usage;
|
|
1939
|
+
}
|
|
1940
|
+
catch (error) {
|
|
1941
|
+
if (!(error instanceof RunStoreError) || error.code !== "not_found") {
|
|
1942
|
+
throw error;
|
|
1943
|
+
}
|
|
1944
|
+
published = false;
|
|
1945
|
+
}
|
|
1946
|
+
const accounting = usage !== undefined
|
|
1947
|
+
? "reported"
|
|
1948
|
+
: !published &&
|
|
1949
|
+
(state.status === "pending" || state.status === "blocked")
|
|
1950
|
+
? "not_started"
|
|
1951
|
+
: "unreported";
|
|
1952
|
+
if (accounting === "unreported")
|
|
1953
|
+
unaccounted.push(task.id);
|
|
1954
|
+
if (usage !== undefined)
|
|
1955
|
+
total = addUsage(total, usage);
|
|
1956
|
+
tasks.push(Object.freeze({
|
|
1957
|
+
taskId: task.id,
|
|
1958
|
+
status: state.status,
|
|
1959
|
+
accounting,
|
|
1960
|
+
usage,
|
|
1961
|
+
}));
|
|
1962
|
+
}
|
|
1963
|
+
return Object.freeze({
|
|
1964
|
+
runId: manifest.runId,
|
|
1965
|
+
total: Object.freeze({ ...total, cost: Object.freeze({ ...total.cost }) }),
|
|
1966
|
+
unaccounted: Object.freeze(unaccounted),
|
|
1967
|
+
tasks: Object.freeze(tasks),
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
/**
|
|
1971
|
+
* Every reader of a published output applies this: an output for a different
|
|
1972
|
+
* attempt, for a node settled as blocked, or disagreeing with a terminal node
|
|
1973
|
+
* status describes state no reader may act on.
|
|
1974
|
+
*/
|
|
1975
|
+
function assertOutputMatchesState(record, state) {
|
|
1976
|
+
if (record.attempt !== state.attempt ||
|
|
1977
|
+
state.status === "blocked" ||
|
|
1978
|
+
(OUTPUT_STATUSES.has(state.status) &&
|
|
1979
|
+
record.status !== state.status)) {
|
|
1980
|
+
throw new RunStoreError("invalid_record", "Node output does not match current node state");
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
export async function readNodeOutput(stateRoot, runId, taskId) {
|
|
1984
|
+
const manifest = await readRun(stateRoot, runId);
|
|
1985
|
+
const task = findTask(manifest, taskId);
|
|
1986
|
+
const [record, state] = await Promise.all([
|
|
1987
|
+
readNodeOutputRecord(stateRoot, manifest, task),
|
|
1988
|
+
readNodeStateRecord(stateRoot, manifest, task),
|
|
1989
|
+
]);
|
|
1990
|
+
assertOutputMatchesState(record, state);
|
|
1991
|
+
return record;
|
|
1992
|
+
}
|
|
1993
|
+
//# sourceMappingURL=store.js.map
|