pi-better-subagents 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/assignment-outcomes.ts +157 -0
- package/assignment-projection.d.mts +14 -0
- package/assignment-projection.mjs +109 -0
- package/cleanup.ts +12 -0
- package/docs/usage.md +14 -0
- package/finalization.ts +2 -0
- package/health-surface.mjs +2 -0
- package/historical-attention.ts +35 -0
- package/index.ts +206 -48
- package/list.mjs +7 -1
- package/navigator.mjs +23 -10
- package/outage-observations.ts +73 -0
- package/output-payload.ts +25 -6
- package/package.json +3 -1
- package/registry.ts +16 -3
- package/session-store.ts +1 -1
- package/shared-callback-batcher.ts +8 -3
- package/shared-operation-coordinator/assignment-observation.mjs +43 -0
- package/shared-operation-coordinator/index.ts +5 -0
- package/shared-operation-coordinator/src/assignment-outcomes.ts +172 -0
- package/shared-operation-coordinator/src/discovery.ts +128 -0
- package/shared-operation-coordinator/src/errors.ts +2 -0
- package/shared-operation-coordinator/src/operations.ts +88 -6
- package/shared-operation-coordinator/src/outbox.ts +11 -2
- package/shared-operation-coordinator/src/recovery.ts +55 -0
- package/shared-operation-coordinator/src/rows.ts +2 -1
- package/shared-operation-coordinator/src/schema.ts +114 -2
- package/shared-operation-coordinator/src/store.ts +83 -1
- package/shared-operation-coordinator/src/types.ts +46 -1
- package/shared-operation-coordinator/src/waits.ts +47 -5
- package/task-policy.ts +2 -0
- package/tools.ts +24 -7
- package/worker-waits.ts +258 -94
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// Generated from packages/operation-coordinator/src/assignment-outcomes.ts. Do not edit directly.
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { StoreError } from "./errors.ts";
|
|
4
|
+
import { columnText, getRow, getRows, run, type StoreContext } from "./sqlite.ts";
|
|
5
|
+
import { requireWait, readPins } from "./rows.ts";
|
|
6
|
+
import { releaseGenerationPins, voidGenerationOutbox } from "./outbox.ts";
|
|
7
|
+
import { normalizeOwner, requireId, requireText, sameOwner, requireCounter, requireEnum } from "./validate.ts";
|
|
8
|
+
import type { EvidenceRefInput, OwnerIdentity, WaitRef } from "./types.ts";
|
|
9
|
+
|
|
10
|
+
export interface AssignmentOutcomeInput {
|
|
11
|
+
assignmentId: string;
|
|
12
|
+
owner: OwnerIdentity;
|
|
13
|
+
recipient: OwnerIdentity;
|
|
14
|
+
revision: number;
|
|
15
|
+
outcome: "succeeded" | "failed" | "cancelled";
|
|
16
|
+
reason: string;
|
|
17
|
+
evidence: EvidenceRefInput[];
|
|
18
|
+
waits: Array<WaitRef & { generation: number; cancelEpoch: number }>;
|
|
19
|
+
/** Reporting a terminal assignment never proves termination. */
|
|
20
|
+
processUncertain: boolean;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface AssignmentOutcomeRecord extends AssignmentOutcomeInput {
|
|
24
|
+
eventId: string;
|
|
25
|
+
inspectedAt: number | null;
|
|
26
|
+
disposition: { kind: "consumed" | "dismissed"; note: string; at: number } | null;
|
|
27
|
+
deliveryReceipt: { entryId: string; at: number } | null;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function registerAssignment(ctx: StoreContext, input: Pick<AssignmentOutcomeInput, "assignmentId" | "owner" | "recipient"> & { attemptId?: string }): void {
|
|
31
|
+
const id = requireId("assignmentId", input.assignmentId);
|
|
32
|
+
const owner = normalizeOwner("owner", input.owner);
|
|
33
|
+
const recipient = normalizeOwner("recipient", input.recipient);
|
|
34
|
+
if (owner.kind !== "worker-conversation" || recipient.kind !== "parent-session") throw new StoreError("foreign-owner", "Assignment needs a worker and recorded parent.");
|
|
35
|
+
ctx.transaction(() => {
|
|
36
|
+
const existing = getRow(ctx, "SELECT * FROM assignment_bindings WHERE assignment_id = ?", id);
|
|
37
|
+
if (existing) {
|
|
38
|
+
if (!sameOwner(JSON.parse(columnText(existing, "owner_json")), owner) || !sameOwner(JSON.parse(columnText(existing, "recipient_json")), recipient)) throw new StoreError("foreign-owner", "Assignment ownership and recipient are immutable.");
|
|
39
|
+
} else {
|
|
40
|
+
run(ctx, "INSERT INTO assignment_bindings VALUES (?, ?, ?)", id, JSON.stringify(owner), JSON.stringify(recipient));
|
|
41
|
+
}
|
|
42
|
+
if (input.attemptId) {
|
|
43
|
+
const attemptId = requireId("attemptId", input.attemptId);
|
|
44
|
+
const attempt = getRow(ctx, "SELECT assignment_id FROM assignment_attempts WHERE attempt_id = ?", attemptId);
|
|
45
|
+
if (attempt && attempt.assignment_id !== id) throw new StoreError("foreign-owner", "Attempt belongs to another assignment.");
|
|
46
|
+
if (!attempt && getAssignmentOutcome(ctx, id)) throw new StoreError("validation", "A terminal assignment cannot admit a new attempt.");
|
|
47
|
+
run(ctx, "INSERT OR IGNORE INTO assignment_attempts VALUES (?, ?)", attemptId, id);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function getAssignmentOutcome(ctx: StoreContext, assignmentId: string): AssignmentOutcomeRecord | null {
|
|
53
|
+
const row = getRow(ctx, "SELECT record_json FROM assignment_outcomes WHERE assignment_id = ?", requireId("assignmentId", assignmentId));
|
|
54
|
+
return row ? JSON.parse(columnText(row, "record_json")) as AssignmentOutcomeRecord : null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function save(ctx: StoreContext, record: AssignmentOutcomeRecord): void {
|
|
58
|
+
run(ctx, "UPDATE assignment_outcomes SET record_json = ? WHERE assignment_id = ?", JSON.stringify(record), record.assignmentId);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function recordAssignmentOutcome(ctx: StoreContext, input: AssignmentOutcomeInput): AssignmentOutcomeRecord {
|
|
62
|
+
const assignmentId = requireId("assignmentId", input.assignmentId);
|
|
63
|
+
const owner = normalizeOwner("owner", input.owner);
|
|
64
|
+
const recipient = normalizeOwner("recipient", input.recipient);
|
|
65
|
+
if (owner.kind !== "worker-conversation" || recipient.kind !== "parent-session") {
|
|
66
|
+
throw new StoreError("foreign-owner", "An assignment outcome needs its worker and recorded parent in the same workspace.");
|
|
67
|
+
}
|
|
68
|
+
const revision = requireCounter("revision", input.revision);
|
|
69
|
+
if (revision < 1) throw new StoreError("validation", "Outcome revision must be positive.");
|
|
70
|
+
const outcome = requireEnum("outcome", input.outcome, ["succeeded", "failed", "cancelled"] as const);
|
|
71
|
+
const reason = requireText("reason", input.reason, 8192);
|
|
72
|
+
if (typeof input.processUncertain !== "boolean" || !Array.isArray(input.evidence) || input.evidence.length > 100
|
|
73
|
+
|| !Array.isArray(input.waits) || input.waits.length > 100) throw new StoreError("validation", "Invalid outcome evidence or wait bounds.");
|
|
74
|
+
const evidence = input.evidence.map((e) => ({
|
|
75
|
+
kind: requireEnum("evidence.kind", e.kind, ["terminal-record", "log", "summary", "artifact"] as const),
|
|
76
|
+
ref: requireText("evidence.ref", e.ref, 1024),
|
|
77
|
+
...(e.summary === undefined ? {} : { summary: requireText("evidence.summary", e.summary, 2048) }),
|
|
78
|
+
detailLost: e.detailLost === true,
|
|
79
|
+
...(e.bytes === undefined ? {} : { bytes: requireCounter("evidence.bytes", e.bytes) }),
|
|
80
|
+
}));
|
|
81
|
+
const waits = input.waits.map((w) => ({ waitId: requireId("waitId", w.waitId), generation: requireCounter("generation", w.generation), cancelEpoch: requireCounter("cancelEpoch", w.cancelEpoch) }));
|
|
82
|
+
const normalized = { assignmentId, owner, recipient, revision, outcome, reason, evidence, waits, processUncertain: input.processUncertain };
|
|
83
|
+
return ctx.transaction(() => {
|
|
84
|
+
const binding = getRow(ctx, "SELECT * FROM assignment_bindings WHERE assignment_id = ?", assignmentId);
|
|
85
|
+
if (!binding || !sameOwner(JSON.parse(columnText(binding, "owner_json")), owner)
|
|
86
|
+
|| !sameOwner(JSON.parse(columnText(binding, "recipient_json")), recipient)) throw new StoreError("foreign-owner", "Outcome must match the assignment's captured worker and recipient.");
|
|
87
|
+
const existing = getAssignmentOutcome(ctx, assignmentId);
|
|
88
|
+
if (existing) {
|
|
89
|
+
const { eventId: _event, inspectedAt: _read, disposition: _disposition, deliveryReceipt: _receipt, ...decision } = existing;
|
|
90
|
+
if (JSON.stringify(decision) !== JSON.stringify(normalized)) throw new StoreError("validation", "A terminal assignment cannot be rewritten or reopened.");
|
|
91
|
+
return existing;
|
|
92
|
+
}
|
|
93
|
+
const linked = waits.map((ref) => {
|
|
94
|
+
const wait = requireWait(ctx, ref.waitId, ref.generation);
|
|
95
|
+
if (!sameOwner(wait.owner, owner) || wait.origin?.assignmentRef !== assignmentId || wait.cancelEpoch !== ref.cancelEpoch) {
|
|
96
|
+
throw new StoreError("foreign-owner", "Outcome wait relationship or cancellation epoch does not match the assignment.");
|
|
97
|
+
}
|
|
98
|
+
if (outcome === "succeeded" && wait.acknowledgement === null) throw new StoreError("wait-phase", "Success requires dependency interpretation.");
|
|
99
|
+
return wait;
|
|
100
|
+
});
|
|
101
|
+
// Reject omission of a known assignment wait: every dispatch path must close.
|
|
102
|
+
for (const row of getRows(ctx, "SELECT wait_id, generation FROM waits WHERE owner_kind = ? AND owner_session_id = ? AND owner_workspace = ? AND json_extract(origin_json, '$.assignmentRef') = ?", owner.kind, owner.sessionId, owner.workspace, assignmentId)) {
|
|
103
|
+
if (!waits.some((w) => w.waitId === row.wait_id && w.generation === row.generation)) throw new StoreError("validation", "Outcome omitted an assignment wait.");
|
|
104
|
+
}
|
|
105
|
+
const eventId = `assignment_${createHash("sha256").update(assignmentId).digest("hex")}`;
|
|
106
|
+
const record: AssignmentOutcomeRecord = { ...normalized, eventId, inspectedAt: null, disposition: null, deliveryReceipt: null };
|
|
107
|
+
const at = ctx.now();
|
|
108
|
+
run(ctx, "INSERT INTO assignment_outcomes VALUES (?, ?, ?)", assignmentId, eventId, JSON.stringify(record));
|
|
109
|
+
const retained = evidence.map((e) => ({ ...e, operationId: assignmentId, terminalRevision: revision }));
|
|
110
|
+
for (const wait of linked) {
|
|
111
|
+
run(ctx, "INSERT INTO assignment_outcome_waits VALUES (?, ?, ?)", assignmentId, wait.waitId, wait.generation);
|
|
112
|
+
retained.push(...readPins(ctx, wait.waitId, wait.generation, false).map((pin) => ({ operationId: pin.operationId, terminalRevision: pin.terminalRevision, kind: pin.kind, ref: pin.ref, summary: pin.summary ?? undefined, detailLost: pin.detailLost, bytes: pin.bytes ?? undefined })));
|
|
113
|
+
releaseGenerationPins(ctx, wait.waitId, wait.generation, `transferred:${eventId}`, at);
|
|
114
|
+
voidGenerationOutbox(ctx, wait.waitId, wait.generation, "assignment-terminal", at);
|
|
115
|
+
if (!["consumed", "cancelled"].includes(wait.phase)) {
|
|
116
|
+
run(ctx, "UPDATE waits SET dispatch_disarmed = 1, phase = 'held', phase_reason = ?, updated_at = ? WHERE wait_id = ? AND generation = ?", `assignment-terminal:${outcome}`, at, wait.waitId, wait.generation);
|
|
117
|
+
}
|
|
118
|
+
// A published claim may still own a writer. Keep its lease and launch identity.
|
|
119
|
+
run(ctx, "UPDATE resume_claims SET state = 'invalidated', invalidated_at = ?, reason = 'assignment-terminal' WHERE wait_id = ? AND generation = ? AND state = 'claimed'", at, wait.waitId, wait.generation);
|
|
120
|
+
}
|
|
121
|
+
for (const e of retained) run(ctx, `INSERT OR IGNORE INTO evidence_pins (pin_id, wait_id, generation, operation_id, terminal_revision, kind, ref, summary, detail_lost, bytes, pinned_at) VALUES (?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?)`, `pin_${randomUUID()}`, eventId, e.operationId, e.terminalRevision, e.kind, e.ref, e.summary ?? null, e.detailLost ? 1 : 0, e.bytes ?? null, at);
|
|
122
|
+
run(ctx, `INSERT INTO outbox_events (event_id, event_type, operation_id, terminal_revision, wait_id, generation, recipient_kind, recipient_session_id, recipient_workspace, recipient_branch_id, recipient_goal_id, state, created_at) VALUES (?, 'assignment-outcome', ?, ?, ?, 1, ?, ?, ?, ?, ?, 'pending', ?)`, eventId, assignmentId, revision, eventId, recipient.kind, recipient.sessionId, recipient.workspace, recipient.branchId ?? null, recipient.goalId ?? null, at);
|
|
123
|
+
return record;
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function inspectAssignmentOutcome(ctx: StoreContext, input: { assignmentId: string; recipient: OwnerIdentity; revision: number }): AssignmentOutcomeRecord {
|
|
128
|
+
return ctx.transaction(() => {
|
|
129
|
+
const record = requireRecipient(ctx, input);
|
|
130
|
+
record.inspectedAt ??= ctx.now();
|
|
131
|
+
save(ctx, record);
|
|
132
|
+
return record;
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Call after reconciling the stable event ID with the recipient's durable transcript. */
|
|
137
|
+
export function receiptAssignmentOutcome(ctx: StoreContext, input: { assignmentId: string; recipient: OwnerIdentity; revision: number; entryId: string }): AssignmentOutcomeRecord {
|
|
138
|
+
const entryId = requireId("entryId", input.entryId);
|
|
139
|
+
return ctx.transaction(() => {
|
|
140
|
+
const record = requireRecipient(ctx, input);
|
|
141
|
+
if (record.deliveryReceipt) return record;
|
|
142
|
+
record.deliveryReceipt = { entryId, at: ctx.now() };
|
|
143
|
+
save(ctx, record);
|
|
144
|
+
run(ctx, "UPDATE outbox_events SET state = CASE WHEN state = 'acknowledged' THEN state ELSE 'delivered' END, delivered_at = ?, dispatch_claim_json = NULL WHERE event_id = ?", ctx.now(), record.eventId);
|
|
145
|
+
return record;
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function requireRecipient(ctx: StoreContext, input: { assignmentId: string; recipient: OwnerIdentity; revision: number }): AssignmentOutcomeRecord {
|
|
150
|
+
const record = getAssignmentOutcome(ctx, input.assignmentId);
|
|
151
|
+
if (!record || !sameOwner(record.recipient, normalizeOwner("recipient", input.recipient))) throw new StoreError("foreign-owner", "Outcome belongs to a different parent.");
|
|
152
|
+
if (record.revision !== input.revision) throw new StoreError("stale-evidence", "Outcome revision does not match.");
|
|
153
|
+
return record;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export function consumeAssignmentOutcome(ctx: StoreContext, input: { assignmentId: string; recipient: OwnerIdentity; revision: number; disposition: "consumed" | "dismissed"; note: string }): AssignmentOutcomeRecord {
|
|
157
|
+
const kind = requireEnum("disposition", input.disposition, ["consumed", "dismissed"] as const);
|
|
158
|
+
const note = requireText("note", input.note, 8192);
|
|
159
|
+
return ctx.transaction(() => {
|
|
160
|
+
const record = requireRecipient(ctx, input);
|
|
161
|
+
if (record.inspectedAt === null) throw new StoreError("stale-evidence", "Inspect the exact outcome before disposing it.");
|
|
162
|
+
if (record.disposition) {
|
|
163
|
+
if (record.disposition.kind !== kind || record.disposition.note !== note) throw new StoreError("validation", "Outcome disposition already recorded.");
|
|
164
|
+
return record;
|
|
165
|
+
}
|
|
166
|
+
record.disposition = { kind, note, at: ctx.now() };
|
|
167
|
+
save(ctx, record);
|
|
168
|
+
releaseGenerationPins(ctx, record.eventId, 1, `outcome-${kind}`, ctx.now());
|
|
169
|
+
run(ctx, "UPDATE outbox_events SET state = 'acknowledged', acknowledged_at = ?, dispatch_claim_json = NULL WHERE event_id = ?", ctx.now(), record.eventId);
|
|
170
|
+
return record;
|
|
171
|
+
});
|
|
172
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// Generated from packages/operation-coordinator/src/discovery.ts. Do not edit directly.
|
|
2
|
+
/**
|
|
3
|
+
* Cross-process discovery coordination.
|
|
4
|
+
*
|
|
5
|
+
* Per-operation `supervision` authority decides who may inspect an operation.
|
|
6
|
+
* A separate discovery claim decides which process pays for the global running
|
|
7
|
+
* scan and sidecar directory walk. Followers read only their indexed assigned
|
|
8
|
+
* operations and cheap change-notification state.
|
|
9
|
+
*
|
|
10
|
+
* The claim is not a lease with elapsed-time expiry: replacement is proven by
|
|
11
|
+
* process identity or an exact-token release. Acquisition is compare-and-swap
|
|
12
|
+
* against the observed claim, so simultaneous contenders cannot both scan.
|
|
13
|
+
*/
|
|
14
|
+
import { StoreError } from "./errors.ts";
|
|
15
|
+
import {
|
|
16
|
+
columnNumber,
|
|
17
|
+
columnText,
|
|
18
|
+
getRow,
|
|
19
|
+
run,
|
|
20
|
+
type StoreContext,
|
|
21
|
+
} from "./sqlite.ts";
|
|
22
|
+
import { requireId } from "./validate.ts";
|
|
23
|
+
import type {
|
|
24
|
+
DiscoveryClaimRecord,
|
|
25
|
+
ReleaseDiscoveryClaimInput,
|
|
26
|
+
TryClaimDiscoveryInput,
|
|
27
|
+
} from "./types.ts";
|
|
28
|
+
|
|
29
|
+
const CHANGE_REVISION_KEY = "change_revision";
|
|
30
|
+
const DISCOVERY_TOKEN_LIMIT = 200;
|
|
31
|
+
|
|
32
|
+
function mapClaim(row: Record<string, unknown>): DiscoveryClaimRecord {
|
|
33
|
+
return {
|
|
34
|
+
scope: columnText(row, "scope"),
|
|
35
|
+
hostId: columnText(row, "host_id"),
|
|
36
|
+
token: columnText(row, "token"),
|
|
37
|
+
claimedAt: columnNumber(row, "claimed_at"),
|
|
38
|
+
updatedAt: columnNumber(row, "updated_at"),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function readClaim(ctx: StoreContext, scope: string): DiscoveryClaimRecord | null {
|
|
43
|
+
const row = getRow(ctx, "SELECT * FROM discovery_claims WHERE scope = ?", scope);
|
|
44
|
+
return row ? mapClaim(row) : null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function readDiscoveryClaim(ctx: StoreContext, scope: unknown): DiscoveryClaimRecord | null {
|
|
48
|
+
return readClaim(ctx, requireId("scope", scope));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function tryClaimDiscovery(
|
|
52
|
+
ctx: StoreContext,
|
|
53
|
+
input: TryClaimDiscoveryInput,
|
|
54
|
+
): DiscoveryClaimRecord | null {
|
|
55
|
+
const scope = requireId("scope", input.scope);
|
|
56
|
+
const hostId = requireId("hostId", input.hostId);
|
|
57
|
+
const token = requireId("token", input.token);
|
|
58
|
+
if (token.length > DISCOVERY_TOKEN_LIMIT) {
|
|
59
|
+
throw new StoreError("validation", `Invalid token: must be at most ${DISCOVERY_TOKEN_LIMIT} characters`);
|
|
60
|
+
}
|
|
61
|
+
const expectedHostId = input.expected === null ? null : requireId("expected.hostId", input.expected.hostId);
|
|
62
|
+
const expectedToken = input.expected === null ? null : requireId("expected.token", input.expected.token);
|
|
63
|
+
|
|
64
|
+
return ctx.transaction(() => {
|
|
65
|
+
const current = readClaim(ctx, scope);
|
|
66
|
+
if (input.expected === null) {
|
|
67
|
+
if (current) return null;
|
|
68
|
+
} else if (!current || current.hostId !== expectedHostId || current.token !== expectedToken) {
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
const at = ctx.now();
|
|
72
|
+
run(
|
|
73
|
+
ctx,
|
|
74
|
+
`INSERT INTO discovery_claims (scope, host_id, token, claimed_at, updated_at)
|
|
75
|
+
VALUES (?, ?, ?, ?, ?)
|
|
76
|
+
ON CONFLICT(scope) DO UPDATE SET
|
|
77
|
+
host_id = excluded.host_id,
|
|
78
|
+
token = excluded.token,
|
|
79
|
+
claimed_at = excluded.claimed_at,
|
|
80
|
+
updated_at = excluded.updated_at`,
|
|
81
|
+
scope,
|
|
82
|
+
hostId,
|
|
83
|
+
token,
|
|
84
|
+
at,
|
|
85
|
+
at,
|
|
86
|
+
);
|
|
87
|
+
return readClaim(ctx, scope);
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export function releaseDiscoveryClaim(
|
|
92
|
+
ctx: StoreContext,
|
|
93
|
+
input: ReleaseDiscoveryClaimInput,
|
|
94
|
+
): boolean {
|
|
95
|
+
const scope = requireId("scope", input.scope);
|
|
96
|
+
const hostId = requireId("hostId", input.hostId);
|
|
97
|
+
const token = requireId("token", input.token);
|
|
98
|
+
return ctx.transaction(() =>
|
|
99
|
+
run(
|
|
100
|
+
ctx,
|
|
101
|
+
"DELETE FROM discovery_claims WHERE scope = ? AND host_id = ? AND token = ?",
|
|
102
|
+
scope,
|
|
103
|
+
hostId,
|
|
104
|
+
token,
|
|
105
|
+
) > 0
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Monotonic revision for registration, authority changes and terminal publication. */
|
|
110
|
+
export function getChangeRevision(ctx: StoreContext): number {
|
|
111
|
+
const row = getRow(ctx, "SELECT value FROM coordinator_state WHERE key = ?", CHANGE_REVISION_KEY);
|
|
112
|
+
if (!row) return 0;
|
|
113
|
+
const revision = Number(columnText(row, "value"));
|
|
114
|
+
if (!Number.isSafeInteger(revision) || revision < 0) {
|
|
115
|
+
throw new StoreError("store-corruption", "The coordinator change revision is not a non-negative integer.");
|
|
116
|
+
}
|
|
117
|
+
return revision;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Called only inside an existing store transaction. */
|
|
121
|
+
export function bumpChangeRevision(ctx: StoreContext): void {
|
|
122
|
+
run(
|
|
123
|
+
ctx,
|
|
124
|
+
`INSERT INTO coordinator_state (key, value) VALUES (?, '1')
|
|
125
|
+
ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + 1 AS TEXT)`,
|
|
126
|
+
CHANGE_REVISION_KEY,
|
|
127
|
+
);
|
|
128
|
+
}
|
|
@@ -25,6 +25,8 @@ export type StoreErrorCode =
|
|
|
25
25
|
| "task-ownership"
|
|
26
26
|
/** The caller is not the recorded owner of the record it wants to change. */
|
|
27
27
|
| "foreign-owner"
|
|
28
|
+
/** Another host took supervision after the caller read the operation. */
|
|
29
|
+
| "supervision-conflict"
|
|
28
30
|
/** A different terminal event is already recorded for this operation. */
|
|
29
31
|
| "terminal-conflict"
|
|
30
32
|
/** The owner already has a pending wait generation. */
|
|
@@ -25,6 +25,7 @@ import type {
|
|
|
25
25
|
ObserverHealth,
|
|
26
26
|
OperationFilter,
|
|
27
27
|
OperationRecord,
|
|
28
|
+
ReleaseSupervisionInput,
|
|
28
29
|
RecordSupervisionInput,
|
|
29
30
|
RecordTerminalOutcomeInput,
|
|
30
31
|
RecordTerminalOutcomeResult,
|
|
@@ -49,6 +50,7 @@ import {
|
|
|
49
50
|
} from "./validate.ts";
|
|
50
51
|
import { mapOutbox } from "./rows.ts";
|
|
51
52
|
import { settleWaitsForOperation } from "./waits.ts";
|
|
53
|
+
import { bumpChangeRevision } from "./discovery.ts";
|
|
52
54
|
|
|
53
55
|
const HEALTH_VALUES: readonly ObserverHealth[] = ["active", "unmonitored", "lost"];
|
|
54
56
|
const OUTCOME_VALUES: readonly TerminalOutcome[] = ["succeeded", "failed", "cancelled", "lost"];
|
|
@@ -158,9 +160,9 @@ export function registerOperation(
|
|
|
158
160
|
`INSERT INTO operations (
|
|
159
161
|
operation_id, task_id, launch_token,
|
|
160
162
|
owner_kind, owner_session_id, owner_workspace, owner_branch_id, owner_goal_id,
|
|
161
|
-
observer_kind, observer_version, supervision_json, authority_ref, intent_ref,
|
|
163
|
+
observer_kind, observer_version, supervision_json, supervision_host_id, authority_ref, intent_ref,
|
|
162
164
|
phase, health, health_reason, terminal_revision, created_at, updated_at
|
|
163
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', 'active', NULL, 0, ?, ?)`,
|
|
165
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', 'active', NULL, 0, ?, ?)`,
|
|
164
166
|
operationId,
|
|
165
167
|
taskId,
|
|
166
168
|
launchToken,
|
|
@@ -172,6 +174,7 @@ export function registerOperation(
|
|
|
172
174
|
observer.kind,
|
|
173
175
|
observer.version,
|
|
174
176
|
supervision ? JSON.stringify(supervision) : null,
|
|
177
|
+
supervision?.hostId ?? null,
|
|
175
178
|
authorityRef,
|
|
176
179
|
intentRef,
|
|
177
180
|
at,
|
|
@@ -190,6 +193,7 @@ export function registerOperation(
|
|
|
190
193
|
assertTaskAvailable();
|
|
191
194
|
throw error;
|
|
192
195
|
}
|
|
196
|
+
bumpChangeRevision(ctx);
|
|
193
197
|
return { operation: requireOperation(ctx, operationId), created: true };
|
|
194
198
|
});
|
|
195
199
|
}
|
|
@@ -205,6 +209,11 @@ export function recordSupervision(
|
|
|
205
209
|
const operationId = requireId("operationId", input.operationId);
|
|
206
210
|
const owner = normalizeOwner("owner", input.owner);
|
|
207
211
|
const supervision = normalizeSupervision(input.supervision);
|
|
212
|
+
const expectedHostId = input.expectedHostId === undefined
|
|
213
|
+
? undefined
|
|
214
|
+
: input.expectedHostId === null
|
|
215
|
+
? null
|
|
216
|
+
: requireId("expectedHostId", input.expectedHostId);
|
|
208
217
|
const health = input.health === undefined
|
|
209
218
|
? null
|
|
210
219
|
: requireEnum("health", input.health, HEALTH_VALUES);
|
|
@@ -212,6 +221,12 @@ export function recordSupervision(
|
|
|
212
221
|
if (health === null && supervision === null) {
|
|
213
222
|
throw new StoreError("validation", "Invalid supervision update: nothing to record");
|
|
214
223
|
}
|
|
224
|
+
if (supervision !== null && expectedHostId === undefined) {
|
|
225
|
+
throw new StoreError(
|
|
226
|
+
"validation",
|
|
227
|
+
"Invalid expectedHostId: replacing supervision requires the observed current host id, or null when no supervisor was recorded.",
|
|
228
|
+
);
|
|
229
|
+
}
|
|
215
230
|
|
|
216
231
|
return ctx.transaction(() => {
|
|
217
232
|
const operation = requireOperation(ctx, operationId);
|
|
@@ -221,6 +236,15 @@ export function recordSupervision(
|
|
|
221
236
|
`Operation ${operationId} belongs to another execution owner (differs in ${describeOwnerDiff(operation.owner, owner)}).`,
|
|
222
237
|
);
|
|
223
238
|
}
|
|
239
|
+
if (expectedHostId !== undefined) {
|
|
240
|
+
const currentHostId = operation.supervision?.hostId ?? null;
|
|
241
|
+
if (currentHostId !== expectedHostId) {
|
|
242
|
+
throw new StoreError(
|
|
243
|
+
"supervision-conflict",
|
|
244
|
+
`Operation ${operationId} is now supervised by ${currentHostId ?? "no host"}, not the observed ${expectedHostId ?? "no host"}; the stale replacement was refused.`,
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
224
248
|
const nextHealth = health ?? operation.health;
|
|
225
249
|
const nextReason = nextHealth === "active" ? null : (reason ?? operation.healthReason);
|
|
226
250
|
if (nextHealth !== "active" && nextReason === null) {
|
|
@@ -229,17 +253,62 @@ export function recordSupervision(
|
|
|
229
253
|
"Invalid reason: observer health other than active requires a recorded reason",
|
|
230
254
|
);
|
|
231
255
|
}
|
|
256
|
+
const nextSupervision = supervision ?? operation.supervision;
|
|
257
|
+
const nextHostId = nextSupervision?.hostId ?? null;
|
|
258
|
+
const authorityChanged = (operation.supervision?.hostId ?? null) !== nextHostId;
|
|
232
259
|
run(
|
|
233
260
|
ctx,
|
|
234
|
-
"UPDATE operations SET supervision_json = ?, health = ?, health_reason = ?, updated_at = ? WHERE operation_id = ?",
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
: null,
|
|
261
|
+
"UPDATE operations SET supervision_json = ?, supervision_host_id = ?, health = ?, health_reason = ?, updated_at = ? WHERE operation_id = ?",
|
|
262
|
+
nextSupervision ? JSON.stringify(nextSupervision) : null,
|
|
263
|
+
nextHostId,
|
|
238
264
|
nextHealth,
|
|
239
265
|
nextReason,
|
|
240
266
|
ctx.now(),
|
|
241
267
|
operationId,
|
|
242
268
|
);
|
|
269
|
+
if (authorityChanged) bumpChangeRevision(ctx);
|
|
270
|
+
return requireOperation(ctx, operationId);
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Release a closed host's supervision authority without clearing a newer
|
|
276
|
+
* holder. The update is conditional on the exact recorded host, so a host that
|
|
277
|
+
* shut down while another process already took over cannot erase the takeover.
|
|
278
|
+
* A terminal operation keeps its final supervision record; only running work
|
|
279
|
+
* becomes explicitly unmonitored for the next host.
|
|
280
|
+
*/
|
|
281
|
+
export function releaseSupervision(
|
|
282
|
+
ctx: StoreContext,
|
|
283
|
+
input: ReleaseSupervisionInput,
|
|
284
|
+
): OperationRecord | null {
|
|
285
|
+
const operationId = requireId("operationId", input.operationId);
|
|
286
|
+
const owner = normalizeOwner("owner", input.owner);
|
|
287
|
+
const hostId = requireId("hostId", input.hostId);
|
|
288
|
+
const reason = optionalText("reason", input.reason, 1024)
|
|
289
|
+
?? "the supervising host closed; another host may take over";
|
|
290
|
+
|
|
291
|
+
return ctx.transaction(() => {
|
|
292
|
+
const operation = requireOperation(ctx, operationId);
|
|
293
|
+
if (!sameOwner(operation.owner, owner)) {
|
|
294
|
+
throw new StoreError(
|
|
295
|
+
"foreign-owner",
|
|
296
|
+
`Operation ${operationId} belongs to another execution owner (differs in ${describeOwnerDiff(operation.owner, owner)}).`,
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
if (operation.supervision?.hostId !== hostId) return null;
|
|
300
|
+
if (operation.phase === "terminal") return operation;
|
|
301
|
+
run(
|
|
302
|
+
ctx,
|
|
303
|
+
`UPDATE operations
|
|
304
|
+
SET supervision_json = NULL, supervision_host_id = NULL,
|
|
305
|
+
health = 'unmonitored', health_reason = ?, updated_at = ?
|
|
306
|
+
WHERE operation_id = ?`,
|
|
307
|
+
reason,
|
|
308
|
+
ctx.now(),
|
|
309
|
+
operationId,
|
|
310
|
+
);
|
|
311
|
+
bumpChangeRevision(ctx);
|
|
243
312
|
return requireOperation(ctx, operationId);
|
|
244
313
|
});
|
|
245
314
|
}
|
|
@@ -320,6 +389,7 @@ export function recordTerminalOutcome(
|
|
|
320
389
|
at,
|
|
321
390
|
operationId,
|
|
322
391
|
);
|
|
392
|
+
bumpChangeRevision(ctx);
|
|
323
393
|
const settledOperation = requireOperation(ctx, operationId);
|
|
324
394
|
const settled = settleWaitsForOperation(ctx, settledOperation, at);
|
|
325
395
|
const outbox = listOutboxForRevision(ctx, operationId, 1);
|
|
@@ -367,6 +437,18 @@ export function listOperations(
|
|
|
367
437
|
.map(mapOperation);
|
|
368
438
|
}
|
|
369
439
|
|
|
440
|
+
/** Running operations assigned to one supervising host, using the phase/host index. */
|
|
441
|
+
export function listRunningSupervisedBy(
|
|
442
|
+
ctx: StoreContext,
|
|
443
|
+
hostId: unknown,
|
|
444
|
+
): OperationRecord[] {
|
|
445
|
+
return getRows(
|
|
446
|
+
ctx,
|
|
447
|
+
"SELECT * FROM operations WHERE phase = 'running' AND supervision_host_id = ? ORDER BY created_at, operation_id",
|
|
448
|
+
requireId("hostId", hostId),
|
|
449
|
+
).map(mapOperation);
|
|
450
|
+
}
|
|
451
|
+
|
|
370
452
|
function listOutboxForRevision(
|
|
371
453
|
ctx: StoreContext,
|
|
372
454
|
operationId: string,
|
|
@@ -318,11 +318,20 @@ export function claimOutbox(ctx: StoreContext, input: ClaimOutboxInput): OutboxE
|
|
|
318
318
|
assertDispatchTokenUnused(ctx, claimToken);
|
|
319
319
|
const clauses = [
|
|
320
320
|
"state = 'pending'",
|
|
321
|
-
`EXISTS (SELECT 1 FROM waits w WHERE w.wait_id = outbox_events.wait_id
|
|
321
|
+
`(event_type = 'assignment-outcome' OR EXISTS (SELECT 1 FROM waits w WHERE w.wait_id = outbox_events.wait_id
|
|
322
322
|
AND w.generation = outbox_events.generation
|
|
323
|
-
AND w.phase IN (${placeholders(DELIVERABLE_WAIT_PHASES.length)}))`,
|
|
323
|
+
AND w.phase IN (${placeholders(DELIVERABLE_WAIT_PHASES.length)})))`,
|
|
324
324
|
];
|
|
325
325
|
const params: Array<string | number | null> = [...DELIVERABLE_WAIT_PHASES];
|
|
326
|
+
if (input.eventId !== undefined) {
|
|
327
|
+
clauses.push("event_id = ?");
|
|
328
|
+
params.push(requireId("eventId", input.eventId));
|
|
329
|
+
}
|
|
330
|
+
if (input.type !== undefined) {
|
|
331
|
+
if (!["dependency-ready", "assignment-outcome"].includes(input.type)) throw new StoreError("validation", "Unknown outbox event type.");
|
|
332
|
+
clauses.push("event_type = ?");
|
|
333
|
+
params.push(input.type);
|
|
334
|
+
}
|
|
326
335
|
if (input.waitId !== undefined) {
|
|
327
336
|
clauses.push("wait_id = ?");
|
|
328
337
|
params.push(requireId("waitId", input.waitId));
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// Generated from packages/operation-coordinator/src/recovery.ts. Do not edit directly.
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { StoreError } from "./errors.ts";
|
|
4
|
+
import { getRow, getRows, columnText, run, type StoreContext } from "./sqlite.ts";
|
|
5
|
+
import { requireWait } from "./rows.ts";
|
|
6
|
+
import { holdFailedResume } from "./waits.ts";
|
|
7
|
+
import { getAssignmentOutcome, recordAssignmentOutcome } from "./assignment-outcomes.ts";
|
|
8
|
+
import type { HoldFailedResumeInput } from "./types.ts";
|
|
9
|
+
|
|
10
|
+
export interface RepairResumeInput extends HoldFailedResumeInput {
|
|
11
|
+
/** Verified original checkpoint, not the failed attempt's new entry. */
|
|
12
|
+
checkpointId: string;
|
|
13
|
+
expectedCancelEpoch: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** The exact exit is fenced and the allowance committed in the same transaction. */
|
|
17
|
+
export function repairFailedResume(ctx: StoreContext, input: RepairResumeInput) {
|
|
18
|
+
return ctx.transaction(() => {
|
|
19
|
+
const wait = requireWait(ctx, input.waitId, input.generation);
|
|
20
|
+
const assignmentId = wait.origin?.assignmentRef;
|
|
21
|
+
if (!assignmentId || !wait.park || input.checkpointId !== wait.park.checkpointId) {
|
|
22
|
+
throw new StoreError("resume-proof", "Repair requires the assignment's original checkpoint identity.");
|
|
23
|
+
}
|
|
24
|
+
if (wait.cancelEpoch !== input.expectedCancelEpoch) throw new StoreError("stale-cancel-epoch", "Repair cancellation epoch changed.");
|
|
25
|
+
const publication = getRow(ctx, "SELECT claim_id FROM resume_claims WHERE claim_id = ? AND claim_token = ? AND attempt_id = ? AND launch_token = ? AND wait_id = ? AND generation = ? AND cancel_epoch = ?", input.claimId, input.claimToken, input.attemptId, input.launchToken, wait.waitId, wait.generation, wait.cancelEpoch);
|
|
26
|
+
if (!publication) throw new StoreError("claim-state", "Repair does not match the exact published writer.");
|
|
27
|
+
const fault = JSON.stringify({ category: "unacknowledged-exit", checkpointId: input.checkpointId,
|
|
28
|
+
dependencies: wait.dependencies.map((d) => [d.operationId, d.terminalRevision]).sort((a, b) => String(a[0]).localeCompare(String(b[0]))) });
|
|
29
|
+
const key = createHash("sha256").update(JSON.stringify([assignmentId, wait.waitId, wait.generation, fault])).digest("hex");
|
|
30
|
+
const prior = getRow(ctx, "SELECT claim_id FROM assignment_repairs WHERE fault_key = ?", key);
|
|
31
|
+
const terminal = getAssignmentOutcome(ctx, assignmentId);
|
|
32
|
+
if (terminal) {
|
|
33
|
+
holdFailedResume(ctx, input);
|
|
34
|
+
return { phase: "held" as const, repaired: false };
|
|
35
|
+
}
|
|
36
|
+
// Duplicate exit notifications cannot exhaust a repair that has not run.
|
|
37
|
+
if (prior && columnText(prior, "claim_id") === input.claimId) return { phase: wait.phase, repaired: false };
|
|
38
|
+
holdFailedResume(ctx, input);
|
|
39
|
+
if (prior) {
|
|
40
|
+
const binding = getRow(ctx, "SELECT recipient_json FROM assignment_bindings WHERE assignment_id = ?", assignmentId);
|
|
41
|
+
if (!binding) throw new StoreError("foreign-owner", "Repair has no captured parent recipient.");
|
|
42
|
+
recordAssignmentOutcome(ctx, { assignmentId, owner: wait.owner, recipient: JSON.parse(columnText(binding, "recipient_json")),
|
|
43
|
+
revision: 1, outcome: "failed", reason: "Automatic repair repeated the same unacknowledged-exit fault; the one-repair allowance is exhausted.",
|
|
44
|
+
processUncertain: false, evidence: [{ kind: "summary", ref: `repair:${key}`, summary: input.reason }],
|
|
45
|
+
waits: getRows(ctx, "SELECT wait_id, generation FROM waits WHERE json_extract(origin_json, '$.assignmentRef') = ?", assignmentId).map((row) => {
|
|
46
|
+
const linked = requireWait(ctx, columnText(row, "wait_id"), Number(row.generation));
|
|
47
|
+
return { waitId: linked.waitId, generation: linked.generation, cancelEpoch: linked.cancelEpoch };
|
|
48
|
+
}) });
|
|
49
|
+
return { phase: "held" as const, repaired: false };
|
|
50
|
+
}
|
|
51
|
+
run(ctx, "INSERT INTO assignment_repairs VALUES (?, ?, ?, ?, ?, ?)", key, assignmentId, wait.waitId, wait.generation, fault, input.claimId);
|
|
52
|
+
run(ctx, "UPDATE waits SET phase = 'ready', phase_reason = 'bounded-repair', updated_at = ? WHERE wait_id = ? AND generation = ?", ctx.now(), wait.waitId, wait.generation);
|
|
53
|
+
return { phase: "ready" as const, repaired: true };
|
|
54
|
+
});
|
|
55
|
+
}
|
|
@@ -413,12 +413,13 @@ export function mapOutbox(ctx: StoreContext, row: Record<string, unknown>): Outb
|
|
|
413
413
|
const terminalRevision = columnNumber(row, "terminal_revision");
|
|
414
414
|
return {
|
|
415
415
|
eventId: columnText(row, "event_id"),
|
|
416
|
+
type: columnText(row, "event_type") as OutboxEventRecord["type"],
|
|
416
417
|
operationId,
|
|
417
418
|
terminalRevision,
|
|
418
419
|
waitId,
|
|
419
420
|
generation,
|
|
420
421
|
recipient: ownerFromRow(row, "recipient"),
|
|
421
|
-
evidence: readPinsForRevision(ctx, waitId, generation, operationId, terminalRevision),
|
|
422
|
+
evidence: row.event_type === "assignment-outcome" ? readPins(ctx, waitId, generation, false) : readPinsForRevision(ctx, waitId, generation, operationId, terminalRevision),
|
|
422
423
|
state: columnText(row, "state") as OutboxState,
|
|
423
424
|
attempts: columnNumber(row, "attempts"),
|
|
424
425
|
dispatchClaim: columnJson<OutboxDispatchClaim>(row, "dispatch_claim_json"),
|