paseo-agy-acp 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +296 -0
- package/LICENSE +201 -0
- package/README.md +571 -0
- package/README.zh-CN.md +540 -0
- package/assets/official-kernel-compat/rc01/paseo_model_compat.py +172 -0
- package/dist/ACP Connector/acp/session/paseo-context.d.ts +6 -0
- package/dist/ACP Connector/acp/session/paseo-context.js +83 -0
- package/dist/ACP Connector/acp/session/paseo-context.js.map +1 -0
- package/dist/ACP Connector/acp/session/turn-scheduler.d.ts +90 -0
- package/dist/ACP Connector/acp/session/turn-scheduler.js +247 -0
- package/dist/ACP Connector/acp/session/turn-scheduler.js.map +1 -0
- package/dist/ACP Connector/admission/dispatch-boundary.d.ts +19 -0
- package/dist/ACP Connector/admission/dispatch-boundary.js +16 -0
- package/dist/ACP Connector/admission/dispatch-boundary.js.map +1 -0
- package/dist/ACP Connector/admission/errors.d.ts +18 -0
- package/dist/ACP Connector/admission/errors.js +46 -0
- package/dist/ACP Connector/admission/errors.js.map +1 -0
- package/dist/ACP Connector/admission/owner-instance.d.ts +31 -0
- package/dist/ACP Connector/admission/owner-instance.js +105 -0
- package/dist/ACP Connector/admission/owner-instance.js.map +1 -0
- package/dist/ACP Connector/admission/runtime.d.ts +20 -0
- package/dist/ACP Connector/admission/runtime.js +157 -0
- package/dist/ACP Connector/admission/runtime.js.map +1 -0
- package/dist/ACP Connector/admission/startup-recovery.d.ts +22 -0
- package/dist/ACP Connector/admission/startup-recovery.js +129 -0
- package/dist/ACP Connector/admission/startup-recovery.js.map +1 -0
- package/dist/ACP Connector/admission/turn-coordinator.d.ts +57 -0
- package/dist/ACP Connector/admission/turn-coordinator.js +369 -0
- package/dist/ACP Connector/admission/turn-coordinator.js.map +1 -0
- package/dist/ACP Connector/agent.d.ts +4 -0
- package/dist/ACP Connector/agent.js +6 -0
- package/dist/ACP Connector/agent.js.map +1 -0
- package/dist/ACP Connector/main.d.ts +2 -0
- package/dist/ACP Connector/main.js +31 -0
- package/dist/ACP Connector/main.js.map +1 -0
- package/dist/ACP Connector/official-kernel/admission-fence.d.ts +10 -0
- package/dist/ACP Connector/official-kernel/admission-fence.js +52 -0
- package/dist/ACP Connector/official-kernel/admission-fence.js.map +1 -0
- package/dist/ACP Connector/official-kernel/context.d.ts +3 -0
- package/dist/ACP Connector/official-kernel/context.js +52 -0
- package/dist/ACP Connector/official-kernel/context.js.map +1 -0
- package/dist/ACP Connector/official-kernel/errors.d.ts +5 -0
- package/dist/ACP Connector/official-kernel/errors.js +39 -0
- package/dist/ACP Connector/official-kernel/errors.js.map +1 -0
- package/dist/ACP Connector/official-kernel/identity.d.ts +3 -0
- package/dist/ACP Connector/official-kernel/identity.js +17 -0
- package/dist/ACP Connector/official-kernel/identity.js.map +1 -0
- package/dist/ACP Connector/official-kernel/json-rpc.d.ts +34 -0
- package/dist/ACP Connector/official-kernel/json-rpc.js +26 -0
- package/dist/ACP Connector/official-kernel/json-rpc.js.map +1 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-lifecycle.d.ts +115 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-lifecycle.js +1904 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-lifecycle.js.map +1 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-pins.d.ts +14 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-pins.js +29 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-pins.js.map +1 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-rc01-recipe.d.ts +2 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-rc01-recipe.js +32 -0
- package/dist/ACP Connector/official-kernel/kernel-compat-rc01-recipe.js.map +1 -0
- package/dist/ACP Connector/official-kernel/kernel.d.ts +3 -0
- package/dist/ACP Connector/official-kernel/kernel.js +12 -0
- package/dist/ACP Connector/official-kernel/kernel.js.map +1 -0
- package/dist/ACP Connector/official-kernel/login.d.ts +1 -0
- package/dist/ACP Connector/official-kernel/login.js +73 -0
- package/dist/ACP Connector/official-kernel/login.js.map +1 -0
- package/dist/ACP Connector/official-kernel/mcp-rewrite.d.ts +1 -0
- package/dist/ACP Connector/official-kernel/mcp-rewrite.js +39 -0
- package/dist/ACP Connector/official-kernel/mcp-rewrite.js.map +1 -0
- package/dist/ACP Connector/official-kernel/mode-map.d.ts +5 -0
- package/dist/ACP Connector/official-kernel/mode-map.js +35 -0
- package/dist/ACP Connector/official-kernel/mode-map.js.map +1 -0
- package/dist/ACP Connector/official-kernel/ndjson.d.ts +3 -0
- package/dist/ACP Connector/official-kernel/ndjson.js +29 -0
- package/dist/ACP Connector/official-kernel/ndjson.js.map +1 -0
- package/dist/ACP Connector/official-kernel/proxy.d.ts +14 -0
- package/dist/ACP Connector/official-kernel/proxy.js +197 -0
- package/dist/ACP Connector/official-kernel/proxy.js.map +1 -0
- package/dist/ACP Connector/official-kernel/run.d.ts +8 -0
- package/dist/ACP Connector/official-kernel/run.js +22 -0
- package/dist/ACP Connector/official-kernel/run.js.map +1 -0
- package/dist/ACP Connector/official-kernel/spawn.d.ts +5 -0
- package/dist/ACP Connector/official-kernel/spawn.js +33 -0
- package/dist/ACP Connector/official-kernel/spawn.js.map +1 -0
- package/dist/Admission Controller/controller.d.ts +338 -0
- package/dist/Admission Controller/controller.js +2089 -0
- package/dist/Admission Controller/controller.js.map +1 -0
- package/dist/Admission Controller/key-derivation.d.ts +24 -0
- package/dist/Admission Controller/key-derivation.js +141 -0
- package/dist/Admission Controller/key-derivation.js.map +1 -0
- package/dist/Admission Controller/key-store.d.ts +8 -0
- package/dist/Admission Controller/key-store.js +280 -0
- package/dist/Admission Controller/key-store.js.map +1 -0
- package/dist/Admission Controller/policy-limits.d.ts +6 -0
- package/dist/Admission Controller/policy-limits.js +11 -0
- package/dist/Admission Controller/policy-limits.js.map +1 -0
- package/dist/Admission Controller/process-evidence.d.ts +111 -0
- package/dist/Admission Controller/process-evidence.js +549 -0
- package/dist/Admission Controller/process-evidence.js.map +1 -0
- package/dist/Admission Controller/runtime-config.d.ts +23 -0
- package/dist/Admission Controller/runtime-config.js +86 -0
- package/dist/Admission Controller/runtime-config.js.map +1 -0
- package/dist/Admission Controller/schema.d.ts +12 -0
- package/dist/Admission Controller/schema.js +422 -0
- package/dist/Admission Controller/schema.js.map +1 -0
- package/package.json +72 -0
- package/scripts/prepare-admission-state-dir.mjs +62 -0
- package/scripts/prepare-official-kernel-compat.mjs +387 -0
|
@@ -0,0 +1,2089 @@
|
|
|
1
|
+
import { createCipheriv, createDecipheriv, createHmac, randomBytes, randomUUID } from "node:crypto";
|
|
2
|
+
import Database from "better-sqlite3";
|
|
3
|
+
import { ADMISSION_SCHEMA_VERSION, assertAdmissionSchemaIntegrity } from "./schema.js";
|
|
4
|
+
import { captureLinuxProcessIdentity, inspectLinuxProcessGroup, nativeLinuxProcessEvidenceReaders, observeLinuxProcessIdentity } from "./process-evidence.js";
|
|
5
|
+
import { isAllowedAdmissionActiveTurns, isAllowedAdmissionConcurrentStarts } from "./policy-limits.js";
|
|
6
|
+
export { isAllowedAdmissionActiveTurns, isAllowedAdmissionConcurrentStarts } from "./policy-limits.js";
|
|
7
|
+
const MAX_DISPATCH_CONTENTION_RECHECKS = 500;
|
|
8
|
+
const DISPATCH_CONTENTION_RECHECK_DELAY_MS = 2;
|
|
9
|
+
const MAX_RUNTIME_PID = 2_147_483_647;
|
|
10
|
+
const LEASE_HEARTBEAT_STALE_MS = 4_000;
|
|
11
|
+
const dispatchContentionRetrySignal = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT));
|
|
12
|
+
export class AdmissionConflictError extends Error {
|
|
13
|
+
constructor(_requestId) {
|
|
14
|
+
super("request identity was reused with different immutable metadata");
|
|
15
|
+
this.name = "AdmissionConflictError";
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export class PayloadExpiredError extends Error {
|
|
19
|
+
constructor(_requestId) {
|
|
20
|
+
super("request payload has expired");
|
|
21
|
+
this.name = "PayloadExpiredError";
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export class PayloadConflictError extends Error {
|
|
25
|
+
constructor(_requestId) {
|
|
26
|
+
super("request already has a different durable payload");
|
|
27
|
+
this.name = "PayloadConflictError";
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export class LeaseFenceError extends Error {
|
|
31
|
+
constructor(_leaseId) {
|
|
32
|
+
super("lease is not owned by the supplied generation fence");
|
|
33
|
+
this.name = "LeaseFenceError";
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
/** Raised when durable recovery inventory data cannot be trusted. */
|
|
37
|
+
export class RecoverableDispatchInventoryError extends Error {
|
|
38
|
+
constructor() {
|
|
39
|
+
super("recoverable dispatch inventory contains an invalid durable row");
|
|
40
|
+
this.name = "RecoverableDispatchInventoryError";
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
class AdmissionControllerInjectedFaultError extends Error {
|
|
44
|
+
constructor() {
|
|
45
|
+
super("admission transaction fault injection");
|
|
46
|
+
this.name = "AdmissionControllerInjectedFaultError";
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export class AdmissionMigrationError extends Error {
|
|
50
|
+
constructor(detail) {
|
|
51
|
+
super(`admission schema migration failed: ${detail}`);
|
|
52
|
+
this.name = "AdmissionMigrationError";
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export class AdmissionRuntimeError extends Error {
|
|
56
|
+
constructor(message) {
|
|
57
|
+
super(`admission runtime error: ${message}`);
|
|
58
|
+
this.name = "AdmissionRuntimeError";
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* A local, cross-process admission plane. It deliberately refuses to infer
|
|
63
|
+
* that a dispatched turn is safe to replay after a crash.
|
|
64
|
+
*/
|
|
65
|
+
export class AdmissionController {
|
|
66
|
+
databasePath;
|
|
67
|
+
policy;
|
|
68
|
+
#db;
|
|
69
|
+
#encryptionKey;
|
|
70
|
+
#contentFingerprintKey;
|
|
71
|
+
#faultInjection;
|
|
72
|
+
#queuedOwnerIdentity;
|
|
73
|
+
constructor(options) {
|
|
74
|
+
this.databasePath = options.databasePath;
|
|
75
|
+
this.policy = validatePolicy(options.policy);
|
|
76
|
+
this.#encryptionKey = validatePurposeKey(options.encryptionKey, "encryption");
|
|
77
|
+
this.#contentFingerprintKey = validatePurposeKey(options.contentFingerprintKey, "content fingerprint");
|
|
78
|
+
this.#faultInjection = validateFaultInjection(options.faultInjection);
|
|
79
|
+
this.#queuedOwnerIdentity = captureControllerQueuedOwnerIdentity();
|
|
80
|
+
this.#db = new Database(options.databasePath);
|
|
81
|
+
this.#db.pragma("foreign_keys = ON");
|
|
82
|
+
this.#db.pragma("journal_mode = WAL");
|
|
83
|
+
this.#db.pragma("synchronous = FULL");
|
|
84
|
+
this.#db.pragma("busy_timeout = 5000");
|
|
85
|
+
this.migrate();
|
|
86
|
+
}
|
|
87
|
+
close() {
|
|
88
|
+
this.#db.close();
|
|
89
|
+
this.#encryptionKey?.fill(0);
|
|
90
|
+
this.#contentFingerprintKey?.fill(0);
|
|
91
|
+
}
|
|
92
|
+
get schemaVersion() {
|
|
93
|
+
const ledger = this.#db
|
|
94
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_migrations'")
|
|
95
|
+
.get();
|
|
96
|
+
if (ledger === undefined)
|
|
97
|
+
return 0;
|
|
98
|
+
const row = this.#db
|
|
99
|
+
.prepare("SELECT MAX(version) AS version FROM schema_migrations")
|
|
100
|
+
.get();
|
|
101
|
+
return row.version ?? 0;
|
|
102
|
+
}
|
|
103
|
+
enqueue(input) {
|
|
104
|
+
return this.transaction(() => {
|
|
105
|
+
const result = this.enqueueRequest(input);
|
|
106
|
+
this.persistQueuedOwnerReference(input, input.now, result.existed);
|
|
107
|
+
return result;
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
enqueueWithPayload(input, plaintext, expiresAt) {
|
|
111
|
+
this.validatePayloadExpiry(input.now, expiresAt);
|
|
112
|
+
const keyVersion = 1;
|
|
113
|
+
const contentFingerprint = this.contentFingerprint("turn", input.requestId, plaintext);
|
|
114
|
+
const encrypted = this.encrypt(plaintext, this.payloadAad(input.requestId, keyVersion));
|
|
115
|
+
return this.transaction(() => {
|
|
116
|
+
const result = this.enqueueRequest(input);
|
|
117
|
+
const request = this.requireRequestState(input.requestId);
|
|
118
|
+
if (request.state !== "queued")
|
|
119
|
+
throw new Error("request is no longer queued");
|
|
120
|
+
this.persistQueuedOwnerReference(input, input.now, result.existed);
|
|
121
|
+
const existing = this.#db
|
|
122
|
+
.prepare("SELECT content_fingerprint FROM turn_payloads WHERE request_id = ?")
|
|
123
|
+
.get(input.requestId);
|
|
124
|
+
if (existing) {
|
|
125
|
+
if (existing.content_fingerprint !== contentFingerprint)
|
|
126
|
+
throw new PayloadConflictError(input.requestId);
|
|
127
|
+
return { requestId: input.requestId, existed: true };
|
|
128
|
+
}
|
|
129
|
+
this.insertPayload(input.requestId, encrypted, keyVersion, contentFingerprint, expiresAt, input.now);
|
|
130
|
+
return result;
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
persistPayload(requestId, plaintext, now, expiresAt) {
|
|
134
|
+
this.validatePayloadExpiry(now, expiresAt);
|
|
135
|
+
const keyVersion = 1;
|
|
136
|
+
const contentFingerprint = this.contentFingerprint("turn", requestId, plaintext);
|
|
137
|
+
const encrypted = this.encrypt(plaintext, this.payloadAad(requestId, keyVersion));
|
|
138
|
+
this.transaction(() => {
|
|
139
|
+
const request = this.requireRequestState(requestId);
|
|
140
|
+
if (request.state !== "queued")
|
|
141
|
+
throw new Error("request is no longer queued");
|
|
142
|
+
const existing = this.#db
|
|
143
|
+
.prepare("SELECT content_fingerprint FROM turn_payloads WHERE request_id = ?")
|
|
144
|
+
.get(requestId);
|
|
145
|
+
if (existing) {
|
|
146
|
+
if (existing.content_fingerprint !== contentFingerprint)
|
|
147
|
+
throw new PayloadConflictError(requestId);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
this.insertPayload(requestId, encrypted, keyVersion, contentFingerprint, expiresAt, now);
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
readPayload(requestId, now) {
|
|
154
|
+
const row = this.transaction(() => {
|
|
155
|
+
const row = this.#db
|
|
156
|
+
.prepare("SELECT nonce, ciphertext, auth_tag, key_version, content_fingerprint, expires_at FROM turn_payloads WHERE request_id = ?")
|
|
157
|
+
.get(requestId);
|
|
158
|
+
if (!row)
|
|
159
|
+
throw new Error("no payload is available");
|
|
160
|
+
if (row.expires_at <= now) {
|
|
161
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(requestId);
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
return row;
|
|
165
|
+
});
|
|
166
|
+
if (!row)
|
|
167
|
+
throw new PayloadExpiredError(requestId);
|
|
168
|
+
if (row.content_fingerprint === null) {
|
|
169
|
+
throw new Error("request payload predates authenticated row binding");
|
|
170
|
+
}
|
|
171
|
+
return this.decrypt({ nonce: row.nonce, ciphertext: row.ciphertext, authTag: row.auth_tag }, this.payloadAad(requestId, row.key_version));
|
|
172
|
+
}
|
|
173
|
+
/** Cancelling after admission needs process and provider-terminal evidence. */
|
|
174
|
+
cancelQueued(requestId, now) {
|
|
175
|
+
this.transaction(() => {
|
|
176
|
+
const request = this.#db
|
|
177
|
+
.prepare("SELECT state FROM turn_requests WHERE request_id = ?")
|
|
178
|
+
.get(requestId);
|
|
179
|
+
if (!request)
|
|
180
|
+
throw new Error("unknown request");
|
|
181
|
+
if (request.state !== "queued")
|
|
182
|
+
throw new Error("request is no longer queued");
|
|
183
|
+
this.#db
|
|
184
|
+
.prepare("UPDATE turn_requests SET state = 'cancelled', terminal_at = ? WHERE request_id = ?")
|
|
185
|
+
.run(now, requestId);
|
|
186
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(requestId);
|
|
187
|
+
this.journalTransition("request_cancelled", requestId, "queued", "cancelled", now);
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
admitNext(now, ownerInstanceId) {
|
|
191
|
+
validateIdentifier(ownerInstanceId, "admission owner instance ID");
|
|
192
|
+
validateTimestamp(now, "admission timestamp");
|
|
193
|
+
return this.transaction(() => {
|
|
194
|
+
this.expireQueued(now);
|
|
195
|
+
if (!this.hasDispatchCapacity(now))
|
|
196
|
+
return null;
|
|
197
|
+
const candidate = this.selectEligibleRequest(now);
|
|
198
|
+
if (!candidate)
|
|
199
|
+
return null;
|
|
200
|
+
return this.reserveAdmission(candidate, now, ownerInstanceId);
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Atomically admit this request only when the global selector chose it.
|
|
205
|
+
* Callers may wait on their own request, but cannot bypass the controller's
|
|
206
|
+
* oldest-eligible and agent-fair ordering.
|
|
207
|
+
*/
|
|
208
|
+
admitRequest(requestId, now, ownerInstanceId) {
|
|
209
|
+
validateIdentifier(requestId, "admission request ID");
|
|
210
|
+
validateIdentifier(ownerInstanceId, "admission owner instance ID");
|
|
211
|
+
validateTimestamp(now, "admission timestamp");
|
|
212
|
+
return this.transaction(() => {
|
|
213
|
+
this.expireQueued(now);
|
|
214
|
+
if (!this.hasDispatchCapacity(now))
|
|
215
|
+
return null;
|
|
216
|
+
const candidate = this.selectEligibleRequest(now);
|
|
217
|
+
if (candidate?.request_id !== requestId)
|
|
218
|
+
return null;
|
|
219
|
+
return this.reserveAdmission(candidate, now, ownerInstanceId);
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
getQueueSnapshot(requestId, now) {
|
|
223
|
+
validateIdentifier(requestId, "queue snapshot request ID");
|
|
224
|
+
validateTimestamp(now, "queue snapshot timestamp");
|
|
225
|
+
return this.transaction(() => {
|
|
226
|
+
this.expireQueued(now);
|
|
227
|
+
const rows = this.orderedQueuedRequests();
|
|
228
|
+
const index = rows.findIndex((row) => row.request_id === requestId);
|
|
229
|
+
if (index < 0)
|
|
230
|
+
return null;
|
|
231
|
+
const row = rows[index];
|
|
232
|
+
const eligible = rows.filter((candidate) => !this.isCooldownActive(candidate.provider, candidate.model, now));
|
|
233
|
+
const eligibleIndex = eligible.findIndex((candidate) => candidate.request_id === requestId);
|
|
234
|
+
return Object.freeze({
|
|
235
|
+
requestId,
|
|
236
|
+
position: index + 1,
|
|
237
|
+
eligiblePosition: eligibleIndex < 0 ? null : eligibleIndex + 1,
|
|
238
|
+
enqueuedAt: row.enqueued_at,
|
|
239
|
+
waitedMs: Math.max(0, now - row.enqueued_at),
|
|
240
|
+
cooldownUntil: this.cooldownUntil(row.provider, row.model, now)
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
markStarting(fence, now) {
|
|
245
|
+
this.transaction(() => {
|
|
246
|
+
const lease = this.requireLease(fence);
|
|
247
|
+
if (lease.phase !== "admitted")
|
|
248
|
+
throw new Error("lease is not admitted");
|
|
249
|
+
const starts = this.#db
|
|
250
|
+
.prepare("SELECT COUNT(*) AS count FROM leases WHERE phase IN ('starting', 'dispatch_intent')")
|
|
251
|
+
.get();
|
|
252
|
+
if (starts.count >= this.policy.maxConcurrentStarts)
|
|
253
|
+
throw new Error("concurrent start limit reached");
|
|
254
|
+
this.#db
|
|
255
|
+
.prepare("DELETE FROM start_history WHERE started_at < ?")
|
|
256
|
+
.run(now - this.policy.minStartIntervalMs);
|
|
257
|
+
const latestStart = this.#db
|
|
258
|
+
.prepare("SELECT started_at FROM start_history ORDER BY started_at DESC LIMIT 1")
|
|
259
|
+
.get();
|
|
260
|
+
if (latestStart && now < latestStart.started_at + this.policy.minStartIntervalMs) {
|
|
261
|
+
throw new Error("start interval has not elapsed");
|
|
262
|
+
}
|
|
263
|
+
this.#db.prepare("INSERT INTO start_history (lease_id, started_at) VALUES (?, ?)").run(fence.leaseId, now);
|
|
264
|
+
this.setLeasePhase(lease, "starting", now);
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Persist a verified Linux process record and dispatch_intent in one SQLite
|
|
269
|
+
* transaction. The following dispatch-boundary commit callback is an exact replay
|
|
270
|
+
* check, so no crash window exists between those two callbacks.
|
|
271
|
+
*/
|
|
272
|
+
recordProcessIdentity(input) {
|
|
273
|
+
const outcome = this.persistProcessIdentityAndDispatchIntent(input);
|
|
274
|
+
return outcome.status === "committed"
|
|
275
|
+
? { status: "recorded", idempotent: outcome.idempotent }
|
|
276
|
+
: { status: "not_recorded", reason: outcome.reason };
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Confirm the atomic identity-and-intent persistence. A fresh call performs
|
|
280
|
+
* the same transaction; a matching dispatch_intent record is idempotent.
|
|
281
|
+
*/
|
|
282
|
+
commitDispatchIntent(input) {
|
|
283
|
+
const outcome = this.persistProcessIdentityAndDispatchIntent(input);
|
|
284
|
+
return outcome.status === "committed"
|
|
285
|
+
? { status: "committed", idempotent: outcome.idempotent }
|
|
286
|
+
: { status: "not_committed", reason: outcome.reason };
|
|
287
|
+
}
|
|
288
|
+
markDispatchIntent(fence, now) {
|
|
289
|
+
this.transition(fence, "starting", "dispatch_intent", now);
|
|
290
|
+
}
|
|
291
|
+
markActive(fence, now) {
|
|
292
|
+
this.transaction(() => {
|
|
293
|
+
const lease = this.requireLease(fence);
|
|
294
|
+
if (lease.phase !== "dispatch_intent")
|
|
295
|
+
throw new Error("lease is not dispatch_intent");
|
|
296
|
+
this.setLeasePhase(lease, "active", now);
|
|
297
|
+
// Once the irreversible write has been issued, the payload must never be
|
|
298
|
+
// available to a recovery path that could repeat the business prompt.
|
|
299
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(lease.request_id);
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
markDispatchAmbiguous(fence, now) {
|
|
303
|
+
this.transition(fence, "dispatch_intent", "dispatch_ambiguous", now);
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* End a reserved turn only when the Connector still has direct proof that no
|
|
307
|
+
* business prompt write occurred. This is terminal, never a requeue.
|
|
308
|
+
*/
|
|
309
|
+
abandonBeforePrompt(fence, now, outcome) {
|
|
310
|
+
validateTimestamp(now, "pre-prompt abandonment timestamp");
|
|
311
|
+
this.transaction(() => {
|
|
312
|
+
const lease = this.requireLease(fence);
|
|
313
|
+
if (lease.phase !== "admitted" && lease.phase !== "starting" && lease.phase !== "dispatch_intent") {
|
|
314
|
+
throw new Error("lease is not safely abandonable before prompt write");
|
|
315
|
+
}
|
|
316
|
+
this.#db
|
|
317
|
+
.prepare("UPDATE turn_requests SET state = ?, terminal_at = ? WHERE request_id = ?")
|
|
318
|
+
.run(outcome, now, lease.request_id);
|
|
319
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(lease.request_id);
|
|
320
|
+
const released = this.#db
|
|
321
|
+
.prepare("DELETE FROM leases WHERE lease_id = ? AND owner_instance_id = ? AND generation = ?")
|
|
322
|
+
.run(fence.leaseId, fence.ownerInstanceId, fence.generation);
|
|
323
|
+
if (released.changes !== 1)
|
|
324
|
+
throw new LeaseFenceError(fence.leaseId);
|
|
325
|
+
this.journalTransition("request_abandoned", lease.request_id, lease.phase, outcome, now);
|
|
326
|
+
this.completeSoftDrainTo1IfSettled(fence.ownerInstanceId, now);
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
/** Retain an uncertain dispatch as visible capacity debt; never requeue it. */
|
|
330
|
+
markExecutionRecoveryRequired(fence, now) {
|
|
331
|
+
validateTimestamp(now, "execution recovery timestamp");
|
|
332
|
+
this.transaction(() => {
|
|
333
|
+
const lease = this.requireLease(fence);
|
|
334
|
+
if (lease.phase === "recovery_required")
|
|
335
|
+
return;
|
|
336
|
+
if (lease.phase !== "starting" &&
|
|
337
|
+
lease.phase !== "dispatch_intent" &&
|
|
338
|
+
lease.phase !== "dispatch_ambiguous" &&
|
|
339
|
+
lease.phase !== "active") {
|
|
340
|
+
throw new Error("lease cannot enter execution recovery");
|
|
341
|
+
}
|
|
342
|
+
const updated = this.#db
|
|
343
|
+
.prepare(`UPDATE leases SET phase = 'recovery_required', heartbeat_at = ?
|
|
344
|
+
WHERE lease_id = ? AND owner_instance_id = ? AND generation = ?`)
|
|
345
|
+
.run(now, fence.leaseId, fence.ownerInstanceId, fence.generation);
|
|
346
|
+
if (updated.changes !== 1)
|
|
347
|
+
throw new LeaseFenceError(fence.leaseId);
|
|
348
|
+
this.#db
|
|
349
|
+
.prepare("UPDATE turn_requests SET state = 'recovery_required', terminal_at = ? WHERE request_id = ?")
|
|
350
|
+
.run(now, lease.request_id);
|
|
351
|
+
this.journalTransition("request_recovery_required", lease.request_id, lease.phase, "recovery_required", now);
|
|
352
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(lease.request_id);
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* Release only the local seat after the Connector has independently proved
|
|
357
|
+
* that the persisted connector, child, and process group are all gone. The
|
|
358
|
+
* request deliberately remains recovery_required and cannot be replayed.
|
|
359
|
+
*/
|
|
360
|
+
releaseExitedRecoverySeat(fence, now) {
|
|
361
|
+
validateTimestamp(now, "exited recovery seat timestamp");
|
|
362
|
+
this.transaction(() => {
|
|
363
|
+
const lease = this.requireLease(fence);
|
|
364
|
+
if (lease.phase === "provider_terminal" ||
|
|
365
|
+
lease.phase === "completed" ||
|
|
366
|
+
lease.phase === "failed" ||
|
|
367
|
+
lease.phase === "cancelled" ||
|
|
368
|
+
lease.phase === "queue_timeout") {
|
|
369
|
+
throw new Error("lease is not an unresolved local execution");
|
|
370
|
+
}
|
|
371
|
+
this.#db
|
|
372
|
+
.prepare("UPDATE turn_requests SET state = 'recovery_required', terminal_at = ? WHERE request_id = ?")
|
|
373
|
+
.run(now, lease.request_id);
|
|
374
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(lease.request_id);
|
|
375
|
+
const released = this.#db
|
|
376
|
+
.prepare("DELETE FROM leases WHERE lease_id = ? AND owner_instance_id = ? AND generation = ?")
|
|
377
|
+
.run(fence.leaseId, fence.ownerInstanceId, fence.generation);
|
|
378
|
+
if (released.changes !== 1)
|
|
379
|
+
throw new LeaseFenceError(fence.leaseId);
|
|
380
|
+
this.journalTransition("request_recovery_seat_released", lease.request_id, lease.phase, "recovery_required", now);
|
|
381
|
+
this.completeSoftDrainTo1IfSettled(fence.ownerInstanceId, now);
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Persist the terminal state and release the seat used by the existing live
|
|
386
|
+
* ACP update path. No delivery replay record is created.
|
|
387
|
+
*/
|
|
388
|
+
completeLiveTurn(fence, now, completion) {
|
|
389
|
+
validateTimestamp(now, "live turn completion timestamp");
|
|
390
|
+
validateLiveTurnCompletion(completion);
|
|
391
|
+
this.transaction(() => {
|
|
392
|
+
const lease = this.requireLease(fence);
|
|
393
|
+
if (lease.phase !== "active")
|
|
394
|
+
throw new Error("lease is not active");
|
|
395
|
+
const request = this.getRequest(lease.request_id);
|
|
396
|
+
if (request === null)
|
|
397
|
+
throw new Error("unknown request");
|
|
398
|
+
this.#db
|
|
399
|
+
.prepare("UPDATE turn_requests SET state = 'provider_terminal', terminal_at = ? WHERE request_id = ?")
|
|
400
|
+
.run(now, lease.request_id);
|
|
401
|
+
this.journalTransition("request_provider_terminal", lease.request_id, "active", "provider_terminal", now);
|
|
402
|
+
if (completion.failure?.category === "provider_capacity") {
|
|
403
|
+
const notBefore = now + this.policy.capacityCooldownMs;
|
|
404
|
+
this.#db
|
|
405
|
+
.prepare(`INSERT INTO cooldowns (provider, model, not_before, updated_at)
|
|
406
|
+
VALUES (?, ?, ?, ?)
|
|
407
|
+
ON CONFLICT(provider, model) DO UPDATE SET
|
|
408
|
+
not_before = MAX(cooldowns.not_before, excluded.not_before),
|
|
409
|
+
updated_at = MAX(cooldowns.updated_at, excluded.updated_at)`)
|
|
410
|
+
.run(request.provider, request.model, notBefore, now);
|
|
411
|
+
}
|
|
412
|
+
this.#db
|
|
413
|
+
.prepare("UPDATE turn_requests SET state = ?, terminal_at = ? WHERE request_id = ?")
|
|
414
|
+
.run(completion.outcome, now, lease.request_id);
|
|
415
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(lease.request_id);
|
|
416
|
+
const released = this.#db
|
|
417
|
+
.prepare("DELETE FROM leases WHERE lease_id = ? AND owner_instance_id = ? AND generation = ?")
|
|
418
|
+
.run(fence.leaseId, fence.ownerInstanceId, fence.generation);
|
|
419
|
+
if (released.changes !== 1)
|
|
420
|
+
throw new LeaseFenceError(fence.leaseId);
|
|
421
|
+
this.journalTransition("request_released", lease.request_id, "provider_terminal", completion.outcome, now);
|
|
422
|
+
this.completeSoftDrainTo1IfSettled(fence.ownerInstanceId, now);
|
|
423
|
+
});
|
|
424
|
+
}
|
|
425
|
+
heartbeat(fence, now) {
|
|
426
|
+
const result = this.#db
|
|
427
|
+
.prepare(`UPDATE leases
|
|
428
|
+
SET heartbeat_at = ?, suspect_since = NULL, suspect_reason = NULL
|
|
429
|
+
WHERE lease_id = ? AND owner_instance_id = ? AND generation = ?`)
|
|
430
|
+
.run(now, fence.leaseId, fence.ownerInstanceId, fence.generation);
|
|
431
|
+
if (result.changes !== 1)
|
|
432
|
+
throw new LeaseFenceError(fence.leaseId);
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Runtime recovery is evidence-only: a stale heartbeat can mark suspicion,
|
|
436
|
+
* but only connector/child/process-group proof can release local capacity.
|
|
437
|
+
*/
|
|
438
|
+
reapSuspects(now, readers) {
|
|
439
|
+
validateTimestamp(now, "runtime reaper timestamp");
|
|
440
|
+
const processIds = readRuntimeProcessIds(readers);
|
|
441
|
+
let released = 0;
|
|
442
|
+
let retained = 0;
|
|
443
|
+
let markedRecoveryRequired = 0;
|
|
444
|
+
let suspected = 0;
|
|
445
|
+
let queuedSettled = 0;
|
|
446
|
+
for (const queuedOwner of this.listRecoverableQueuedOwners()) {
|
|
447
|
+
const owner = observeLinuxProcessIdentity(queuedOwner.owner, readers);
|
|
448
|
+
if (isGoneIdentity(owner)) {
|
|
449
|
+
if (this.settleQueuedOwnerDeath(queuedOwner.requestId, queuedOwner.owner.ownerInstanceId, now)) {
|
|
450
|
+
queuedSettled += 1;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
const dispatches = this.listRecoverableDispatches();
|
|
455
|
+
for (const dispatch of dispatches) {
|
|
456
|
+
if (isHeartbeatStale(dispatch.heartbeatAt, now) || dispatch.processIdentity === null || processIds === null) {
|
|
457
|
+
if (this.markSuspect(dispatch.fence, now, suspectReason(dispatch, processIds)))
|
|
458
|
+
suspected += 1;
|
|
459
|
+
}
|
|
460
|
+
if (dispatch.processIdentity === null || processIds === null) {
|
|
461
|
+
retained += 1;
|
|
462
|
+
continue;
|
|
463
|
+
}
|
|
464
|
+
const connector = observeLinuxProcessIdentity(dispatch.processIdentity.connector, readers);
|
|
465
|
+
const child = observeLinuxProcessIdentity(dispatch.processIdentity.child, readers);
|
|
466
|
+
const residue = inspectLinuxProcessGroup(dispatch.processIdentity.child, processIds, readers);
|
|
467
|
+
if (isGoneIdentity(connector) && isGoneIdentity(child) && residue === "empty") {
|
|
468
|
+
try {
|
|
469
|
+
this.releaseExitedRecoverySeat(dispatch.fence, now);
|
|
470
|
+
released += 1;
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
catch {
|
|
474
|
+
retained += 1;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
if (connector === "unverifiable" || child === "unverifiable" || residue === "unverifiable") {
|
|
479
|
+
if (this.markSuspect(dispatch.fence, now, "identity_unverifiable"))
|
|
480
|
+
suspected += 1;
|
|
481
|
+
retained += 1;
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
if (connector !== "same") {
|
|
485
|
+
try {
|
|
486
|
+
this.markExecutionRecoveryRequired(dispatch.fence, now);
|
|
487
|
+
markedRecoveryRequired += 1;
|
|
488
|
+
}
|
|
489
|
+
catch {
|
|
490
|
+
// A concurrent owner may have advanced or settled the exact fence.
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
retained += 1;
|
|
494
|
+
}
|
|
495
|
+
return Object.freeze({
|
|
496
|
+
inspected: dispatches.length,
|
|
497
|
+
released,
|
|
498
|
+
retained,
|
|
499
|
+
markedRecoveryRequired,
|
|
500
|
+
suspected,
|
|
501
|
+
queuedSettled
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
setCapacityCooldown(provider, model, notBefore, now) {
|
|
505
|
+
if (!Number.isSafeInteger(notBefore) || !Number.isSafeInteger(now) || notBefore < now) {
|
|
506
|
+
throw new Error("capacity cooldown must use integer timestamps and cannot end before it is recorded");
|
|
507
|
+
}
|
|
508
|
+
this.#db
|
|
509
|
+
.prepare(`INSERT INTO cooldowns (provider, model, not_before, updated_at)
|
|
510
|
+
VALUES (?, ?, ?, ?)
|
|
511
|
+
ON CONFLICT(provider, model) DO UPDATE SET
|
|
512
|
+
not_before = MAX(cooldowns.not_before, excluded.not_before),
|
|
513
|
+
updated_at = MAX(cooldowns.updated_at, excluded.updated_at)`)
|
|
514
|
+
.run(provider, model, notBefore, now);
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Read an identifier-free page of the append-only audit journal. The exact
|
|
518
|
+
* input shape deliberately provides no request/session/provider lookup seam.
|
|
519
|
+
*/
|
|
520
|
+
readSanitizedEvents(input) {
|
|
521
|
+
const page = normalizeSanitizedEventPageRequest(input);
|
|
522
|
+
const rows = this.#db
|
|
523
|
+
.prepare(`SELECT event_seq, kind, from_state, to_state, occurred_at, correlation_hmac
|
|
524
|
+
FROM events WHERE event_seq > ? ORDER BY event_seq ASC LIMIT ?`)
|
|
525
|
+
.all(page.afterEventSeq, page.limit);
|
|
526
|
+
return Object.freeze(rows.map((row) => normalizeSanitizedEventRow(row)));
|
|
527
|
+
}
|
|
528
|
+
getRequest(requestId) {
|
|
529
|
+
const row = this.#db
|
|
530
|
+
.prepare(`SELECT request_id, session_id, agent_id, fingerprint, provider, model, state, enqueued_at
|
|
531
|
+
, lease_generation
|
|
532
|
+
FROM turn_requests WHERE request_id = ?`)
|
|
533
|
+
.get(requestId);
|
|
534
|
+
return row ? toStoredRequest(row) : null;
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Enumerate durable, nonterminal dispatches for connector startup recovery.
|
|
538
|
+
* This deliberately reads no prompt or outbox table, never decrypts data,
|
|
539
|
+
* and returns a null identity instead of implying that a dispatch is safe to
|
|
540
|
+
* resume when immutable process evidence is absent.
|
|
541
|
+
*/
|
|
542
|
+
listRecoverableDispatches() {
|
|
543
|
+
const rows = this.#db
|
|
544
|
+
.prepare(`SELECT lease.lease_id AS lease_id,
|
|
545
|
+
lease.request_id AS lease_request_id,
|
|
546
|
+
lease.generation AS lease_generation,
|
|
547
|
+
lease.owner_instance_id AS lease_owner_instance_id,
|
|
548
|
+
lease.phase AS lease_phase,
|
|
549
|
+
lease.heartbeat_at AS lease_heartbeat_at,
|
|
550
|
+
request.request_id AS request_id,
|
|
551
|
+
request.session_id AS request_session_id,
|
|
552
|
+
request.provider AS request_provider,
|
|
553
|
+
request.model AS request_model,
|
|
554
|
+
request.state AS request_state,
|
|
555
|
+
request.lease_generation AS request_lease_generation,
|
|
556
|
+
request.enqueued_at AS request_enqueued_at,
|
|
557
|
+
identity.lease_id AS identity_lease_id,
|
|
558
|
+
identity.request_id AS identity_request_id,
|
|
559
|
+
identity.lease_generation AS identity_lease_generation,
|
|
560
|
+
identity.owner_instance_id AS identity_owner_instance_id,
|
|
561
|
+
identity.prompt_channel AS identity_prompt_channel,
|
|
562
|
+
identity.connector_owner_instance_id AS identity_connector_owner_instance_id,
|
|
563
|
+
identity.connector_created_at AS identity_connector_created_at,
|
|
564
|
+
identity.connector_boot_id AS identity_connector_boot_id,
|
|
565
|
+
identity.connector_pid AS identity_connector_pid,
|
|
566
|
+
identity.connector_start_time_ticks AS identity_connector_start_time_ticks,
|
|
567
|
+
identity.connector_pid_namespace_inode AS identity_connector_pid_namespace_inode,
|
|
568
|
+
identity.connector_ppid AS identity_connector_ppid,
|
|
569
|
+
identity.connector_pgrp AS identity_connector_pgrp,
|
|
570
|
+
identity.connector_session AS identity_connector_session,
|
|
571
|
+
identity.child_boot_id AS identity_child_boot_id,
|
|
572
|
+
identity.child_pid AS identity_child_pid,
|
|
573
|
+
identity.child_start_time_ticks AS identity_child_start_time_ticks,
|
|
574
|
+
identity.child_pid_namespace_inode AS identity_child_pid_namespace_inode,
|
|
575
|
+
identity.child_ppid AS identity_child_ppid,
|
|
576
|
+
identity.child_pgrp AS identity_child_pgrp,
|
|
577
|
+
identity.child_session AS identity_child_session
|
|
578
|
+
FROM leases AS lease
|
|
579
|
+
LEFT JOIN turn_requests AS request ON request.request_id = lease.request_id
|
|
580
|
+
LEFT JOIN lease_process_identities AS identity ON identity.lease_id = lease.lease_id
|
|
581
|
+
ORDER BY request.enqueued_at ASC, request.request_id ASC, lease.lease_id ASC`)
|
|
582
|
+
.all();
|
|
583
|
+
const inventory = [];
|
|
584
|
+
for (const row of rows) {
|
|
585
|
+
const dispatch = toRecoverableDispatch(row);
|
|
586
|
+
if (dispatch !== null)
|
|
587
|
+
inventory.push(dispatch);
|
|
588
|
+
}
|
|
589
|
+
return Object.freeze(inventory);
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Enumerate queued requests whose owning connector was durably recorded at
|
|
593
|
+
* enqueue time. This returns no payload and grants no replay authority.
|
|
594
|
+
*/
|
|
595
|
+
listRecoverableQueuedOwners() {
|
|
596
|
+
const rows = this.#db
|
|
597
|
+
.prepare(`SELECT request.request_id AS request_id,
|
|
598
|
+
owner.owner_instance_id AS owner_instance_id,
|
|
599
|
+
owner.created_at AS created_at,
|
|
600
|
+
owner.boot_id AS boot_id,
|
|
601
|
+
owner.pid AS pid,
|
|
602
|
+
owner.start_time_ticks AS start_time_ticks,
|
|
603
|
+
owner.pid_namespace_inode AS pid_namespace_inode,
|
|
604
|
+
owner.ppid AS ppid,
|
|
605
|
+
owner.pgrp AS pgrp,
|
|
606
|
+
owner.session AS session,
|
|
607
|
+
owner.recorded_at AS recorded_at
|
|
608
|
+
FROM turn_requests AS request
|
|
609
|
+
JOIN queued_owner_instances AS owner
|
|
610
|
+
ON owner.owner_instance_id = request.queued_owner_instance_id
|
|
611
|
+
WHERE request.state = 'queued'
|
|
612
|
+
AND request.queued_owner_instance_id IS NOT NULL
|
|
613
|
+
ORDER BY request.enqueued_at ASC, request.request_id ASC`)
|
|
614
|
+
.all();
|
|
615
|
+
const owners = [];
|
|
616
|
+
for (const row of rows)
|
|
617
|
+
owners.push(toRecoverableQueuedOwner(row));
|
|
618
|
+
return Object.freeze(owners);
|
|
619
|
+
}
|
|
620
|
+
settleQueuedOwnerDeath(requestId, ownerInstanceId, now) {
|
|
621
|
+
validateIdentifier(requestId, "queued owner request ID");
|
|
622
|
+
validateIdentifier(ownerInstanceId, "queued owner instance ID");
|
|
623
|
+
validateTimestamp(now, "queued owner death timestamp");
|
|
624
|
+
return this.transaction(() => {
|
|
625
|
+
const request = this.#db
|
|
626
|
+
.prepare(`SELECT state, queued_owner_instance_id
|
|
627
|
+
FROM turn_requests
|
|
628
|
+
WHERE request_id = ?`)
|
|
629
|
+
.get(requestId);
|
|
630
|
+
if (request === undefined || request.state !== "queued")
|
|
631
|
+
return false;
|
|
632
|
+
if (request.queued_owner_instance_id !== ownerInstanceId)
|
|
633
|
+
return false;
|
|
634
|
+
const lease = this.#db
|
|
635
|
+
.prepare("SELECT 1 FROM leases WHERE request_id = ? LIMIT 1")
|
|
636
|
+
.get(requestId);
|
|
637
|
+
if (lease !== undefined)
|
|
638
|
+
return false;
|
|
639
|
+
const result = this.#db
|
|
640
|
+
.prepare(`UPDATE turn_requests
|
|
641
|
+
SET state = 'cancelled', terminal_at = ?
|
|
642
|
+
WHERE request_id = ?
|
|
643
|
+
AND state = 'queued'
|
|
644
|
+
AND queued_owner_instance_id = ?
|
|
645
|
+
AND NOT EXISTS (SELECT 1 FROM leases WHERE request_id = ?)`)
|
|
646
|
+
.run(now, requestId, ownerInstanceId, requestId);
|
|
647
|
+
if (result.changes !== 1)
|
|
648
|
+
return false;
|
|
649
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(requestId);
|
|
650
|
+
this.journalTransition("queued_owner_dead", requestId, "queued", "cancelled", now);
|
|
651
|
+
return true;
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
claimDurablePolicy(policy, ownerInstanceId, now) {
|
|
655
|
+
const normalizedPolicy = validatePolicy(policy);
|
|
656
|
+
validateIdentifier(ownerInstanceId, "durable policy owner instance ID");
|
|
657
|
+
validateTimestamp(now, "durable policy claim timestamp");
|
|
658
|
+
const policyFingerprint = this.policyFingerprint(normalizedPolicy);
|
|
659
|
+
this.transaction(() => {
|
|
660
|
+
const result = this.#db
|
|
661
|
+
.prepare(`INSERT INTO policy_state (
|
|
662
|
+
id, max_active_turns, max_concurrent_starts, min_start_interval_ms,
|
|
663
|
+
queue_timeout_ms, capacity_cooldown_ms, drain_state, policy_fingerprint,
|
|
664
|
+
updated_at, updated_by_owner_instance_id
|
|
665
|
+
)
|
|
666
|
+
SELECT 1, ?, ?, ?, ?, ?, 'steady', ?, ?, ?
|
|
667
|
+
WHERE NOT EXISTS (SELECT 1 FROM policy_state WHERE id = 1)`)
|
|
668
|
+
.run(normalizedPolicy.maxActiveTurns, normalizedPolicy.maxConcurrentStarts, normalizedPolicy.minStartIntervalMs, normalizedPolicy.queueTimeoutMs, normalizedPolicy.capacityCooldownMs, policyFingerprint, now, ownerInstanceId);
|
|
669
|
+
if (result.changes === 1)
|
|
670
|
+
return;
|
|
671
|
+
this.assertDurablePolicyMatchInTransaction(normalizedPolicy, policyFingerprint);
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
assertDurablePolicyMatch(policy, ownerInstanceId, now) {
|
|
675
|
+
const normalizedPolicy = validatePolicy(policy);
|
|
676
|
+
validateIdentifier(ownerInstanceId, "durable policy owner instance ID");
|
|
677
|
+
validateTimestamp(now, "durable policy assertion timestamp");
|
|
678
|
+
const policyFingerprint = this.policyFingerprint(normalizedPolicy);
|
|
679
|
+
this.transaction(() => this.assertDurablePolicyMatchInTransaction(normalizedPolicy, policyFingerprint));
|
|
680
|
+
}
|
|
681
|
+
beginSoftDrainTo1(ownerInstanceId, now) {
|
|
682
|
+
validateIdentifier(ownerInstanceId, "soft drain owner instance ID");
|
|
683
|
+
validateTimestamp(now, "soft drain timestamp");
|
|
684
|
+
this.transaction(() => {
|
|
685
|
+
const row = this.requirePolicyStateInTransaction();
|
|
686
|
+
const drainState = policyDrainState(row);
|
|
687
|
+
const maxActiveTurns = policyStateInteger(row.max_active_turns, "durable policy max_active_turns");
|
|
688
|
+
if (drainState === "steady" && maxActiveTurns === 1)
|
|
689
|
+
return;
|
|
690
|
+
if (drainState !== "steady" && drainState !== "soft_draining_to_1") {
|
|
691
|
+
throw new AdmissionRuntimeError("durable policy drain state is not recognized");
|
|
692
|
+
}
|
|
693
|
+
if (maxActiveTurns !== 3 && drainState !== "soft_draining_to_1") {
|
|
694
|
+
throw new AdmissionRuntimeError("soft drain can only start from a steady three-seat policy");
|
|
695
|
+
}
|
|
696
|
+
if (drainState === "steady") {
|
|
697
|
+
const result = this.#db
|
|
698
|
+
.prepare(`UPDATE policy_state
|
|
699
|
+
SET drain_state = 'soft_draining_to_1',
|
|
700
|
+
updated_at = ?,
|
|
701
|
+
updated_by_owner_instance_id = ?
|
|
702
|
+
WHERE id = 1 AND drain_state = 'steady' AND max_active_turns = 3`)
|
|
703
|
+
.run(now, ownerInstanceId);
|
|
704
|
+
if (result.changes !== 1)
|
|
705
|
+
throw new AdmissionRuntimeError("soft drain could not be started atomically");
|
|
706
|
+
}
|
|
707
|
+
this.completeSoftDrainTo1IfSettled(ownerInstanceId, now);
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
migrate() {
|
|
711
|
+
this.assertRenameColumnAvailable();
|
|
712
|
+
try {
|
|
713
|
+
this.transaction(() => {
|
|
714
|
+
this.#db.pragma("foreign_keys = ON");
|
|
715
|
+
let applied = this.schemaVersion;
|
|
716
|
+
if (applied > ADMISSION_SCHEMA_VERSION) {
|
|
717
|
+
throw new AdmissionMigrationError(`schema version ${applied} is newer than this connector supports`);
|
|
718
|
+
}
|
|
719
|
+
if (applied === 0) {
|
|
720
|
+
this.createInitialV1Schema(Date.now());
|
|
721
|
+
applied = 1;
|
|
722
|
+
}
|
|
723
|
+
if (applied === 1) {
|
|
724
|
+
this.migrateV1ToV2(Date.now());
|
|
725
|
+
applied = 2;
|
|
726
|
+
}
|
|
727
|
+
if (applied === 2) {
|
|
728
|
+
this.migrateV2ToV3(Date.now());
|
|
729
|
+
applied = 3;
|
|
730
|
+
}
|
|
731
|
+
if (applied !== ADMISSION_SCHEMA_VERSION) {
|
|
732
|
+
throw new AdmissionMigrationError(`schema version ${applied} is not supported`);
|
|
733
|
+
}
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
catch (error) {
|
|
737
|
+
if (error instanceof AdmissionMigrationError)
|
|
738
|
+
throw error;
|
|
739
|
+
throw new AdmissionMigrationError("v2 DDL transaction rolled back");
|
|
740
|
+
}
|
|
741
|
+
assertAdmissionSchemaIntegrity(this.#db);
|
|
742
|
+
}
|
|
743
|
+
assertRenameColumnAvailable() {
|
|
744
|
+
const row = this.#db.prepare("SELECT sqlite_version() AS version").get();
|
|
745
|
+
if (compareSqliteVersions(row.version, "3.25.0") < 0) {
|
|
746
|
+
throw new AdmissionMigrationError(`SQLite ${row.version} does not support ALTER TABLE RENAME COLUMN`);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
createInitialV1Schema(appliedAt) {
|
|
750
|
+
const existing = this.#db
|
|
751
|
+
.prepare(`SELECT COUNT(*) AS count FROM sqlite_master
|
|
752
|
+
WHERE type = 'table' AND name IN (
|
|
753
|
+
'turn_requests', 'leases', 'cooldowns', 'turn_payloads',
|
|
754
|
+
'lease_process_identities', 'start_history', 'sessions', 'events',
|
|
755
|
+
'policy_state', 'queued_owner_instances'
|
|
756
|
+
)`)
|
|
757
|
+
.get();
|
|
758
|
+
if (existing.count > 0) {
|
|
759
|
+
throw new AdmissionMigrationError("unversioned admission tables require an explicit migration before use");
|
|
760
|
+
}
|
|
761
|
+
this.#db.exec(`
|
|
762
|
+
CREATE TABLE schema_migrations (
|
|
763
|
+
version INTEGER PRIMARY KEY,
|
|
764
|
+
name TEXT NOT NULL,
|
|
765
|
+
applied_at INTEGER NOT NULL
|
|
766
|
+
);
|
|
767
|
+
CREATE TABLE turn_requests (
|
|
768
|
+
request_id TEXT PRIMARY KEY,
|
|
769
|
+
session_id TEXT NOT NULL,
|
|
770
|
+
parent_id TEXT NOT NULL,
|
|
771
|
+
fingerprint TEXT NOT NULL,
|
|
772
|
+
provider TEXT NOT NULL,
|
|
773
|
+
model TEXT NOT NULL,
|
|
774
|
+
state TEXT NOT NULL,
|
|
775
|
+
enqueued_at INTEGER NOT NULL,
|
|
776
|
+
deadline_at INTEGER NOT NULL,
|
|
777
|
+
lease_generation INTEGER NOT NULL DEFAULT 0,
|
|
778
|
+
terminal_at INTEGER
|
|
779
|
+
);
|
|
780
|
+
CREATE TABLE leases (
|
|
781
|
+
lease_id TEXT PRIMARY KEY,
|
|
782
|
+
request_id TEXT NOT NULL UNIQUE REFERENCES turn_requests(request_id),
|
|
783
|
+
generation INTEGER NOT NULL,
|
|
784
|
+
owner_instance_id TEXT NOT NULL,
|
|
785
|
+
phase TEXT NOT NULL,
|
|
786
|
+
acquired_at INTEGER NOT NULL,
|
|
787
|
+
heartbeat_at INTEGER NOT NULL
|
|
788
|
+
);
|
|
789
|
+
CREATE TABLE cooldowns (
|
|
790
|
+
provider TEXT NOT NULL,
|
|
791
|
+
model TEXT NOT NULL,
|
|
792
|
+
not_before INTEGER NOT NULL,
|
|
793
|
+
updated_at INTEGER NOT NULL,
|
|
794
|
+
PRIMARY KEY (provider, model)
|
|
795
|
+
);
|
|
796
|
+
CREATE TABLE turn_payloads (
|
|
797
|
+
request_id TEXT PRIMARY KEY REFERENCES turn_requests(request_id) ON DELETE CASCADE,
|
|
798
|
+
nonce BLOB NOT NULL,
|
|
799
|
+
ciphertext BLOB NOT NULL,
|
|
800
|
+
auth_tag BLOB NOT NULL,
|
|
801
|
+
key_version INTEGER NOT NULL,
|
|
802
|
+
content_fingerprint TEXT NOT NULL,
|
|
803
|
+
expires_at INTEGER NOT NULL,
|
|
804
|
+
created_at INTEGER NOT NULL
|
|
805
|
+
);
|
|
806
|
+
CREATE TABLE lease_process_identities (
|
|
807
|
+
lease_id TEXT PRIMARY KEY REFERENCES leases(lease_id) ON DELETE CASCADE,
|
|
808
|
+
request_id TEXT NOT NULL REFERENCES turn_requests(request_id),
|
|
809
|
+
lease_generation INTEGER NOT NULL,
|
|
810
|
+
owner_instance_id TEXT NOT NULL,
|
|
811
|
+
prompt_channel TEXT NOT NULL,
|
|
812
|
+
connector_owner_instance_id TEXT NOT NULL,
|
|
813
|
+
connector_created_at TEXT NOT NULL,
|
|
814
|
+
connector_boot_id TEXT NOT NULL,
|
|
815
|
+
connector_pid INTEGER NOT NULL,
|
|
816
|
+
connector_start_time_ticks TEXT NOT NULL,
|
|
817
|
+
connector_pid_namespace_inode INTEGER NOT NULL,
|
|
818
|
+
connector_ppid INTEGER NOT NULL,
|
|
819
|
+
connector_pgrp INTEGER NOT NULL,
|
|
820
|
+
connector_session INTEGER NOT NULL,
|
|
821
|
+
child_boot_id TEXT NOT NULL,
|
|
822
|
+
child_pid INTEGER NOT NULL,
|
|
823
|
+
child_start_time_ticks TEXT NOT NULL,
|
|
824
|
+
child_pid_namespace_inode INTEGER NOT NULL,
|
|
825
|
+
child_ppid INTEGER NOT NULL,
|
|
826
|
+
child_pgrp INTEGER NOT NULL,
|
|
827
|
+
child_session INTEGER NOT NULL,
|
|
828
|
+
recorded_at INTEGER NOT NULL
|
|
829
|
+
);
|
|
830
|
+
CREATE TABLE start_history (
|
|
831
|
+
lease_id TEXT PRIMARY KEY,
|
|
832
|
+
started_at INTEGER NOT NULL
|
|
833
|
+
);
|
|
834
|
+
CREATE TABLE sessions (
|
|
835
|
+
session_id TEXT NOT NULL PRIMARY KEY,
|
|
836
|
+
conversation_id TEXT,
|
|
837
|
+
conversation_cursor INTEGER NOT NULL,
|
|
838
|
+
model TEXT NOT NULL,
|
|
839
|
+
effort TEXT NOT NULL,
|
|
840
|
+
mode TEXT NOT NULL,
|
|
841
|
+
cwd TEXT NOT NULL,
|
|
842
|
+
roots_json TEXT NOT NULL,
|
|
843
|
+
v2_user_message_ids_json TEXT NOT NULL,
|
|
844
|
+
updated_at INTEGER NOT NULL
|
|
845
|
+
);
|
|
846
|
+
CREATE TABLE events (
|
|
847
|
+
event_seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
848
|
+
kind TEXT NOT NULL,
|
|
849
|
+
from_state TEXT NOT NULL,
|
|
850
|
+
to_state TEXT NOT NULL,
|
|
851
|
+
occurred_at INTEGER NOT NULL,
|
|
852
|
+
correlation_hmac TEXT NOT NULL
|
|
853
|
+
);
|
|
854
|
+
CREATE INDEX turn_requests_queue ON turn_requests(state, enqueued_at);
|
|
855
|
+
CREATE INDEX leases_phase ON leases(phase);
|
|
856
|
+
CREATE UNIQUE INDEX lease_process_identities_request ON lease_process_identities(request_id);
|
|
857
|
+
CREATE INDEX start_history_started ON start_history(started_at);
|
|
858
|
+
CREATE INDEX sessions_updated_at_session_id ON sessions(updated_at DESC, session_id ASC);
|
|
859
|
+
CREATE INDEX sessions_cwd_updated_at_session_id ON sessions(cwd, updated_at DESC, session_id ASC);
|
|
860
|
+
CREATE INDEX events_occurred ON events(occurred_at, event_seq);
|
|
861
|
+
`);
|
|
862
|
+
this.#db
|
|
863
|
+
.prepare("INSERT INTO schema_migrations (version, name, applied_at) VALUES (1, 'shared-admission-queue', ?)")
|
|
864
|
+
.run(appliedAt);
|
|
865
|
+
}
|
|
866
|
+
migrateV1ToV2(appliedAt) {
|
|
867
|
+
this.#db.exec(`
|
|
868
|
+
ALTER TABLE turn_requests RENAME COLUMN parent_id TO agent_id;
|
|
869
|
+
ALTER TABLE turn_requests ADD COLUMN queued_owner_instance_id TEXT NULL;
|
|
870
|
+
ALTER TABLE turn_requests ADD COLUMN queued_owner_recorded_at INTEGER NULL;
|
|
871
|
+
CREATE INDEX IF NOT EXISTS turn_requests_queued_owner
|
|
872
|
+
ON turn_requests(queued_owner_instance_id)
|
|
873
|
+
WHERE queued_owner_instance_id IS NOT NULL;
|
|
874
|
+
ALTER TABLE leases ADD COLUMN suspect_since INTEGER NULL;
|
|
875
|
+
ALTER TABLE leases ADD COLUMN suspect_reason TEXT CHECK (suspect_reason IS NULL OR suspect_reason IN ('heartbeat_expired', 'identity_unverifiable'));
|
|
876
|
+
CREATE TABLE policy_state (
|
|
877
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
878
|
+
max_active_turns INTEGER NOT NULL CHECK (max_active_turns >= 1),
|
|
879
|
+
max_concurrent_starts INTEGER NOT NULL CHECK (max_concurrent_starts >= 1),
|
|
880
|
+
min_start_interval_ms INTEGER NOT NULL CHECK (min_start_interval_ms >= 2000),
|
|
881
|
+
queue_timeout_ms INTEGER NOT NULL CHECK (queue_timeout_ms > 0 AND queue_timeout_ms <= 1800000),
|
|
882
|
+
capacity_cooldown_ms INTEGER NOT NULL CHECK (capacity_cooldown_ms >= 30000),
|
|
883
|
+
drain_state TEXT NOT NULL CHECK (drain_state IN ('steady', 'soft_draining_to_1')),
|
|
884
|
+
policy_fingerprint TEXT NOT NULL,
|
|
885
|
+
updated_at INTEGER NOT NULL,
|
|
886
|
+
updated_by_owner_instance_id TEXT NOT NULL
|
|
887
|
+
);
|
|
888
|
+
CREATE TABLE queued_owner_instances (
|
|
889
|
+
owner_instance_id TEXT PRIMARY KEY,
|
|
890
|
+
created_at TEXT NOT NULL,
|
|
891
|
+
boot_id TEXT NOT NULL,
|
|
892
|
+
pid INTEGER NOT NULL,
|
|
893
|
+
start_time_ticks TEXT NOT NULL,
|
|
894
|
+
pid_namespace_inode INTEGER NOT NULL,
|
|
895
|
+
ppid INTEGER NOT NULL,
|
|
896
|
+
pgrp INTEGER NOT NULL,
|
|
897
|
+
session INTEGER NOT NULL,
|
|
898
|
+
recorded_at INTEGER NOT NULL
|
|
899
|
+
);
|
|
900
|
+
`);
|
|
901
|
+
this.#db
|
|
902
|
+
.prepare("INSERT INTO schema_migrations (version, name, applied_at) VALUES (2, 'shared-admission-queue-v2', ?)")
|
|
903
|
+
.run(appliedAt);
|
|
904
|
+
}
|
|
905
|
+
migrateV2ToV3(appliedAt) {
|
|
906
|
+
this.#db.exec(`
|
|
907
|
+
CREATE TABLE policy_state_v3 (
|
|
908
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
909
|
+
max_active_turns INTEGER NOT NULL CHECK (max_active_turns >= 1),
|
|
910
|
+
max_concurrent_starts INTEGER NOT NULL CHECK (max_concurrent_starts >= 1),
|
|
911
|
+
min_start_interval_ms INTEGER NOT NULL CHECK (min_start_interval_ms >= 2000),
|
|
912
|
+
queue_timeout_ms INTEGER NOT NULL CHECK (queue_timeout_ms > 0 AND queue_timeout_ms <= 1800000),
|
|
913
|
+
capacity_cooldown_ms INTEGER NOT NULL CHECK (capacity_cooldown_ms >= 30000),
|
|
914
|
+
drain_state TEXT NOT NULL CHECK (drain_state IN ('steady', 'soft_draining_to_1')),
|
|
915
|
+
policy_fingerprint TEXT NOT NULL,
|
|
916
|
+
updated_at INTEGER NOT NULL,
|
|
917
|
+
updated_by_owner_instance_id TEXT NOT NULL
|
|
918
|
+
);
|
|
919
|
+
INSERT INTO policy_state_v3 SELECT * FROM policy_state;
|
|
920
|
+
DROP TABLE policy_state;
|
|
921
|
+
ALTER TABLE policy_state_v3 RENAME TO policy_state;
|
|
922
|
+
`);
|
|
923
|
+
this.#db
|
|
924
|
+
.prepare("INSERT INTO schema_migrations (version, name, applied_at) VALUES (3, 'shared-admission-queue-v3', ?)")
|
|
925
|
+
.run(appliedAt);
|
|
926
|
+
}
|
|
927
|
+
transaction(fn) {
|
|
928
|
+
return this.#db.transaction(fn)();
|
|
929
|
+
}
|
|
930
|
+
runInjectedTransactionFault(callback) {
|
|
931
|
+
if (callback === undefined)
|
|
932
|
+
return;
|
|
933
|
+
try {
|
|
934
|
+
callback();
|
|
935
|
+
}
|
|
936
|
+
catch {
|
|
937
|
+
throw new AdmissionControllerInjectedFaultError();
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
persistProcessIdentityAndDispatchIntent(input) {
|
|
941
|
+
const record = normalizeVerifiedLinuxProcessRecord(input);
|
|
942
|
+
if (record === null)
|
|
943
|
+
return { status: "not_committed", reason: "invalid_process_identity" };
|
|
944
|
+
try {
|
|
945
|
+
return this.transaction(() => this.persistProcessIdentityAndDispatchIntentInTransaction(record));
|
|
946
|
+
}
|
|
947
|
+
catch (error) {
|
|
948
|
+
// Do not expose driver errors or partially durable state to a prompt writer.
|
|
949
|
+
return ((isSqliteTransactionContention(error) ? this.recheckDispatchIdentityAfterContention(record) : null) ?? {
|
|
950
|
+
status: "not_committed",
|
|
951
|
+
reason: "transaction_fault"
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
/** A failed writer may inspect only a committed, exact winner; it never retries a write. */
|
|
956
|
+
recheckDispatchIdentityAfterContention(record) {
|
|
957
|
+
for (let attempt = 0; attempt < MAX_DISPATCH_CONTENTION_RECHECKS; attempt += 1) {
|
|
958
|
+
try {
|
|
959
|
+
const outcome = this.transaction(() => this.inspectDispatchIdentityContentionWinner(record));
|
|
960
|
+
if (outcome !== "pending")
|
|
961
|
+
return outcome;
|
|
962
|
+
}
|
|
963
|
+
catch (error) {
|
|
964
|
+
if (!isSqliteTransactionContention(error))
|
|
965
|
+
return null;
|
|
966
|
+
}
|
|
967
|
+
if (attempt < MAX_DISPATCH_CONTENTION_RECHECKS - 1) {
|
|
968
|
+
Atomics.wait(dispatchContentionRetrySignal, 0, 0, DISPATCH_CONTENTION_RECHECK_DELAY_MS);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
return null;
|
|
972
|
+
}
|
|
973
|
+
/** Reads the lease and identity under one snapshot so no partial winner can be inferred. */
|
|
974
|
+
inspectDispatchIdentityContentionWinner(record) {
|
|
975
|
+
const lease = this.#db
|
|
976
|
+
.prepare(`SELECT lease.lease_id, lease.request_id, lease.generation, lease.owner_instance_id,
|
|
977
|
+
lease.phase, request.state AS request_state,
|
|
978
|
+
request.lease_generation AS request_lease_generation
|
|
979
|
+
FROM leases AS lease
|
|
980
|
+
JOIN turn_requests AS request ON request.request_id = lease.request_id
|
|
981
|
+
WHERE lease.lease_id = ?`)
|
|
982
|
+
.get(record.leaseId);
|
|
983
|
+
if (lease === undefined ||
|
|
984
|
+
lease.request_id !== record.requestId ||
|
|
985
|
+
lease.generation !== record.generation ||
|
|
986
|
+
lease.owner_instance_id !== record.ownerInstanceId ||
|
|
987
|
+
lease.request_lease_generation !== record.generation) {
|
|
988
|
+
return null;
|
|
989
|
+
}
|
|
990
|
+
const identity = this.findLeaseProcessIdentity(record.leaseId);
|
|
991
|
+
if (lease.phase === "dispatch_intent" && lease.request_state === "dispatch_intent") {
|
|
992
|
+
if (identity === undefined ||
|
|
993
|
+
identity.request_id !== record.requestId ||
|
|
994
|
+
identity.lease_generation !== record.generation ||
|
|
995
|
+
identity.owner_instance_id !== record.ownerInstanceId) {
|
|
996
|
+
return null;
|
|
997
|
+
}
|
|
998
|
+
return sameLeaseProcessIdentity(identity, record)
|
|
999
|
+
? { status: "committed", idempotent: true }
|
|
1000
|
+
: { status: "not_committed", reason: "conflicting_intent" };
|
|
1001
|
+
}
|
|
1002
|
+
return lease.phase === "starting" && lease.request_state === "starting" && identity === undefined
|
|
1003
|
+
? "pending"
|
|
1004
|
+
: null;
|
|
1005
|
+
}
|
|
1006
|
+
persistProcessIdentityAndDispatchIntentInTransaction(record) {
|
|
1007
|
+
const lease = this.#db
|
|
1008
|
+
.prepare("SELECT lease_id, request_id, generation, owner_instance_id, phase FROM leases WHERE lease_id = ?")
|
|
1009
|
+
.get(record.leaseId);
|
|
1010
|
+
if (lease === undefined ||
|
|
1011
|
+
lease.request_id !== record.requestId ||
|
|
1012
|
+
lease.generation !== record.generation ||
|
|
1013
|
+
lease.owner_instance_id !== record.ownerInstanceId) {
|
|
1014
|
+
return { status: "not_committed", reason: "stale_lease" };
|
|
1015
|
+
}
|
|
1016
|
+
const existing = this.findLeaseProcessIdentity(record.leaseId);
|
|
1017
|
+
if (lease.phase === "dispatch_intent") {
|
|
1018
|
+
return existing !== undefined && sameLeaseProcessIdentity(existing, record)
|
|
1019
|
+
? { status: "committed", idempotent: true }
|
|
1020
|
+
: { status: "not_committed", reason: "conflicting_intent" };
|
|
1021
|
+
}
|
|
1022
|
+
if (lease.phase !== "starting")
|
|
1023
|
+
return { status: "not_committed", reason: "stale_lease" };
|
|
1024
|
+
if (existing !== undefined)
|
|
1025
|
+
return { status: "not_committed", reason: "conflicting_intent" };
|
|
1026
|
+
const committedAt = Date.now();
|
|
1027
|
+
const phase = this.#db
|
|
1028
|
+
.prepare(`UPDATE leases
|
|
1029
|
+
SET phase = 'dispatch_intent', heartbeat_at = ?
|
|
1030
|
+
WHERE lease_id = ? AND owner_instance_id = ? AND generation = ? AND phase = 'starting'`)
|
|
1031
|
+
.run(committedAt, record.leaseId, record.ownerInstanceId, record.generation);
|
|
1032
|
+
if (phase.changes !== 1)
|
|
1033
|
+
return { status: "not_committed", reason: "stale_lease" };
|
|
1034
|
+
this.insertLeaseProcessIdentity(record, committedAt);
|
|
1035
|
+
this.#faultInjection?.afterProcessIdentityPersisted?.();
|
|
1036
|
+
const request = this.#db
|
|
1037
|
+
.prepare(`UPDATE turn_requests
|
|
1038
|
+
SET state = 'dispatch_intent'
|
|
1039
|
+
WHERE request_id = ? AND lease_generation = ? AND state = 'starting'`)
|
|
1040
|
+
.run(record.requestId, record.generation);
|
|
1041
|
+
if (request.changes !== 1) {
|
|
1042
|
+
throw new Error("lease and request state diverged at dispatch intent persistence");
|
|
1043
|
+
}
|
|
1044
|
+
this.journalTransition("request_dispatch_intent", record.requestId, "starting", "dispatch_intent", committedAt);
|
|
1045
|
+
return { status: "committed", idempotent: false };
|
|
1046
|
+
}
|
|
1047
|
+
findLeaseProcessIdentity(leaseId) {
|
|
1048
|
+
return this.#db
|
|
1049
|
+
.prepare(`SELECT lease_id, request_id, lease_generation, owner_instance_id, prompt_channel,
|
|
1050
|
+
connector_owner_instance_id, connector_created_at, connector_boot_id, connector_pid,
|
|
1051
|
+
connector_start_time_ticks, connector_pid_namespace_inode, connector_ppid, connector_pgrp,
|
|
1052
|
+
connector_session, child_boot_id, child_pid, child_start_time_ticks, child_pid_namespace_inode,
|
|
1053
|
+
child_ppid, child_pgrp, child_session
|
|
1054
|
+
FROM lease_process_identities
|
|
1055
|
+
WHERE lease_id = ?`)
|
|
1056
|
+
.get(leaseId);
|
|
1057
|
+
}
|
|
1058
|
+
insertLeaseProcessIdentity(record, recordedAt) {
|
|
1059
|
+
const { connector, child } = record.processIdentity;
|
|
1060
|
+
this.#db
|
|
1061
|
+
.prepare(`INSERT INTO lease_process_identities (
|
|
1062
|
+
lease_id, request_id, lease_generation, owner_instance_id, prompt_channel,
|
|
1063
|
+
connector_owner_instance_id, connector_created_at, connector_boot_id, connector_pid,
|
|
1064
|
+
connector_start_time_ticks, connector_pid_namespace_inode, connector_ppid, connector_pgrp,
|
|
1065
|
+
connector_session, child_boot_id, child_pid, child_start_time_ticks, child_pid_namespace_inode,
|
|
1066
|
+
child_ppid, child_pgrp, child_session, recorded_at
|
|
1067
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1068
|
+
.run(record.leaseId, record.requestId, record.generation, record.ownerInstanceId, record.promptChannel, connector.ownerInstanceId, connector.createdAt, connector.bootId, connector.pid, connector.startTimeTicks, connector.pidNamespaceInode, connector.ppid, connector.pgrp, connector.session, child.bootId, child.pid, child.startTimeTicks, child.pidNamespaceInode, child.ppid, child.pgrp, child.session, recordedAt);
|
|
1069
|
+
}
|
|
1070
|
+
persistQueuedOwnerReference(input, recordedAt, requestExisted) {
|
|
1071
|
+
const owner = normalizeVerifiedLinuxConnectorIdentity(input.ownerIdentity ?? this.#queuedOwnerIdentity);
|
|
1072
|
+
const existingOwner = this.findQueuedOwnerIdentity(owner.ownerInstanceId);
|
|
1073
|
+
if (existingOwner === undefined) {
|
|
1074
|
+
this.insertQueuedOwnerIdentity(owner, recordedAt);
|
|
1075
|
+
}
|
|
1076
|
+
else if (!sameQueuedOwnerIdentity(existingOwner, owner)) {
|
|
1077
|
+
throw new AdmissionConflictError(input.requestId);
|
|
1078
|
+
}
|
|
1079
|
+
const request = this.#db
|
|
1080
|
+
.prepare(`SELECT state, queued_owner_instance_id
|
|
1081
|
+
FROM turn_requests
|
|
1082
|
+
WHERE request_id = ?`)
|
|
1083
|
+
.get(input.requestId);
|
|
1084
|
+
if (request === undefined)
|
|
1085
|
+
throw new Error("unknown request");
|
|
1086
|
+
if (request.state !== "queued")
|
|
1087
|
+
throw new Error("request is no longer queued");
|
|
1088
|
+
if (request.queued_owner_instance_id === owner.ownerInstanceId)
|
|
1089
|
+
return;
|
|
1090
|
+
if (request.queued_owner_instance_id !== null) {
|
|
1091
|
+
if (requestExisted)
|
|
1092
|
+
return;
|
|
1093
|
+
throw new AdmissionConflictError(input.requestId);
|
|
1094
|
+
}
|
|
1095
|
+
const result = this.#db
|
|
1096
|
+
.prepare(`UPDATE turn_requests
|
|
1097
|
+
SET queued_owner_instance_id = ?, queued_owner_recorded_at = ?
|
|
1098
|
+
WHERE request_id = ?
|
|
1099
|
+
AND state = 'queued'
|
|
1100
|
+
AND queued_owner_instance_id IS NULL`)
|
|
1101
|
+
.run(owner.ownerInstanceId, recordedAt, input.requestId);
|
|
1102
|
+
if (result.changes !== 1)
|
|
1103
|
+
throw new AdmissionConflictError(input.requestId);
|
|
1104
|
+
}
|
|
1105
|
+
findQueuedOwnerIdentity(ownerInstanceId) {
|
|
1106
|
+
return this.#db
|
|
1107
|
+
.prepare(`SELECT owner_instance_id, created_at, boot_id, pid, start_time_ticks,
|
|
1108
|
+
pid_namespace_inode, ppid, pgrp, session
|
|
1109
|
+
FROM queued_owner_instances
|
|
1110
|
+
WHERE owner_instance_id = ?`)
|
|
1111
|
+
.get(ownerInstanceId);
|
|
1112
|
+
}
|
|
1113
|
+
insertQueuedOwnerIdentity(owner, recordedAt) {
|
|
1114
|
+
this.#db
|
|
1115
|
+
.prepare(`INSERT INTO queued_owner_instances (
|
|
1116
|
+
owner_instance_id, created_at, boot_id, pid, start_time_ticks,
|
|
1117
|
+
pid_namespace_inode, ppid, pgrp, session, recorded_at
|
|
1118
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1119
|
+
.run(owner.ownerInstanceId, owner.createdAt, owner.bootId, owner.pid, owner.startTimeTicks, owner.pidNamespaceInode, owner.ppid, owner.pgrp, owner.session, recordedAt);
|
|
1120
|
+
}
|
|
1121
|
+
expireQueued(now) {
|
|
1122
|
+
const expired = this.#db
|
|
1123
|
+
.prepare("SELECT request_id FROM turn_requests WHERE state = 'queued' AND deadline_at <= ? ORDER BY request_id ASC")
|
|
1124
|
+
.all(now);
|
|
1125
|
+
for (const row of expired) {
|
|
1126
|
+
const result = this.#db
|
|
1127
|
+
.prepare("UPDATE turn_requests SET state = 'queue_timeout', terminal_at = ? WHERE request_id = ? AND state = 'queued'")
|
|
1128
|
+
.run(now, row.request_id);
|
|
1129
|
+
if (result.changes === 1) {
|
|
1130
|
+
this.#db.prepare("DELETE FROM turn_payloads WHERE request_id = ?").run(row.request_id);
|
|
1131
|
+
this.journalTransition("request_queue_timed_out", row.request_id, "queued", "queue_timeout", now);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
activeLeaseCount() {
|
|
1136
|
+
const row = this.#db
|
|
1137
|
+
.prepare(`SELECT COUNT(*) AS count FROM leases
|
|
1138
|
+
WHERE phase IN ('admitted', 'starting', 'dispatch_intent', 'dispatch_ambiguous', 'active', 'recovery_required')`)
|
|
1139
|
+
.get();
|
|
1140
|
+
return row.count;
|
|
1141
|
+
}
|
|
1142
|
+
hasSeatCapacity() {
|
|
1143
|
+
const policyState = this.readPolicyStateInTransaction();
|
|
1144
|
+
if (policyState !== undefined) {
|
|
1145
|
+
const drainState = policyDrainState(policyState);
|
|
1146
|
+
if (drainState === "soft_draining_to_1")
|
|
1147
|
+
return false;
|
|
1148
|
+
return this.activeLeaseCount() < policyStateInteger(policyState.max_active_turns, "durable policy max_active_turns");
|
|
1149
|
+
}
|
|
1150
|
+
return this.activeLeaseCount() < this.policy.maxActiveTurns;
|
|
1151
|
+
}
|
|
1152
|
+
markSuspect(fence, now, reason) {
|
|
1153
|
+
validateTimestamp(now, "lease suspect timestamp");
|
|
1154
|
+
validateSuspectReason(reason);
|
|
1155
|
+
const result = this.#db
|
|
1156
|
+
.prepare(`UPDATE leases
|
|
1157
|
+
SET suspect_since = COALESCE(suspect_since, ?),
|
|
1158
|
+
suspect_reason = ?
|
|
1159
|
+
WHERE lease_id = ?
|
|
1160
|
+
AND owner_instance_id = ?
|
|
1161
|
+
AND generation = ?`)
|
|
1162
|
+
.run(now, reason, fence.leaseId, fence.ownerInstanceId, fence.generation);
|
|
1163
|
+
return result.changes === 1;
|
|
1164
|
+
}
|
|
1165
|
+
completeSoftDrainTo1IfSettled(ownerInstanceId, now) {
|
|
1166
|
+
const row = this.readPolicyStateInTransaction();
|
|
1167
|
+
if (row === undefined || policyDrainState(row) !== "soft_draining_to_1")
|
|
1168
|
+
return;
|
|
1169
|
+
if (this.activeLeaseCount() !== 0)
|
|
1170
|
+
return;
|
|
1171
|
+
const terminalPolicy = policyFromState(row, 1);
|
|
1172
|
+
const policyFingerprint = this.policyFingerprint(terminalPolicy);
|
|
1173
|
+
const result = this.#db
|
|
1174
|
+
.prepare(`UPDATE policy_state
|
|
1175
|
+
SET max_active_turns = 1,
|
|
1176
|
+
max_concurrent_starts = ?,
|
|
1177
|
+
min_start_interval_ms = ?,
|
|
1178
|
+
queue_timeout_ms = ?,
|
|
1179
|
+
capacity_cooldown_ms = ?,
|
|
1180
|
+
drain_state = 'steady',
|
|
1181
|
+
policy_fingerprint = ?,
|
|
1182
|
+
updated_at = ?,
|
|
1183
|
+
updated_by_owner_instance_id = ?
|
|
1184
|
+
WHERE id = 1 AND drain_state = 'soft_draining_to_1'`)
|
|
1185
|
+
.run(terminalPolicy.maxConcurrentStarts, terminalPolicy.minStartIntervalMs, terminalPolicy.queueTimeoutMs, terminalPolicy.capacityCooldownMs, policyFingerprint, now, ownerInstanceId);
|
|
1186
|
+
if (result.changes !== 1)
|
|
1187
|
+
throw new AdmissionRuntimeError("soft drain could not be completed atomically");
|
|
1188
|
+
this.journalTransition("policy_drain_completed", "policy-state", "policy_soft_draining_to_1", "policy_steady", now);
|
|
1189
|
+
}
|
|
1190
|
+
hasDispatchCapacity(now) {
|
|
1191
|
+
if (!this.hasSeatCapacity())
|
|
1192
|
+
return false;
|
|
1193
|
+
const starts = this.#db
|
|
1194
|
+
.prepare("SELECT COUNT(*) AS count FROM leases WHERE phase IN ('admitted', 'starting', 'dispatch_intent')")
|
|
1195
|
+
.get();
|
|
1196
|
+
if (starts.count >= this.policy.maxConcurrentStarts)
|
|
1197
|
+
return false;
|
|
1198
|
+
const cutoff = now - this.policy.minStartIntervalMs;
|
|
1199
|
+
const recentStart = this.#db
|
|
1200
|
+
.prepare("SELECT 1 FROM start_history WHERE started_at > ? LIMIT 1")
|
|
1201
|
+
.get(cutoff);
|
|
1202
|
+
if (recentStart !== undefined)
|
|
1203
|
+
return false;
|
|
1204
|
+
const recentReservation = this.#db
|
|
1205
|
+
.prepare("SELECT 1 FROM leases WHERE phase IN ('admitted', 'starting', 'dispatch_intent') AND acquired_at > ? LIMIT 1")
|
|
1206
|
+
.get(cutoff);
|
|
1207
|
+
return recentReservation === undefined;
|
|
1208
|
+
}
|
|
1209
|
+
reserveAdmission(candidate, now, ownerInstanceId) {
|
|
1210
|
+
const leaseId = randomUUID();
|
|
1211
|
+
const generation = candidate.lease_generation + 1;
|
|
1212
|
+
const reserved = this.#db
|
|
1213
|
+
.prepare("UPDATE turn_requests SET state = 'admitted', lease_generation = ? WHERE request_id = ? AND state = 'queued'")
|
|
1214
|
+
.run(generation, candidate.request_id);
|
|
1215
|
+
if (reserved.changes !== 1)
|
|
1216
|
+
throw new Error("selected admission request is no longer queued");
|
|
1217
|
+
this.#db
|
|
1218
|
+
.prepare(`INSERT INTO leases (lease_id, request_id, generation, owner_instance_id, phase, acquired_at, heartbeat_at)
|
|
1219
|
+
VALUES (?, ?, ?, ?, 'admitted', ?, ?)`)
|
|
1220
|
+
.run(leaseId, candidate.request_id, generation, ownerInstanceId, now, now);
|
|
1221
|
+
this.journalTransition("request_admitted", candidate.request_id, "queued", "admitted", now);
|
|
1222
|
+
return Object.freeze({ leaseId, requestId: candidate.request_id, generation, ownerInstanceId });
|
|
1223
|
+
}
|
|
1224
|
+
orderedQueuedRequests() {
|
|
1225
|
+
return this.#db
|
|
1226
|
+
.prepare(`SELECT turn_requests.request_id, session_id, agent_id, fingerprint, provider, model,
|
|
1227
|
+
turn_requests.state, enqueued_at, lease_generation
|
|
1228
|
+
FROM turn_requests
|
|
1229
|
+
INNER JOIN turn_payloads payload ON payload.request_id = turn_requests.request_id
|
|
1230
|
+
WHERE turn_requests.state = 'queued' AND payload.content_fingerprint IS NOT NULL
|
|
1231
|
+
ORDER BY enqueued_at ASC, turn_requests.request_id ASC`)
|
|
1232
|
+
.all();
|
|
1233
|
+
}
|
|
1234
|
+
selectEligibleRequest(now) {
|
|
1235
|
+
const rows = this.orderedQueuedRequests();
|
|
1236
|
+
const activeAgents = new Set(this.#db
|
|
1237
|
+
.prepare(`SELECT DISTINCT request.agent_id AS agent_id
|
|
1238
|
+
FROM leases lease JOIN turn_requests request ON request.request_id = lease.request_id
|
|
1239
|
+
WHERE lease.phase IN ('admitted', 'starting', 'dispatch_intent', 'dispatch_ambiguous', 'active', 'recovery_required')`)
|
|
1240
|
+
.all().map((row) => row.agent_id));
|
|
1241
|
+
const eligible = rows.filter((row) => !this.isCooldownActive(row.provider, row.model, now));
|
|
1242
|
+
if (eligible.length === 0)
|
|
1243
|
+
return null;
|
|
1244
|
+
return eligible.find((row) => !activeAgents.has(row.agent_id)) ?? eligible[0];
|
|
1245
|
+
}
|
|
1246
|
+
isCooldownActive(provider, model, now) {
|
|
1247
|
+
return this.cooldownUntil(provider, model, now) !== null;
|
|
1248
|
+
}
|
|
1249
|
+
cooldownUntil(provider, model, now) {
|
|
1250
|
+
const row = this.#db
|
|
1251
|
+
.prepare("SELECT not_before FROM cooldowns WHERE provider = ? AND model = ?")
|
|
1252
|
+
.get(provider, model);
|
|
1253
|
+
return row !== undefined && row.not_before > now ? row.not_before : null;
|
|
1254
|
+
}
|
|
1255
|
+
requireLease(fence) {
|
|
1256
|
+
const lease = this.requireLeaseById(fence.leaseId);
|
|
1257
|
+
if (lease.generation !== fence.generation || lease.owner_instance_id !== fence.ownerInstanceId) {
|
|
1258
|
+
throw new LeaseFenceError(fence.leaseId);
|
|
1259
|
+
}
|
|
1260
|
+
return lease;
|
|
1261
|
+
}
|
|
1262
|
+
requireLeaseById(leaseId) {
|
|
1263
|
+
const lease = this.#db
|
|
1264
|
+
.prepare("SELECT lease_id, request_id, generation, owner_instance_id, phase FROM leases WHERE lease_id = ?")
|
|
1265
|
+
.get(leaseId);
|
|
1266
|
+
if (!lease)
|
|
1267
|
+
throw new Error("unknown lease");
|
|
1268
|
+
return lease;
|
|
1269
|
+
}
|
|
1270
|
+
requireRequest(requestId) {
|
|
1271
|
+
const row = this.#db.prepare("SELECT 1 FROM turn_requests WHERE request_id = ?").get(requestId);
|
|
1272
|
+
if (!row)
|
|
1273
|
+
throw new Error("unknown request");
|
|
1274
|
+
}
|
|
1275
|
+
requireRequestState(requestId) {
|
|
1276
|
+
const row = this.#db
|
|
1277
|
+
.prepare("SELECT state FROM turn_requests WHERE request_id = ?")
|
|
1278
|
+
.get(requestId);
|
|
1279
|
+
if (!row)
|
|
1280
|
+
throw new Error("unknown request");
|
|
1281
|
+
return row;
|
|
1282
|
+
}
|
|
1283
|
+
requireEncryptionKey() {
|
|
1284
|
+
if (!this.#encryptionKey)
|
|
1285
|
+
throw new Error("payload persistence requires an encryption key");
|
|
1286
|
+
return this.#encryptionKey;
|
|
1287
|
+
}
|
|
1288
|
+
requireContentFingerprintKey() {
|
|
1289
|
+
if (!this.#contentFingerprintKey) {
|
|
1290
|
+
throw new Error("admission event correlation requires a content fingerprint key");
|
|
1291
|
+
}
|
|
1292
|
+
return this.#contentFingerprintKey;
|
|
1293
|
+
}
|
|
1294
|
+
journalTransition(kind, requestId, fromState, toState, occurredAt) {
|
|
1295
|
+
validateIdentifier(requestId, "event correlation input");
|
|
1296
|
+
validateTimestamp(occurredAt, "event timestamp");
|
|
1297
|
+
if (!isAllowedSanitizedEventTransition(kind, fromState, toState)) {
|
|
1298
|
+
throw new Error("admission event transition is not allowlisted");
|
|
1299
|
+
}
|
|
1300
|
+
const correlationHmac = createHmac("sha256", this.requireContentFingerprintKey())
|
|
1301
|
+
.update(JSON.stringify(["paseo-agy-acp", "admission-event-correlation", 1]), "utf8")
|
|
1302
|
+
.update(Buffer.from([0]))
|
|
1303
|
+
.update(requestId, "utf8")
|
|
1304
|
+
.digest("hex");
|
|
1305
|
+
this.#db
|
|
1306
|
+
.prepare(`INSERT INTO events (kind, from_state, to_state, occurred_at, correlation_hmac)
|
|
1307
|
+
VALUES (?, ?, ?, ?, ?)`)
|
|
1308
|
+
.run(kind, fromState, toState, occurredAt, correlationHmac);
|
|
1309
|
+
}
|
|
1310
|
+
encrypt(plaintext, aad) {
|
|
1311
|
+
const nonce = randomBytes(12);
|
|
1312
|
+
const cipher = createCipheriv("aes-256-gcm", this.requireEncryptionKey(), nonce);
|
|
1313
|
+
cipher.setAAD(Buffer.from(aad, "utf8"));
|
|
1314
|
+
return {
|
|
1315
|
+
nonce,
|
|
1316
|
+
ciphertext: Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]),
|
|
1317
|
+
authTag: cipher.getAuthTag()
|
|
1318
|
+
};
|
|
1319
|
+
}
|
|
1320
|
+
decrypt(payload, aad) {
|
|
1321
|
+
const decipher = createDecipheriv("aes-256-gcm", this.requireEncryptionKey(), payload.nonce);
|
|
1322
|
+
decipher.setAAD(Buffer.from(aad, "utf8"));
|
|
1323
|
+
decipher.setAuthTag(payload.authTag);
|
|
1324
|
+
return Buffer.concat([decipher.update(payload.ciphertext), decipher.final()]).toString("utf8");
|
|
1325
|
+
}
|
|
1326
|
+
enqueueRequest(input) {
|
|
1327
|
+
validateEnqueueRequest(input);
|
|
1328
|
+
const agentId = requestAgentId(input);
|
|
1329
|
+
const existing = this.#db
|
|
1330
|
+
.prepare(`SELECT session_id, agent_id, fingerprint, provider, model
|
|
1331
|
+
FROM turn_requests WHERE request_id = ?`)
|
|
1332
|
+
.get(input.requestId);
|
|
1333
|
+
if (existing) {
|
|
1334
|
+
if (existing.session_id !== input.sessionId ||
|
|
1335
|
+
existing.agent_id !== agentId ||
|
|
1336
|
+
existing.fingerprint !== input.fingerprint ||
|
|
1337
|
+
existing.provider !== input.provider ||
|
|
1338
|
+
existing.model !== input.model) {
|
|
1339
|
+
throw new AdmissionConflictError(input.requestId);
|
|
1340
|
+
}
|
|
1341
|
+
return { requestId: input.requestId, existed: true };
|
|
1342
|
+
}
|
|
1343
|
+
this.#db
|
|
1344
|
+
.prepare(`INSERT INTO turn_requests
|
|
1345
|
+
(request_id, session_id, agent_id, fingerprint, provider, model, state, enqueued_at, deadline_at)
|
|
1346
|
+
VALUES (?, ?, ?, ?, ?, ?, 'queued', ?, ?)`)
|
|
1347
|
+
.run(input.requestId, input.sessionId, agentId, input.fingerprint, input.provider, input.model, input.now, input.now + this.policy.queueTimeoutMs);
|
|
1348
|
+
this.journalTransition("request_enqueued", input.requestId, "absent", "queued", input.now);
|
|
1349
|
+
return { requestId: input.requestId, existed: false };
|
|
1350
|
+
}
|
|
1351
|
+
insertPayload(requestId, encrypted, keyVersion, contentFingerprint, expiresAt, now) {
|
|
1352
|
+
this.#db
|
|
1353
|
+
.prepare(`INSERT INTO turn_payloads
|
|
1354
|
+
(request_id, nonce, ciphertext, auth_tag, key_version, content_fingerprint, expires_at, created_at)
|
|
1355
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
1356
|
+
.run(requestId, encrypted.nonce, encrypted.ciphertext, encrypted.authTag, keyVersion, contentFingerprint, expiresAt, now);
|
|
1357
|
+
}
|
|
1358
|
+
contentFingerprint(domain, id, plaintext) {
|
|
1359
|
+
return createHmac("sha256", this.requireContentFingerprintKey())
|
|
1360
|
+
.update(JSON.stringify(["paseo-agy-acp", domain, "content", 1, id]), "utf8")
|
|
1361
|
+
.update(Buffer.from([0]))
|
|
1362
|
+
.update(plaintext, "utf8")
|
|
1363
|
+
.digest("hex");
|
|
1364
|
+
}
|
|
1365
|
+
payloadAad(requestId, keyVersion) {
|
|
1366
|
+
return JSON.stringify(["paseo-agy-acp", "turn", 1, requestId, keyVersion]);
|
|
1367
|
+
}
|
|
1368
|
+
policyFingerprint(policy) {
|
|
1369
|
+
return createHmac("sha256", this.requireContentFingerprintKey())
|
|
1370
|
+
.update(JSON.stringify(["paseo-agy-acp", "admission-policy", 1]), "utf8")
|
|
1371
|
+
.update(Buffer.from([0]))
|
|
1372
|
+
.update(JSON.stringify(normalizedPolicyTuple(policy)), "utf8")
|
|
1373
|
+
.digest("hex");
|
|
1374
|
+
}
|
|
1375
|
+
assertDurablePolicyMatchInTransaction(policy, policyFingerprint) {
|
|
1376
|
+
const row = this.readPolicyStateInTransaction();
|
|
1377
|
+
if (row === undefined) {
|
|
1378
|
+
throw new AdmissionRuntimeError("durable policy has not been claimed");
|
|
1379
|
+
}
|
|
1380
|
+
if (!policyStateMatches(row, policy, policyFingerprint)) {
|
|
1381
|
+
throw new AdmissionRuntimeError("durable policy does not match shared runtime policy");
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
readPolicyStateInTransaction() {
|
|
1385
|
+
return this.#db
|
|
1386
|
+
.prepare(`SELECT max_active_turns, max_concurrent_starts, min_start_interval_ms,
|
|
1387
|
+
queue_timeout_ms, capacity_cooldown_ms, drain_state, policy_fingerprint
|
|
1388
|
+
FROM policy_state WHERE id = 1`)
|
|
1389
|
+
.get();
|
|
1390
|
+
}
|
|
1391
|
+
requirePolicyStateInTransaction() {
|
|
1392
|
+
const row = this.readPolicyStateInTransaction();
|
|
1393
|
+
if (row === undefined)
|
|
1394
|
+
throw new AdmissionRuntimeError("durable policy has not been claimed");
|
|
1395
|
+
return row;
|
|
1396
|
+
}
|
|
1397
|
+
validatePayloadExpiry(now, expiresAt) {
|
|
1398
|
+
if (!Number.isFinite(now) || !Number.isFinite(expiresAt) || expiresAt <= now) {
|
|
1399
|
+
throw new Error("payload expiry must be after persistence time");
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
transition(fence, expected, next, now) {
|
|
1403
|
+
this.transaction(() => {
|
|
1404
|
+
const lease = this.requireLease(fence);
|
|
1405
|
+
if (lease.phase !== expected)
|
|
1406
|
+
throw new Error(`lease is not ${expected}`);
|
|
1407
|
+
this.setLeasePhase(lease, next, now);
|
|
1408
|
+
});
|
|
1409
|
+
}
|
|
1410
|
+
setLeasePhase(lease, phase, now) {
|
|
1411
|
+
const result = this.#db
|
|
1412
|
+
.prepare("UPDATE leases SET phase = ?, heartbeat_at = ? WHERE lease_id = ? AND owner_instance_id = ? AND generation = ?")
|
|
1413
|
+
.run(phase, now, lease.lease_id, lease.owner_instance_id, lease.generation);
|
|
1414
|
+
if (result.changes !== 1)
|
|
1415
|
+
throw new LeaseFenceError(lease.lease_id);
|
|
1416
|
+
this.#db.prepare("UPDATE turn_requests SET state = ? WHERE request_id = ?").run(phase, lease.request_id);
|
|
1417
|
+
this.journalTransition(requestTransitionKind(lease.phase, phase), lease.request_id, lease.phase, phase, now);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
const SANITIZED_EVENT_TRANSITIONS = new Set([
|
|
1421
|
+
transitionSignature("request_enqueued", "absent", "queued"),
|
|
1422
|
+
transitionSignature("request_cancelled", "queued", "cancelled"),
|
|
1423
|
+
transitionSignature("queued_owner_dead", "queued", "cancelled"),
|
|
1424
|
+
...["admitted", "starting", "dispatch_intent"].flatMap((fromState) => [
|
|
1425
|
+
transitionSignature("request_abandoned", fromState, "failed"),
|
|
1426
|
+
transitionSignature("request_abandoned", fromState, "cancelled")
|
|
1427
|
+
]),
|
|
1428
|
+
transitionSignature("request_queue_timed_out", "queued", "queue_timeout"),
|
|
1429
|
+
transitionSignature("request_admitted", "queued", "admitted"),
|
|
1430
|
+
transitionSignature("request_starting", "admitted", "starting"),
|
|
1431
|
+
transitionSignature("request_dispatch_intent", "starting", "dispatch_intent"),
|
|
1432
|
+
transitionSignature("request_active", "dispatch_intent", "active"),
|
|
1433
|
+
transitionSignature("request_dispatch_ambiguous", "dispatch_intent", "dispatch_ambiguous"),
|
|
1434
|
+
transitionSignature("request_provider_terminal", "active", "provider_terminal"),
|
|
1435
|
+
transitionSignature("request_released", "provider_terminal", "completed"),
|
|
1436
|
+
transitionSignature("request_released", "provider_terminal", "failed"),
|
|
1437
|
+
transitionSignature("request_released", "provider_terminal", "cancelled"),
|
|
1438
|
+
...["admitted", "starting", "dispatch_intent", "dispatch_ambiguous", "active", "recovery_required"].map((fromState) => transitionSignature("request_recovery_required", fromState, "recovery_required")),
|
|
1439
|
+
...["admitted", "starting", "dispatch_intent", "dispatch_ambiguous", "active", "recovery_required"].map((fromState) => transitionSignature("request_recovery_seat_released", fromState, "recovery_required")),
|
|
1440
|
+
transitionSignature("policy_drain_completed", "policy_soft_draining_to_1", "policy_steady")
|
|
1441
|
+
]);
|
|
1442
|
+
function transitionSignature(kind, fromState, toState) {
|
|
1443
|
+
return `${kind}\0${fromState}\0${toState}`;
|
|
1444
|
+
}
|
|
1445
|
+
function isAllowedSanitizedEventTransition(kind, fromState, toState) {
|
|
1446
|
+
return SANITIZED_EVENT_TRANSITIONS.has(transitionSignature(kind, fromState, toState));
|
|
1447
|
+
}
|
|
1448
|
+
function requestTransitionKind(fromState, toState) {
|
|
1449
|
+
if (fromState === "admitted" && toState === "starting")
|
|
1450
|
+
return "request_starting";
|
|
1451
|
+
if (fromState === "starting" && toState === "dispatch_intent")
|
|
1452
|
+
return "request_dispatch_intent";
|
|
1453
|
+
if (fromState === "dispatch_intent" && toState === "active")
|
|
1454
|
+
return "request_active";
|
|
1455
|
+
if (fromState === "dispatch_intent" && toState === "dispatch_ambiguous") {
|
|
1456
|
+
return "request_dispatch_ambiguous";
|
|
1457
|
+
}
|
|
1458
|
+
throw new Error("request transition is not journalled by the generic lease transition path");
|
|
1459
|
+
}
|
|
1460
|
+
function normalizeSanitizedEventPageRequest(input) {
|
|
1461
|
+
const record = dataRecord(input, ["afterEventSeq", "limit"]);
|
|
1462
|
+
if (record === null)
|
|
1463
|
+
throw new Error("sanitized event page request must have the exact supported shape");
|
|
1464
|
+
if (typeof record.afterEventSeq !== "number" || !Number.isSafeInteger(record.afterEventSeq) || record.afterEventSeq < 0) {
|
|
1465
|
+
throw new Error("sanitized event cursor must be a non-negative safe integer");
|
|
1466
|
+
}
|
|
1467
|
+
if (typeof record.limit !== "number" || !Number.isSafeInteger(record.limit) || record.limit < 1 || record.limit > 1_000) {
|
|
1468
|
+
throw new Error("sanitized event page limit must be between 1 and 1000");
|
|
1469
|
+
}
|
|
1470
|
+
return Object.freeze({ afterEventSeq: record.afterEventSeq, limit: record.limit });
|
|
1471
|
+
}
|
|
1472
|
+
function normalizeSanitizedEventRow(row) {
|
|
1473
|
+
if (typeof row.event_seq !== "number" || !Number.isSafeInteger(row.event_seq) || row.event_seq < 1) {
|
|
1474
|
+
throw new Error("sanitized event journal contains an invalid sequence");
|
|
1475
|
+
}
|
|
1476
|
+
if (!isSanitizedEventKind(row.kind) || !isSanitizedEventState(row.from_state) || !isSanitizedEventState(row.to_state)) {
|
|
1477
|
+
throw new Error("sanitized event journal contains a non-allowlisted transition");
|
|
1478
|
+
}
|
|
1479
|
+
validateTimestamp(row.occurred_at, "sanitized event timestamp");
|
|
1480
|
+
if (typeof row.correlation_hmac !== "string" || !/^[0-9a-f]{64}$/.test(row.correlation_hmac)) {
|
|
1481
|
+
throw new Error("sanitized event journal contains an invalid correlation HMAC");
|
|
1482
|
+
}
|
|
1483
|
+
if (!isAllowedSanitizedEventTransition(row.kind, row.from_state, row.to_state)) {
|
|
1484
|
+
throw new Error("sanitized event journal contains a non-allowlisted transition");
|
|
1485
|
+
}
|
|
1486
|
+
return Object.freeze({
|
|
1487
|
+
eventSeq: row.event_seq,
|
|
1488
|
+
kind: row.kind,
|
|
1489
|
+
fromState: row.from_state,
|
|
1490
|
+
toState: row.to_state,
|
|
1491
|
+
occurredAt: row.occurred_at,
|
|
1492
|
+
correlationHmac: row.correlation_hmac
|
|
1493
|
+
});
|
|
1494
|
+
}
|
|
1495
|
+
function isSanitizedEventKind(value) {
|
|
1496
|
+
return (value === "request_enqueued" ||
|
|
1497
|
+
value === "request_cancelled" ||
|
|
1498
|
+
value === "request_abandoned" ||
|
|
1499
|
+
value === "request_queue_timed_out" ||
|
|
1500
|
+
value === "request_admitted" ||
|
|
1501
|
+
value === "request_starting" ||
|
|
1502
|
+
value === "request_dispatch_intent" ||
|
|
1503
|
+
value === "request_active" ||
|
|
1504
|
+
value === "request_dispatch_ambiguous" ||
|
|
1505
|
+
value === "request_provider_terminal" ||
|
|
1506
|
+
value === "request_released" ||
|
|
1507
|
+
value === "request_recovery_required" ||
|
|
1508
|
+
value === "request_recovery_seat_released" ||
|
|
1509
|
+
value === "queued_owner_dead" ||
|
|
1510
|
+
value === "policy_drain_completed");
|
|
1511
|
+
}
|
|
1512
|
+
function isSanitizedEventState(value) {
|
|
1513
|
+
if (value === "absent")
|
|
1514
|
+
return true;
|
|
1515
|
+
if (value === "policy_steady" || value === "policy_soft_draining_to_1")
|
|
1516
|
+
return true;
|
|
1517
|
+
try {
|
|
1518
|
+
normalizeRequestState(value);
|
|
1519
|
+
return true;
|
|
1520
|
+
}
|
|
1521
|
+
catch {
|
|
1522
|
+
return false;
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
function validatePolicy(policy) {
|
|
1526
|
+
for (const [name, value] of Object.entries(policy)) {
|
|
1527
|
+
if (!Number.isSafeInteger(value) || value < 0 || (name.startsWith("max") && value < 1)) {
|
|
1528
|
+
throw new Error(`invalid admission policy ${name}`);
|
|
1529
|
+
}
|
|
1530
|
+
}
|
|
1531
|
+
if (!isAllowedAdmissionActiveTurns(policy.maxActiveTurns)) {
|
|
1532
|
+
throw new Error("invalid admission policy maxActiveTurns");
|
|
1533
|
+
}
|
|
1534
|
+
if (!isAllowedAdmissionConcurrentStarts(policy.maxConcurrentStarts)) {
|
|
1535
|
+
throw new Error("invalid admission policy maxConcurrentStarts");
|
|
1536
|
+
}
|
|
1537
|
+
return Object.freeze({ ...policy });
|
|
1538
|
+
}
|
|
1539
|
+
function normalizedPolicyTuple(policy) {
|
|
1540
|
+
return Object.freeze([
|
|
1541
|
+
policy.maxActiveTurns,
|
|
1542
|
+
policy.maxConcurrentStarts,
|
|
1543
|
+
policy.minStartIntervalMs,
|
|
1544
|
+
policy.queueTimeoutMs,
|
|
1545
|
+
policy.capacityCooldownMs
|
|
1546
|
+
]);
|
|
1547
|
+
}
|
|
1548
|
+
function policyStateMatches(row, policy, policyFingerprint) {
|
|
1549
|
+
return (row.max_active_turns === policy.maxActiveTurns &&
|
|
1550
|
+
row.max_concurrent_starts === policy.maxConcurrentStarts &&
|
|
1551
|
+
row.min_start_interval_ms === policy.minStartIntervalMs &&
|
|
1552
|
+
row.queue_timeout_ms === policy.queueTimeoutMs &&
|
|
1553
|
+
row.capacity_cooldown_ms === policy.capacityCooldownMs &&
|
|
1554
|
+
row.policy_fingerprint === policyFingerprint);
|
|
1555
|
+
}
|
|
1556
|
+
function policyFromState(row, maxActiveTurns) {
|
|
1557
|
+
return validatePolicy({
|
|
1558
|
+
maxActiveTurns,
|
|
1559
|
+
maxConcurrentStarts: policyStateInteger(row.max_concurrent_starts, "durable policy max_concurrent_starts"),
|
|
1560
|
+
minStartIntervalMs: policyStateInteger(row.min_start_interval_ms, "durable policy min_start_interval_ms"),
|
|
1561
|
+
queueTimeoutMs: policyStateInteger(row.queue_timeout_ms, "durable policy queue_timeout_ms"),
|
|
1562
|
+
capacityCooldownMs: policyStateInteger(row.capacity_cooldown_ms, "durable policy capacity_cooldown_ms")
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
function policyStateInteger(value, label) {
|
|
1566
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) {
|
|
1567
|
+
throw new AdmissionRuntimeError(`${label} is invalid`);
|
|
1568
|
+
}
|
|
1569
|
+
return value;
|
|
1570
|
+
}
|
|
1571
|
+
function policyDrainState(row) {
|
|
1572
|
+
if (row.drain_state !== "steady" && row.drain_state !== "soft_draining_to_1") {
|
|
1573
|
+
throw new AdmissionRuntimeError("durable policy drain state is invalid");
|
|
1574
|
+
}
|
|
1575
|
+
return row.drain_state;
|
|
1576
|
+
}
|
|
1577
|
+
function validateEnqueueRequest(input) {
|
|
1578
|
+
for (const field of ["requestId", "sessionId", "fingerprint", "provider", "model"]) {
|
|
1579
|
+
const value = input[field];
|
|
1580
|
+
if (typeof value !== "string" || value.trim().length === 0 || value.includes("\0")) {
|
|
1581
|
+
throw new Error(`invalid request metadata ${field}`);
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
validateIdentifier(requestAgentId(input), "request metadata agentId");
|
|
1585
|
+
if (!Number.isSafeInteger(input.now) || input.now < 0) {
|
|
1586
|
+
throw new Error("invalid request timestamp");
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
function requestAgentId(input) {
|
|
1590
|
+
const record = input;
|
|
1591
|
+
if (record.parentId !== undefined) {
|
|
1592
|
+
throw new Error("request metadata parentId is not accepted");
|
|
1593
|
+
}
|
|
1594
|
+
if (typeof record.agentId === "string") {
|
|
1595
|
+
return record.agentId;
|
|
1596
|
+
}
|
|
1597
|
+
throw new Error("invalid request metadata agentId");
|
|
1598
|
+
}
|
|
1599
|
+
function validateIdentifier(value, label) {
|
|
1600
|
+
if (typeof value !== "string" ||
|
|
1601
|
+
value.length === 0 ||
|
|
1602
|
+
value.length > 256 ||
|
|
1603
|
+
value.trim() !== value ||
|
|
1604
|
+
/[\u0000-\u001f\u007f]/.test(value)) {
|
|
1605
|
+
throw new Error(`${label} must be a non-empty single-line identifier`);
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
function validateTimestamp(value, label) {
|
|
1609
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) {
|
|
1610
|
+
throw new Error(`${label} must be a non-negative safe integer`);
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
function validateFaultInjection(value) {
|
|
1614
|
+
if (value === undefined)
|
|
1615
|
+
return undefined;
|
|
1616
|
+
try {
|
|
1617
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
1618
|
+
throw new Error("fault injection must be an object");
|
|
1619
|
+
}
|
|
1620
|
+
const callbacks = value;
|
|
1621
|
+
const callback = (value) => {
|
|
1622
|
+
if (value === undefined)
|
|
1623
|
+
return undefined;
|
|
1624
|
+
if (typeof value !== "function") {
|
|
1625
|
+
throw new Error("fault injection callback must be a function");
|
|
1626
|
+
}
|
|
1627
|
+
return value;
|
|
1628
|
+
};
|
|
1629
|
+
const afterProcessIdentityPersisted = callback(callbacks.afterProcessIdentityPersisted);
|
|
1630
|
+
return Object.freeze({
|
|
1631
|
+
afterProcessIdentityPersisted: afterProcessIdentityPersisted === undefined ? undefined : () => afterProcessIdentityPersisted()
|
|
1632
|
+
});
|
|
1633
|
+
}
|
|
1634
|
+
catch (error) {
|
|
1635
|
+
if (error instanceof Error)
|
|
1636
|
+
throw error;
|
|
1637
|
+
throw new Error("fault injection is invalid");
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
function compareSqliteVersions(left, right) {
|
|
1641
|
+
const leftParts = sqliteVersionParts(left);
|
|
1642
|
+
const rightParts = sqliteVersionParts(right);
|
|
1643
|
+
for (let index = 0; index < 3; index += 1) {
|
|
1644
|
+
const difference = leftParts[index] - rightParts[index];
|
|
1645
|
+
if (difference !== 0)
|
|
1646
|
+
return difference;
|
|
1647
|
+
}
|
|
1648
|
+
return 0;
|
|
1649
|
+
}
|
|
1650
|
+
function sqliteVersionParts(value) {
|
|
1651
|
+
const parts = value.split(".").map((part) => Number.parseInt(part, 10));
|
|
1652
|
+
if (parts.length < 2 || parts.some((part) => !Number.isSafeInteger(part) || part < 0)) {
|
|
1653
|
+
throw new AdmissionMigrationError(`SQLite version ${value} could not be parsed`);
|
|
1654
|
+
}
|
|
1655
|
+
return [parts[0], parts[1], parts[2] ?? 0];
|
|
1656
|
+
}
|
|
1657
|
+
function isSqliteTransactionContention(error) {
|
|
1658
|
+
if (!(error instanceof Error) || error.name !== "SqliteError")
|
|
1659
|
+
return false;
|
|
1660
|
+
const code = error.code;
|
|
1661
|
+
return (code === "SQLITE_BUSY" ||
|
|
1662
|
+
code === "SQLITE_BUSY_RECOVERY" ||
|
|
1663
|
+
code === "SQLITE_BUSY_SNAPSHOT" ||
|
|
1664
|
+
code === "SQLITE_BUSY_TIMEOUT" ||
|
|
1665
|
+
code === "SQLITE_LOCKED" ||
|
|
1666
|
+
code === "SQLITE_LOCKED_SHAREDCACHE");
|
|
1667
|
+
}
|
|
1668
|
+
function normalizeVerifiedLinuxProcessRecord(value) {
|
|
1669
|
+
try {
|
|
1670
|
+
const record = dataRecord(value, ["requestId", "leaseId", "generation", "ownerInstanceId", "processIdentity", "promptChannel"]);
|
|
1671
|
+
if (record === null)
|
|
1672
|
+
return null;
|
|
1673
|
+
const processIdentity = dataRecord(record.processIdentity, ["connector", "child"]);
|
|
1674
|
+
if (processIdentity === null)
|
|
1675
|
+
return null;
|
|
1676
|
+
const ownerInstanceId = normalizeOwnerInstanceId(record.ownerInstanceId);
|
|
1677
|
+
const connector = normalizeVerifiedLinuxConnectorIdentity(processIdentity.connector);
|
|
1678
|
+
const child = normalizeVerifiedLinuxProcessIdentity(processIdentity.child);
|
|
1679
|
+
if (connector.ownerInstanceId !== ownerInstanceId)
|
|
1680
|
+
return null;
|
|
1681
|
+
const promptChannel = record.promptChannel;
|
|
1682
|
+
if (promptChannel !== "stdin" && promptChannel !== "pty")
|
|
1683
|
+
return null;
|
|
1684
|
+
return Object.freeze({
|
|
1685
|
+
requestId: normalizeIdentifier(record.requestId, "process record request ID"),
|
|
1686
|
+
leaseId: normalizeIdentifier(record.leaseId, "process record lease ID"),
|
|
1687
|
+
generation: normalizePositiveSafeInteger(record.generation, "process record generation", Number.MAX_SAFE_INTEGER),
|
|
1688
|
+
ownerInstanceId,
|
|
1689
|
+
processIdentity: Object.freeze({ connector, child }),
|
|
1690
|
+
promptChannel
|
|
1691
|
+
});
|
|
1692
|
+
}
|
|
1693
|
+
catch {
|
|
1694
|
+
return null;
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
function captureControllerQueuedOwnerIdentity() {
|
|
1698
|
+
return normalizeVerifiedLinuxConnectorIdentity({
|
|
1699
|
+
ownerInstanceId: randomUUID(),
|
|
1700
|
+
createdAt: new Date().toISOString(),
|
|
1701
|
+
...captureLinuxProcessIdentity(process.pid, nativeLinuxProcessEvidenceReaders)
|
|
1702
|
+
});
|
|
1703
|
+
}
|
|
1704
|
+
function normalizeVerifiedLinuxConnectorIdentity(value) {
|
|
1705
|
+
const record = dataRecord(value, [
|
|
1706
|
+
"ownerInstanceId",
|
|
1707
|
+
"createdAt",
|
|
1708
|
+
"bootId",
|
|
1709
|
+
"pid",
|
|
1710
|
+
"startTimeTicks",
|
|
1711
|
+
"pidNamespaceInode",
|
|
1712
|
+
"ppid",
|
|
1713
|
+
"pgrp",
|
|
1714
|
+
"session"
|
|
1715
|
+
]);
|
|
1716
|
+
if (record === null)
|
|
1717
|
+
throw new Error("connector identity is invalid");
|
|
1718
|
+
return Object.freeze({
|
|
1719
|
+
ownerInstanceId: normalizeOwnerInstanceId(record.ownerInstanceId),
|
|
1720
|
+
createdAt: normalizeCanonicalUtcTimestamp(record.createdAt),
|
|
1721
|
+
...normalizeVerifiedLinuxProcessIdentityFields(record)
|
|
1722
|
+
});
|
|
1723
|
+
}
|
|
1724
|
+
function normalizeVerifiedLinuxProcessIdentity(value) {
|
|
1725
|
+
const record = dataRecord(value, ["bootId", "pid", "startTimeTicks", "pidNamespaceInode", "ppid", "pgrp", "session"]);
|
|
1726
|
+
if (record === null)
|
|
1727
|
+
throw new Error("process identity is invalid");
|
|
1728
|
+
return normalizeVerifiedLinuxProcessIdentityFields(record);
|
|
1729
|
+
}
|
|
1730
|
+
function normalizeVerifiedLinuxProcessIdentityFields(record) {
|
|
1731
|
+
return Object.freeze({
|
|
1732
|
+
bootId: normalizeBootId(record.bootId),
|
|
1733
|
+
pid: normalizePositiveSafeInteger(record.pid, "process PID", 2_147_483_647),
|
|
1734
|
+
startTimeTicks: normalizeStartTimeTicks(record.startTimeTicks),
|
|
1735
|
+
pidNamespaceInode: normalizePositiveSafeInteger(record.pidNamespaceInode, "PID namespace inode", 4_294_967_295),
|
|
1736
|
+
ppid: normalizePositiveSafeInteger(record.ppid, "process parent PID", 2_147_483_647),
|
|
1737
|
+
pgrp: normalizePositiveSafeInteger(record.pgrp, "process group", 2_147_483_647),
|
|
1738
|
+
session: normalizePositiveSafeInteger(record.session, "process session", 2_147_483_647)
|
|
1739
|
+
});
|
|
1740
|
+
}
|
|
1741
|
+
function sameLeaseProcessIdentity(row, record) {
|
|
1742
|
+
const { connector, child } = record.processIdentity;
|
|
1743
|
+
return (row.request_id === record.requestId &&
|
|
1744
|
+
row.lease_generation === record.generation &&
|
|
1745
|
+
row.owner_instance_id === record.ownerInstanceId &&
|
|
1746
|
+
row.prompt_channel === record.promptChannel &&
|
|
1747
|
+
row.connector_owner_instance_id === connector.ownerInstanceId &&
|
|
1748
|
+
row.connector_created_at === connector.createdAt &&
|
|
1749
|
+
row.connector_boot_id === connector.bootId &&
|
|
1750
|
+
row.connector_pid === connector.pid &&
|
|
1751
|
+
row.connector_start_time_ticks === connector.startTimeTicks &&
|
|
1752
|
+
row.connector_pid_namespace_inode === connector.pidNamespaceInode &&
|
|
1753
|
+
row.connector_ppid === connector.ppid &&
|
|
1754
|
+
row.connector_pgrp === connector.pgrp &&
|
|
1755
|
+
row.connector_session === connector.session &&
|
|
1756
|
+
row.child_boot_id === child.bootId &&
|
|
1757
|
+
row.child_pid === child.pid &&
|
|
1758
|
+
row.child_start_time_ticks === child.startTimeTicks &&
|
|
1759
|
+
row.child_pid_namespace_inode === child.pidNamespaceInode &&
|
|
1760
|
+
row.child_ppid === child.ppid &&
|
|
1761
|
+
row.child_pgrp === child.pgrp &&
|
|
1762
|
+
row.child_session === child.session);
|
|
1763
|
+
}
|
|
1764
|
+
function sameQueuedOwnerIdentity(row, owner) {
|
|
1765
|
+
return (row.owner_instance_id === owner.ownerInstanceId &&
|
|
1766
|
+
row.created_at === owner.createdAt &&
|
|
1767
|
+
row.boot_id === owner.bootId &&
|
|
1768
|
+
row.pid === owner.pid &&
|
|
1769
|
+
row.start_time_ticks === owner.startTimeTicks &&
|
|
1770
|
+
row.pid_namespace_inode === owner.pidNamespaceInode &&
|
|
1771
|
+
row.ppid === owner.ppid &&
|
|
1772
|
+
row.pgrp === owner.pgrp &&
|
|
1773
|
+
row.session === owner.session);
|
|
1774
|
+
}
|
|
1775
|
+
function dataRecord(value, expectedKeys) {
|
|
1776
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
1777
|
+
return null;
|
|
1778
|
+
try {
|
|
1779
|
+
const prototype = Object.getPrototypeOf(value);
|
|
1780
|
+
if (prototype !== Object.prototype && prototype !== null)
|
|
1781
|
+
return null;
|
|
1782
|
+
const record = value;
|
|
1783
|
+
const keys = Object.getOwnPropertyNames(record);
|
|
1784
|
+
if (Object.getOwnPropertySymbols(record).length !== 0 ||
|
|
1785
|
+
keys.length !== expectedKeys.length ||
|
|
1786
|
+
keys.some((key) => !expectedKeys.includes(key))) {
|
|
1787
|
+
return null;
|
|
1788
|
+
}
|
|
1789
|
+
for (const key of expectedKeys) {
|
|
1790
|
+
const descriptor = Object.getOwnPropertyDescriptor(record, key);
|
|
1791
|
+
if (descriptor === undefined || !descriptor.enumerable || !("value" in descriptor))
|
|
1792
|
+
return null;
|
|
1793
|
+
}
|
|
1794
|
+
return record;
|
|
1795
|
+
}
|
|
1796
|
+
catch {
|
|
1797
|
+
return null;
|
|
1798
|
+
}
|
|
1799
|
+
}
|
|
1800
|
+
function normalizeIdentifier(value, label) {
|
|
1801
|
+
validateIdentifier(value, label);
|
|
1802
|
+
return value;
|
|
1803
|
+
}
|
|
1804
|
+
function normalizeOwnerInstanceId(value) {
|
|
1805
|
+
if (typeof value !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value)) {
|
|
1806
|
+
throw new Error("process record owner instance ID must be a canonical UUID v4");
|
|
1807
|
+
}
|
|
1808
|
+
return value;
|
|
1809
|
+
}
|
|
1810
|
+
function normalizeCanonicalUtcTimestamp(value) {
|
|
1811
|
+
if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(value)) {
|
|
1812
|
+
throw new Error("connector creation timestamp must be canonical UTC");
|
|
1813
|
+
}
|
|
1814
|
+
const parsed = new Date(value);
|
|
1815
|
+
if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== value) {
|
|
1816
|
+
throw new Error("connector creation timestamp is invalid");
|
|
1817
|
+
}
|
|
1818
|
+
return value;
|
|
1819
|
+
}
|
|
1820
|
+
function normalizeBootId(value) {
|
|
1821
|
+
if (typeof value !== "string" ||
|
|
1822
|
+
!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value) ||
|
|
1823
|
+
/^0{8}-0{4}-0{4}-0{4}-0{12}$/.test(value)) {
|
|
1824
|
+
throw new Error("process boot ID is invalid");
|
|
1825
|
+
}
|
|
1826
|
+
return value;
|
|
1827
|
+
}
|
|
1828
|
+
function normalizeStartTimeTicks(value) {
|
|
1829
|
+
if (typeof value !== "string" || !/^[1-9][0-9]*$/.test(value) || BigInt(value) > 18446744073709551615n) {
|
|
1830
|
+
throw new Error("process start time is invalid");
|
|
1831
|
+
}
|
|
1832
|
+
return value;
|
|
1833
|
+
}
|
|
1834
|
+
function normalizePositiveSafeInteger(value, label, maximum) {
|
|
1835
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum) {
|
|
1836
|
+
throw new Error(`${label} must be a positive safe integer`);
|
|
1837
|
+
}
|
|
1838
|
+
return value;
|
|
1839
|
+
}
|
|
1840
|
+
function readRuntimeProcessIds(readers) {
|
|
1841
|
+
if (typeof readers !== "object" ||
|
|
1842
|
+
readers === null ||
|
|
1843
|
+
typeof readers.listProcessIds !== "function") {
|
|
1844
|
+
return null;
|
|
1845
|
+
}
|
|
1846
|
+
let value;
|
|
1847
|
+
try {
|
|
1848
|
+
value = readers.listProcessIds();
|
|
1849
|
+
}
|
|
1850
|
+
catch {
|
|
1851
|
+
return null;
|
|
1852
|
+
}
|
|
1853
|
+
if (!Array.isArray(value))
|
|
1854
|
+
return null;
|
|
1855
|
+
const ids = [];
|
|
1856
|
+
for (const pid of value) {
|
|
1857
|
+
if (!Number.isSafeInteger(pid) || pid < 1 || pid > MAX_RUNTIME_PID)
|
|
1858
|
+
return null;
|
|
1859
|
+
ids.push(pid);
|
|
1860
|
+
}
|
|
1861
|
+
return Object.freeze([...new Set(ids)].sort((left, right) => left - right));
|
|
1862
|
+
}
|
|
1863
|
+
function suspectReason(dispatch, processIds) {
|
|
1864
|
+
return dispatch.processIdentity === null || processIds === null ? "identity_unverifiable" : "heartbeat_expired";
|
|
1865
|
+
}
|
|
1866
|
+
function validateSuspectReason(reason) {
|
|
1867
|
+
if (reason !== "heartbeat_expired" && reason !== "identity_unverifiable") {
|
|
1868
|
+
throw new Error("lease suspect reason is invalid");
|
|
1869
|
+
}
|
|
1870
|
+
}
|
|
1871
|
+
function isHeartbeatStale(heartbeatAt, now) {
|
|
1872
|
+
return now - heartbeatAt > LEASE_HEARTBEAT_STALE_MS;
|
|
1873
|
+
}
|
|
1874
|
+
function isGoneIdentity(value) {
|
|
1875
|
+
return value === "gone" || value === "pid_reused";
|
|
1876
|
+
}
|
|
1877
|
+
function toRecoverableQueuedOwner(row) {
|
|
1878
|
+
try {
|
|
1879
|
+
return Object.freeze({
|
|
1880
|
+
requestId: normalizeIdentifier(row.request_id, "recoverable queued-owner request ID"),
|
|
1881
|
+
owner: normalizeQueuedOwnerIdentityRow(row)
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
catch {
|
|
1885
|
+
throw new RecoverableDispatchInventoryError();
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
function normalizeQueuedOwnerIdentityRow(row) {
|
|
1889
|
+
return normalizeVerifiedLinuxConnectorIdentity({
|
|
1890
|
+
ownerInstanceId: row.owner_instance_id,
|
|
1891
|
+
createdAt: row.created_at,
|
|
1892
|
+
bootId: row.boot_id,
|
|
1893
|
+
pid: row.pid,
|
|
1894
|
+
startTimeTicks: row.start_time_ticks,
|
|
1895
|
+
pidNamespaceInode: row.pid_namespace_inode,
|
|
1896
|
+
ppid: row.ppid,
|
|
1897
|
+
pgrp: row.pgrp,
|
|
1898
|
+
session: row.session
|
|
1899
|
+
});
|
|
1900
|
+
}
|
|
1901
|
+
function toRecoverableDispatch(row) {
|
|
1902
|
+
try {
|
|
1903
|
+
const requestId = normalizeIdentifier(row.request_id, "recoverable dispatch request ID");
|
|
1904
|
+
const leaseRequestId = normalizeIdentifier(row.lease_request_id, "recoverable dispatch lease request ID");
|
|
1905
|
+
const sessionId = normalizeIdentifier(row.request_session_id, "recoverable dispatch session ID");
|
|
1906
|
+
const provider = normalizeIdentifier(row.request_provider, "recoverable dispatch provider");
|
|
1907
|
+
const model = normalizeIdentifier(row.request_model, "recoverable dispatch model");
|
|
1908
|
+
const fence = Object.freeze({
|
|
1909
|
+
leaseId: normalizeIdentifier(row.lease_id, "recoverable dispatch lease ID"),
|
|
1910
|
+
generation: normalizePositiveSafeInteger(row.lease_generation, "recoverable dispatch lease generation", Number.MAX_SAFE_INTEGER),
|
|
1911
|
+
ownerInstanceId: normalizeIdentifier(row.lease_owner_instance_id, "recoverable dispatch lease owner")
|
|
1912
|
+
});
|
|
1913
|
+
const requestLeaseGeneration = normalizePositiveSafeInteger(row.request_lease_generation, "recoverable dispatch request lease generation", Number.MAX_SAFE_INTEGER);
|
|
1914
|
+
const phase = normalizeRequestState(row.lease_phase);
|
|
1915
|
+
const requestState = normalizeRequestState(row.request_state);
|
|
1916
|
+
validateTimestamp(row.lease_heartbeat_at, "recoverable dispatch heartbeat");
|
|
1917
|
+
validateTimestamp(row.request_enqueued_at, "recoverable dispatch enqueue timestamp");
|
|
1918
|
+
if (leaseRequestId !== requestId || requestLeaseGeneration !== fence.generation) {
|
|
1919
|
+
throw new Error("lease/request fence mismatch");
|
|
1920
|
+
}
|
|
1921
|
+
const processIdentity = toRecoverableDispatchProcessIdentity(row, requestId, fence);
|
|
1922
|
+
if (processIdentity !== null && (phase === "admitted" || phase === "starting")) {
|
|
1923
|
+
throw new Error("process identity predates dispatch intent");
|
|
1924
|
+
}
|
|
1925
|
+
if (!isRecoverableDispatchPhase(phase) || requestState !== phase) {
|
|
1926
|
+
throw new Error("nonterminal lease/request state mismatch");
|
|
1927
|
+
}
|
|
1928
|
+
return Object.freeze({
|
|
1929
|
+
requestId,
|
|
1930
|
+
sessionId,
|
|
1931
|
+
provider,
|
|
1932
|
+
model,
|
|
1933
|
+
fence,
|
|
1934
|
+
phase,
|
|
1935
|
+
heartbeatAt: row.lease_heartbeat_at,
|
|
1936
|
+
processIdentity
|
|
1937
|
+
});
|
|
1938
|
+
}
|
|
1939
|
+
catch {
|
|
1940
|
+
throw new RecoverableDispatchInventoryError();
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
function toRecoverableDispatchProcessIdentity(row, requestId, fence) {
|
|
1944
|
+
const values = [
|
|
1945
|
+
row.identity_lease_id,
|
|
1946
|
+
row.identity_request_id,
|
|
1947
|
+
row.identity_lease_generation,
|
|
1948
|
+
row.identity_owner_instance_id,
|
|
1949
|
+
row.identity_prompt_channel,
|
|
1950
|
+
row.identity_connector_owner_instance_id,
|
|
1951
|
+
row.identity_connector_created_at,
|
|
1952
|
+
row.identity_connector_boot_id,
|
|
1953
|
+
row.identity_connector_pid,
|
|
1954
|
+
row.identity_connector_start_time_ticks,
|
|
1955
|
+
row.identity_connector_pid_namespace_inode,
|
|
1956
|
+
row.identity_connector_ppid,
|
|
1957
|
+
row.identity_connector_pgrp,
|
|
1958
|
+
row.identity_connector_session,
|
|
1959
|
+
row.identity_child_boot_id,
|
|
1960
|
+
row.identity_child_pid,
|
|
1961
|
+
row.identity_child_start_time_ticks,
|
|
1962
|
+
row.identity_child_pid_namespace_inode,
|
|
1963
|
+
row.identity_child_ppid,
|
|
1964
|
+
row.identity_child_pgrp,
|
|
1965
|
+
row.identity_child_session
|
|
1966
|
+
];
|
|
1967
|
+
if (values.every((value) => value === null))
|
|
1968
|
+
return null;
|
|
1969
|
+
if (values.some((value) => value === null || value === undefined)) {
|
|
1970
|
+
throw new Error("partial process identity");
|
|
1971
|
+
}
|
|
1972
|
+
const identityLeaseId = normalizeIdentifier(row.identity_lease_id, "recoverable process identity lease ID");
|
|
1973
|
+
const identityRequestId = normalizeIdentifier(row.identity_request_id, "recoverable process identity request ID");
|
|
1974
|
+
const identityLeaseGeneration = normalizePositiveSafeInteger(row.identity_lease_generation, "recoverable process identity lease generation", Number.MAX_SAFE_INTEGER);
|
|
1975
|
+
const identityOwnerInstanceId = normalizeIdentifier(row.identity_owner_instance_id, "recoverable process identity owner");
|
|
1976
|
+
if (identityLeaseId !== fence.leaseId ||
|
|
1977
|
+
identityRequestId !== requestId ||
|
|
1978
|
+
identityLeaseGeneration !== fence.generation ||
|
|
1979
|
+
identityOwnerInstanceId !== fence.ownerInstanceId) {
|
|
1980
|
+
throw new Error("process identity fence mismatch");
|
|
1981
|
+
}
|
|
1982
|
+
if (row.identity_prompt_channel !== "stdin" && row.identity_prompt_channel !== "pty") {
|
|
1983
|
+
throw new Error("process identity prompt channel is invalid");
|
|
1984
|
+
}
|
|
1985
|
+
const connector = normalizeVerifiedLinuxConnectorIdentity({
|
|
1986
|
+
ownerInstanceId: row.identity_connector_owner_instance_id,
|
|
1987
|
+
createdAt: row.identity_connector_created_at,
|
|
1988
|
+
bootId: row.identity_connector_boot_id,
|
|
1989
|
+
pid: row.identity_connector_pid,
|
|
1990
|
+
startTimeTicks: row.identity_connector_start_time_ticks,
|
|
1991
|
+
pidNamespaceInode: row.identity_connector_pid_namespace_inode,
|
|
1992
|
+
ppid: row.identity_connector_ppid,
|
|
1993
|
+
pgrp: row.identity_connector_pgrp,
|
|
1994
|
+
session: row.identity_connector_session
|
|
1995
|
+
});
|
|
1996
|
+
if (connector.ownerInstanceId !== fence.ownerInstanceId) {
|
|
1997
|
+
throw new Error("connector process identity owner mismatch");
|
|
1998
|
+
}
|
|
1999
|
+
const child = normalizeVerifiedLinuxProcessIdentity({
|
|
2000
|
+
bootId: row.identity_child_boot_id,
|
|
2001
|
+
pid: row.identity_child_pid,
|
|
2002
|
+
startTimeTicks: row.identity_child_start_time_ticks,
|
|
2003
|
+
pidNamespaceInode: row.identity_child_pid_namespace_inode,
|
|
2004
|
+
ppid: row.identity_child_ppid,
|
|
2005
|
+
pgrp: row.identity_child_pgrp,
|
|
2006
|
+
session: row.identity_child_session
|
|
2007
|
+
});
|
|
2008
|
+
return Object.freeze({ promptChannel: row.identity_prompt_channel, connector, child });
|
|
2009
|
+
}
|
|
2010
|
+
function normalizeRequestState(value) {
|
|
2011
|
+
switch (value) {
|
|
2012
|
+
case "queued":
|
|
2013
|
+
case "admitted":
|
|
2014
|
+
case "starting":
|
|
2015
|
+
case "dispatch_intent":
|
|
2016
|
+
case "dispatch_ambiguous":
|
|
2017
|
+
case "active":
|
|
2018
|
+
case "provider_terminal":
|
|
2019
|
+
case "completed":
|
|
2020
|
+
case "failed":
|
|
2021
|
+
case "cancelled":
|
|
2022
|
+
case "queue_timeout":
|
|
2023
|
+
case "recovery_required":
|
|
2024
|
+
return value;
|
|
2025
|
+
default:
|
|
2026
|
+
throw new Error("request state is invalid");
|
|
2027
|
+
}
|
|
2028
|
+
}
|
|
2029
|
+
function isRecoverableDispatchPhase(value) {
|
|
2030
|
+
return (value === "admitted" ||
|
|
2031
|
+
value === "starting" ||
|
|
2032
|
+
value === "dispatch_intent" ||
|
|
2033
|
+
value === "dispatch_ambiguous" ||
|
|
2034
|
+
value === "active" ||
|
|
2035
|
+
value === "recovery_required");
|
|
2036
|
+
}
|
|
2037
|
+
function validateLiveTurnCompletion(value) {
|
|
2038
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
|
2039
|
+
throw new Error("live turn completion must be an object");
|
|
2040
|
+
}
|
|
2041
|
+
if (value.outcome !== "completed" && value.outcome !== "failed" && value.outcome !== "cancelled") {
|
|
2042
|
+
throw new Error("live turn completion outcome is invalid");
|
|
2043
|
+
}
|
|
2044
|
+
if (value.outcome !== "failed" && value.failure !== undefined) {
|
|
2045
|
+
throw new Error("live turn completion failure is inconsistent");
|
|
2046
|
+
}
|
|
2047
|
+
if (value.failure === undefined)
|
|
2048
|
+
return;
|
|
2049
|
+
const categories = new Set([
|
|
2050
|
+
"provider_capacity",
|
|
2051
|
+
"quota",
|
|
2052
|
+
"auth",
|
|
2053
|
+
"permission",
|
|
2054
|
+
"timeout",
|
|
2055
|
+
"transport",
|
|
2056
|
+
"unknown"
|
|
2057
|
+
]);
|
|
2058
|
+
if (!categories.has(value.failure.category))
|
|
2059
|
+
throw new Error("live turn failure category is invalid");
|
|
2060
|
+
if (value.failure.httpStatus !== undefined &&
|
|
2061
|
+
(!Number.isInteger(value.failure.httpStatus) || value.failure.httpStatus < 100 || value.failure.httpStatus > 599)) {
|
|
2062
|
+
throw new Error("live turn failure HTTP status is invalid");
|
|
2063
|
+
}
|
|
2064
|
+
for (const signal of [value.failure.code, value.failure.reason]) {
|
|
2065
|
+
if (signal !== undefined)
|
|
2066
|
+
validateIdentifier(signal, "live turn failure signal");
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
function validatePurposeKey(key, purpose) {
|
|
2070
|
+
if (key === undefined)
|
|
2071
|
+
return undefined;
|
|
2072
|
+
if (!Buffer.isBuffer(key) || key.length !== 32) {
|
|
2073
|
+
throw new Error(`admission ${purpose} key must be exactly 32 bytes`);
|
|
2074
|
+
}
|
|
2075
|
+
return Buffer.from(key);
|
|
2076
|
+
}
|
|
2077
|
+
function toStoredRequest(row) {
|
|
2078
|
+
return {
|
|
2079
|
+
requestId: row.request_id,
|
|
2080
|
+
sessionId: row.session_id,
|
|
2081
|
+
agentId: row.agent_id,
|
|
2082
|
+
fingerprint: row.fingerprint,
|
|
2083
|
+
provider: row.provider,
|
|
2084
|
+
model: row.model,
|
|
2085
|
+
state: row.state,
|
|
2086
|
+
enqueuedAt: row.enqueued_at
|
|
2087
|
+
};
|
|
2088
|
+
}
|
|
2089
|
+
//# sourceMappingURL=controller.js.map
|