polygram 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +5 -3
- package/config.example.json +23 -1
- package/lib/codex/binary.js +287 -0
- package/lib/codex/delivery-finalizer.js +97 -0
- package/lib/codex/diagnostics.js +483 -0
- package/lib/codex/event-normalizer.js +50 -0
- package/lib/codex/host-identity.js +103 -0
- package/lib/codex/model-catalog.js +359 -0
- package/lib/codex/retention-scheduler.js +59 -0
- package/lib/codex/runtime-controller.js +1292 -0
- package/lib/codex/runtime-profile.js +888 -0
- package/lib/codex/spawn-context.js +87 -0
- package/lib/db/auto-resume.js +77 -9
- package/lib/db/codex-reconciliation.js +490 -0
- package/lib/db/codex-retention.js +139 -0
- package/lib/db/sessions.js +102 -1
- package/lib/db.js +3418 -19
- package/lib/error/classify.js +140 -0
- package/lib/handlers/abort.js +143 -5
- package/lib/handlers/autosteer.js +256 -1
- package/lib/handlers/codex-reconciliation.js +230 -0
- package/lib/handlers/config-callback.js +324 -89
- package/lib/handlers/config-ui.js +401 -39
- package/lib/handlers/dispatcher.js +71 -8
- package/lib/handlers/edit-correction.js +11 -1
- package/lib/handlers/edit-redelivery.js +230 -4
- package/lib/handlers/gate-inbound.js +7 -0
- package/lib/handlers/replay-disposition.js +133 -11
- package/lib/handlers/slash-commands.js +443 -37
- package/lib/ipc/client.js +22 -12
- package/lib/ipc/runtime-directory.js +208 -0
- package/lib/ipc/server.js +18 -12
- package/lib/model-costs.js +4 -3
- package/lib/pm-interface.js +33 -0
- package/lib/prompt.js +15 -9
- package/lib/rewind/rewind.js +21 -1
- package/lib/runtime-config.js +605 -0
- package/lib/sdk/build-options.js +4 -1
- package/lib/sdk/callbacks.js +46 -3
- package/migrations/015-agent-runtime-sessions.sql +50 -0
- package/migrations/016-agent-turn-attempts.sql +211 -0
- package/migrations/017-config-runtime-changes.sql +46 -0
- package/ops/README.md +57 -0
- package/package.json +2 -2
- package/polygram.js +697 -48
- package/scripts/doctor.js +37 -8
|
@@ -0,0 +1,1292 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { createHash } = require('node:crypto');
|
|
4
|
+
|
|
5
|
+
const orchestraDefault = require('@shumkov/orchestra');
|
|
6
|
+
const { resolvePinnedCodexBinary } = require('./binary');
|
|
7
|
+
const { resolveCodexHostIdentity } = require('./host-identity');
|
|
8
|
+
const { CodexModelCatalog } = require('./model-catalog');
|
|
9
|
+
const {
|
|
10
|
+
createCodexRuntimeProfileBuilder,
|
|
11
|
+
} = require('./runtime-profile');
|
|
12
|
+
const {
|
|
13
|
+
createCodexRuntimeAvailability,
|
|
14
|
+
resolveCodexRuntimeRequest,
|
|
15
|
+
resolveRuntimeConfig,
|
|
16
|
+
} = require('../runtime-config');
|
|
17
|
+
|
|
18
|
+
const DEFAULT_CATALOG_MAX_AGE_MS = 15 * 60 * 1_000;
|
|
19
|
+
|
|
20
|
+
class CodexRuntimeControllerError extends Error {
|
|
21
|
+
constructor(message, code, action, options) {
|
|
22
|
+
super(message, options);
|
|
23
|
+
this.name = 'CodexRuntimeControllerError';
|
|
24
|
+
this.code = code;
|
|
25
|
+
this.action = action;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function controllerError(message, code, action, cause) {
|
|
30
|
+
return new CodexRuntimeControllerError(
|
|
31
|
+
message,
|
|
32
|
+
code,
|
|
33
|
+
action,
|
|
34
|
+
cause ? { cause } : undefined,
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function boundedString(value, label) {
|
|
39
|
+
if (
|
|
40
|
+
typeof value !== 'string'
|
|
41
|
+
|| value.length === 0
|
|
42
|
+
|| Buffer.byteLength(value) > 4096
|
|
43
|
+
) {
|
|
44
|
+
throw controllerError(
|
|
45
|
+
`Codex ${label} must be a bounded non-empty string.`,
|
|
46
|
+
'CODEX_DEPLOYMENT_CONFIG_INVALID',
|
|
47
|
+
'Configure the dedicated Codex deployment before selecting pm:codex.',
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
return value;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function canonical(value) {
|
|
54
|
+
if (Array.isArray(value)) return value.map(canonical);
|
|
55
|
+
if (value && typeof value === 'object') {
|
|
56
|
+
return Object.fromEntries(
|
|
57
|
+
Object.keys(value).sort().map((key) => [key, canonical(value[key])]),
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
return value;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function digest(value) {
|
|
64
|
+
return createHash('sha256')
|
|
65
|
+
.update(JSON.stringify(canonical(value)))
|
|
66
|
+
.digest('hex');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function sameProfileRequest(profile, request) {
|
|
70
|
+
return profile?.cwd === request.cwd;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function catalogSupports(catalog, request) {
|
|
74
|
+
if (!Array.isArray(catalog?.models)) return false;
|
|
75
|
+
const matches = catalog.models.filter(
|
|
76
|
+
(entry) => entry?.model === request.model,
|
|
77
|
+
);
|
|
78
|
+
return (
|
|
79
|
+
matches.length === 1
|
|
80
|
+
&& Array.isArray(matches[0].supportedReasoningEfforts)
|
|
81
|
+
&& matches[0].supportedReasoningEfforts.includes(request.effort)
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function catalogCapabilityFingerprint(catalog) {
|
|
86
|
+
return digest({
|
|
87
|
+
runtimeVersion: catalog?.runtimeVersion ?? null,
|
|
88
|
+
schemaVersion: catalog?.schemaVersion ?? null,
|
|
89
|
+
models: catalog?.models ?? null,
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function credentialStateFingerprint(attestation) {
|
|
94
|
+
if (
|
|
95
|
+
!attestation
|
|
96
|
+
|| typeof attestation.configFingerprint !== 'object'
|
|
97
|
+
|| attestation.configFingerprint === null
|
|
98
|
+
|| (
|
|
99
|
+
attestation.authFingerprint !== null
|
|
100
|
+
&& attestation.authFingerprint !== undefined
|
|
101
|
+
&& typeof attestation.authFingerprint !== 'object'
|
|
102
|
+
)
|
|
103
|
+
) {
|
|
104
|
+
throw controllerError(
|
|
105
|
+
'Codex credential-state attestation was incomplete.',
|
|
106
|
+
'CODEX_CREDENTIAL_STATE_INVALID',
|
|
107
|
+
'Re-attest the dedicated Codex credential home before preflight.',
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return digest({
|
|
111
|
+
authFingerprint: attestation.authFingerprint ?? null,
|
|
112
|
+
configFingerprint: attestation.configFingerprint,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function deliveryResultString(value, label) {
|
|
117
|
+
if (
|
|
118
|
+
typeof value !== 'string'
|
|
119
|
+
|| value.length === 0
|
|
120
|
+
|| value.length > 512
|
|
121
|
+
) {
|
|
122
|
+
throw controllerError(
|
|
123
|
+
`Codex delivery ${label} must be a bounded non-empty string.`,
|
|
124
|
+
'CODEX_DELIVERY_RESULT_INVALID',
|
|
125
|
+
'Discard the malformed result without settling durable delivery.',
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
return value;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function createCodexRuntimeController({
|
|
132
|
+
config,
|
|
133
|
+
db,
|
|
134
|
+
processEnv = process.env,
|
|
135
|
+
defaultDaemonSecretRoots = [],
|
|
136
|
+
resolveHostIdentity = resolveCodexHostIdentity,
|
|
137
|
+
resolveBinary = resolvePinnedCodexBinary,
|
|
138
|
+
runtimeProfileBuilder = createCodexRuntimeProfileBuilder(),
|
|
139
|
+
modelCatalog = new CodexModelCatalog(),
|
|
140
|
+
attestCodexHome = orchestraDefault.attestPinnedCodexHome,
|
|
141
|
+
createAvailability = createCodexRuntimeAvailability,
|
|
142
|
+
resolveRuntime = resolveRuntimeConfig,
|
|
143
|
+
resolveRequest = resolveCodexRuntimeRequest,
|
|
144
|
+
orchestra = orchestraDefault,
|
|
145
|
+
logger = console,
|
|
146
|
+
preflightAdmissionTimeoutMs = 120_000,
|
|
147
|
+
catalogMaxAgeMs = DEFAULT_CATALOG_MAX_AGE_MS,
|
|
148
|
+
now = Date.now,
|
|
149
|
+
} = {}) {
|
|
150
|
+
if (
|
|
151
|
+
!config
|
|
152
|
+
|| !db
|
|
153
|
+
|| typeof db.reconstructCodexRecovery !== 'function'
|
|
154
|
+
|| typeof db.getCodexLease !== 'function'
|
|
155
|
+
|| typeof db.getCodexAttempt !== 'function'
|
|
156
|
+
|| typeof db.claimCodexDispatchReservation !== 'function'
|
|
157
|
+
|| typeof db.finalizeCodexAcceptedSteer !== 'function'
|
|
158
|
+
|| typeof db.markCodexDispatchDisposition !== 'function'
|
|
159
|
+
|| typeof db.settleCodexQueuedDispatch !== 'function'
|
|
160
|
+
|| typeof db.settleCodexStoppedGeneration !== 'function'
|
|
161
|
+
|| typeof db.recordCodexDeliveryCheckpoint !== 'function'
|
|
162
|
+
|| typeof resolveHostIdentity !== 'function'
|
|
163
|
+
|| typeof resolveBinary !== 'function'
|
|
164
|
+
|| typeof runtimeProfileBuilder?.prepare !== 'function'
|
|
165
|
+
|| typeof modelCatalog?.preflight !== 'function'
|
|
166
|
+
|| typeof attestCodexHome !== 'function'
|
|
167
|
+
|| typeof createAvailability !== 'function'
|
|
168
|
+
|| typeof resolveRuntime !== 'function'
|
|
169
|
+
|| typeof resolveRequest !== 'function'
|
|
170
|
+
|| typeof orchestra?.CodexAppServerClient !== 'function'
|
|
171
|
+
|| !Array.isArray(defaultDaemonSecretRoots)
|
|
172
|
+
|| !Number.isSafeInteger(preflightAdmissionTimeoutMs)
|
|
173
|
+
|| preflightAdmissionTimeoutMs <= 0
|
|
174
|
+
|| !Number.isSafeInteger(catalogMaxAgeMs)
|
|
175
|
+
|| catalogMaxAgeMs <= 0
|
|
176
|
+
|| typeof now !== 'function'
|
|
177
|
+
) {
|
|
178
|
+
throw new TypeError('Codex runtime controller dependencies are incomplete');
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const receiptsBySession = new Map();
|
|
182
|
+
const processesByGeneration = new Map();
|
|
183
|
+
const currentProcessBySession = new Map();
|
|
184
|
+
const retiredStopCancellations = new Map();
|
|
185
|
+
const preparingByKey = new Map();
|
|
186
|
+
let preflightAdmissionTail = Promise.resolve();
|
|
187
|
+
let initialization = null;
|
|
188
|
+
let initializationError = null;
|
|
189
|
+
|
|
190
|
+
async function withPreflightAdmission(operation) {
|
|
191
|
+
let release;
|
|
192
|
+
const admitted = new Promise((resolve) => {
|
|
193
|
+
release = resolve;
|
|
194
|
+
});
|
|
195
|
+
const predecessor = preflightAdmissionTail.catch(() => {});
|
|
196
|
+
preflightAdmissionTail = predecessor.then(() => admitted);
|
|
197
|
+
|
|
198
|
+
let timeout;
|
|
199
|
+
try {
|
|
200
|
+
await Promise.race([
|
|
201
|
+
predecessor,
|
|
202
|
+
new Promise((resolve, reject) => {
|
|
203
|
+
timeout = setTimeout(() => {
|
|
204
|
+
reject(controllerError(
|
|
205
|
+
'Codex native preflight admission timed out.',
|
|
206
|
+
'CODEX_PREFLIGHT_ADMISSION_TIMEOUT',
|
|
207
|
+
'Wait for the active Codex preflight to finish before retrying.',
|
|
208
|
+
));
|
|
209
|
+
}, preflightAdmissionTimeoutMs);
|
|
210
|
+
timeout.unref?.();
|
|
211
|
+
}),
|
|
212
|
+
]);
|
|
213
|
+
} catch (error) {
|
|
214
|
+
release();
|
|
215
|
+
throw error;
|
|
216
|
+
} finally {
|
|
217
|
+
if (timeout) clearTimeout(timeout);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
return await operation();
|
|
222
|
+
} finally {
|
|
223
|
+
release();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
function rememberStopCancellation(binding) {
|
|
228
|
+
retiredStopCancellations.set(binding.generationId, binding);
|
|
229
|
+
while (retiredStopCancellations.size > 32) {
|
|
230
|
+
retiredStopCancellations.delete(
|
|
231
|
+
retiredStopCancellations.keys().next().value,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function invalidatePreparedSession(sessionKey, entry) {
|
|
237
|
+
receiptsBySession.delete(sessionKey);
|
|
238
|
+
if (
|
|
239
|
+
entry?.deploymentIdentity
|
|
240
|
+
&& typeof modelCatalog.invalidate === 'function'
|
|
241
|
+
) {
|
|
242
|
+
modelCatalog.invalidate({
|
|
243
|
+
deploymentIdentity: entry.deploymentIdentity,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function prepareRuntimeView(entry, request, chatId, threadId) {
|
|
249
|
+
const runtimeConfig = resolveRuntime({
|
|
250
|
+
config,
|
|
251
|
+
chatId,
|
|
252
|
+
threadId,
|
|
253
|
+
defaultPm: 'sdk',
|
|
254
|
+
codexAvailability: entry.availability,
|
|
255
|
+
logger,
|
|
256
|
+
});
|
|
257
|
+
if (
|
|
258
|
+
runtimeConfig.runtime !== 'codex'
|
|
259
|
+
|| runtimeConfig.spawnProfileId !== entry.receipt.spawnProfileId
|
|
260
|
+
|| runtimeConfig.cwd !== request.cwd
|
|
261
|
+
|| runtimeConfig.model !== request.model
|
|
262
|
+
|| runtimeConfig.effort !== request.effort
|
|
263
|
+
) {
|
|
264
|
+
throw controllerError(
|
|
265
|
+
'Codex runtime selection changed during preflight.',
|
|
266
|
+
'CODEX_PREFLIGHT_PROFILE_MISMATCH',
|
|
267
|
+
'Retry after configuration changes settle.',
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
return Object.freeze({
|
|
271
|
+
runtimeConfig,
|
|
272
|
+
availability: entry.availability,
|
|
273
|
+
catalog: entry.catalog,
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function deploymentConfig() {
|
|
278
|
+
const configured = config.codex ?? {};
|
|
279
|
+
const codexHome = boundedString(
|
|
280
|
+
configured.home ?? processEnv.POLYGRAM_CODEX_HOME,
|
|
281
|
+
'credential home',
|
|
282
|
+
);
|
|
283
|
+
const configuredSecretRoots = configured.daemonSecretRoots ?? [];
|
|
284
|
+
if (
|
|
285
|
+
!Array.isArray(configuredSecretRoots)
|
|
286
|
+
|| [...defaultDaemonSecretRoots, ...configuredSecretRoots].some((root) => (
|
|
287
|
+
typeof root !== 'string'
|
|
288
|
+
|| root.length === 0
|
|
289
|
+
))
|
|
290
|
+
) {
|
|
291
|
+
throw controllerError(
|
|
292
|
+
'Codex daemon-secret roots must be an array of paths.',
|
|
293
|
+
'CODEX_DEPLOYMENT_CONFIG_INVALID',
|
|
294
|
+
'Configure every daemon secret root before selecting pm:codex.',
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
return {
|
|
298
|
+
codexHome,
|
|
299
|
+
binaryPath: configured.binary ?? processEnv.POLYGRAM_CODEX_BIN,
|
|
300
|
+
daemonSecretRoots: [
|
|
301
|
+
...new Set([...defaultDaemonSecretRoots, ...configuredSecretRoots]),
|
|
302
|
+
],
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function initialize() {
|
|
307
|
+
if (initialization) return initialization;
|
|
308
|
+
if (initializationError) throw initializationError;
|
|
309
|
+
try {
|
|
310
|
+
const identity = resolveHostIdentity();
|
|
311
|
+
const recovery = db.reconstructCodexRecovery({
|
|
312
|
+
stable_host_id: identity.stableHostId,
|
|
313
|
+
boot_session_id: identity.bootSessionId,
|
|
314
|
+
now: Date.now(),
|
|
315
|
+
});
|
|
316
|
+
let codexRecoveryState;
|
|
317
|
+
if (recovery.status === 'clear') {
|
|
318
|
+
codexRecoveryState = Object.freeze({ status: 'clear' });
|
|
319
|
+
} else {
|
|
320
|
+
const lease = db.getCodexLease();
|
|
321
|
+
codexRecoveryState = Object.freeze({
|
|
322
|
+
status: 'quarantined',
|
|
323
|
+
hostIdentity: boundedString(
|
|
324
|
+
lease?.stable_host_id ?? identity.stableHostId,
|
|
325
|
+
'incident stable host identity',
|
|
326
|
+
),
|
|
327
|
+
bootSessionIdentity: boundedString(
|
|
328
|
+
lease?.boot_session_id ?? identity.bootSessionId,
|
|
329
|
+
'incident boot-session identity',
|
|
330
|
+
),
|
|
331
|
+
generationId: boundedString(
|
|
332
|
+
lease?.generation_id ?? 'persisted-codex-quarantine',
|
|
333
|
+
'incident generation identity',
|
|
334
|
+
),
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
initialization = Object.freeze({
|
|
338
|
+
identity: Object.freeze({ ...identity }),
|
|
339
|
+
recovery: Object.freeze({ ...recovery }),
|
|
340
|
+
managerOptions: Object.freeze({
|
|
341
|
+
codexHostIdentity: identity.stableHostId,
|
|
342
|
+
codexBootSessionIdentity: identity.bootSessionId,
|
|
343
|
+
codexRecoveryState,
|
|
344
|
+
codexRetirementVerifier: verifyCodexRetirement,
|
|
345
|
+
}),
|
|
346
|
+
});
|
|
347
|
+
return initialization;
|
|
348
|
+
} catch (cause) {
|
|
349
|
+
initializationError = cause instanceof CodexRuntimeControllerError
|
|
350
|
+
? cause
|
|
351
|
+
: controllerError(
|
|
352
|
+
'Codex boot ownership reconstruction failed closed.',
|
|
353
|
+
'CODEX_BOOT_RECONSTRUCTION_FAILED',
|
|
354
|
+
'Inspect the durable Codex lease and host/boot identity before enabling Codex.',
|
|
355
|
+
cause,
|
|
356
|
+
);
|
|
357
|
+
throw initializationError;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function requireAvailableInitialization() {
|
|
362
|
+
const state = initialize();
|
|
363
|
+
if (state.recovery.status !== 'clear') {
|
|
364
|
+
throw controllerError(
|
|
365
|
+
`Codex remains quarantined: ${state.recovery.reason ?? 'unknown reason'}.`,
|
|
366
|
+
'CODEX_CONTAINMENT_QUARANTINED',
|
|
367
|
+
'Reconcile the incident; containment releases only after a verified same-host reboot.',
|
|
368
|
+
);
|
|
369
|
+
}
|
|
370
|
+
return state;
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
async function prepareSession({ sessionKey, chatId, threadId = null } = {}) {
|
|
374
|
+
boundedString(sessionKey, 'session key');
|
|
375
|
+
const state = requireAvailableInitialization();
|
|
376
|
+
const request = resolveRequest({
|
|
377
|
+
config,
|
|
378
|
+
chatId,
|
|
379
|
+
threadId,
|
|
380
|
+
defaultPm: 'sdk',
|
|
381
|
+
});
|
|
382
|
+
if (!request) return null;
|
|
383
|
+
|
|
384
|
+
const existing = receiptsBySession.get(sessionKey);
|
|
385
|
+
if (existing && sameProfileRequest(
|
|
386
|
+
existing.receipt.expectedStaticProfile,
|
|
387
|
+
request,
|
|
388
|
+
)) {
|
|
389
|
+
const catalogAgeMs = now() - existing.catalogPreparedAt;
|
|
390
|
+
let currentCredentialState;
|
|
391
|
+
try {
|
|
392
|
+
const homeAttestation = await attestCodexHome(
|
|
393
|
+
existing.receipt.expectedStaticProfile.codexHome,
|
|
394
|
+
existing.receipt.expectedStaticProfile.ownedConfigSha256,
|
|
395
|
+
);
|
|
396
|
+
currentCredentialState = credentialStateFingerprint(homeAttestation);
|
|
397
|
+
} catch (error) {
|
|
398
|
+
throw error;
|
|
399
|
+
}
|
|
400
|
+
if (
|
|
401
|
+
currentCredentialState === existing.credentialStateFingerprint
|
|
402
|
+
&& catalogAgeMs >= 0
|
|
403
|
+
&& catalogAgeMs < catalogMaxAgeMs
|
|
404
|
+
&& catalogSupports(existing.catalog, request)
|
|
405
|
+
) {
|
|
406
|
+
return prepareRuntimeView(existing, request, chatId, threadId);
|
|
407
|
+
}
|
|
408
|
+
invalidatePreparedSession(sessionKey, existing);
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const configured = deploymentConfig();
|
|
412
|
+
const key = digest({
|
|
413
|
+
sessionKey,
|
|
414
|
+
request,
|
|
415
|
+
codexHome: configured.codexHome,
|
|
416
|
+
binaryPath: configured.binaryPath,
|
|
417
|
+
daemonSecretRoots: configured.daemonSecretRoots,
|
|
418
|
+
});
|
|
419
|
+
if (preparingByKey.has(key)) return preparingByKey.get(key);
|
|
420
|
+
|
|
421
|
+
const promise = (async () => {
|
|
422
|
+
const binaryReceipt = await resolveBinary({
|
|
423
|
+
binaryPath: configured.binaryPath,
|
|
424
|
+
env: processEnv,
|
|
425
|
+
});
|
|
426
|
+
const expectedStaticProfile = await runtimeProfileBuilder.prepare({
|
|
427
|
+
binaryReceipt,
|
|
428
|
+
codexHome: configured.codexHome,
|
|
429
|
+
workspace: request.cwd,
|
|
430
|
+
daemonSecretRoots: configured.daemonSecretRoots,
|
|
431
|
+
model: request.model,
|
|
432
|
+
effort: request.effort,
|
|
433
|
+
processEnv,
|
|
434
|
+
});
|
|
435
|
+
const homeAttestation = await attestCodexHome(
|
|
436
|
+
expectedStaticProfile.codexHome,
|
|
437
|
+
expectedStaticProfile.ownedConfigSha256,
|
|
438
|
+
);
|
|
439
|
+
const attestedCredentialState = credentialStateFingerprint(
|
|
440
|
+
homeAttestation,
|
|
441
|
+
);
|
|
442
|
+
const deploymentIdentity = `codex-deployment:v1:${digest({
|
|
443
|
+
stableHostId: state.identity.stableHostId,
|
|
444
|
+
codexHome: expectedStaticProfile.codexHome,
|
|
445
|
+
authFingerprint: homeAttestation.authFingerprint ?? null,
|
|
446
|
+
configFingerprint: homeAttestation.configFingerprint,
|
|
447
|
+
})}`;
|
|
448
|
+
const catalog = await withPreflightAdmission(() => (
|
|
449
|
+
modelCatalog.preflight(expectedStaticProfile, {
|
|
450
|
+
deploymentIdentity,
|
|
451
|
+
})
|
|
452
|
+
));
|
|
453
|
+
if (
|
|
454
|
+
catalog?.selected?.model !== request.model
|
|
455
|
+
|| catalog?.selected?.effort !== request.effort
|
|
456
|
+
) {
|
|
457
|
+
throw controllerError(
|
|
458
|
+
'Authenticated Codex catalog did not confirm the selected model and effort.',
|
|
459
|
+
'CODEX_MODEL_SELECTION_MISMATCH',
|
|
460
|
+
'Choose a model and effort returned by the authenticated Codex catalog.',
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
const receipt = catalog.spawnProfile;
|
|
464
|
+
const entry = {
|
|
465
|
+
receipt,
|
|
466
|
+
availability: createAvailability(receipt),
|
|
467
|
+
catalog,
|
|
468
|
+
catalogCapabilityFingerprint:
|
|
469
|
+
catalogCapabilityFingerprint(catalog),
|
|
470
|
+
catalogPreparedAt: now(),
|
|
471
|
+
credentialStateFingerprint: attestedCredentialState,
|
|
472
|
+
deploymentIdentity,
|
|
473
|
+
};
|
|
474
|
+
receiptsBySession.set(sessionKey, entry);
|
|
475
|
+
return prepareRuntimeView(entry, request, chatId, threadId);
|
|
476
|
+
})();
|
|
477
|
+
preparingByKey.set(key, promise);
|
|
478
|
+
try {
|
|
479
|
+
return await promise;
|
|
480
|
+
} finally {
|
|
481
|
+
if (preparingByKey.get(key) === promise) preparingByKey.delete(key);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
async function resolveRuntimeView({
|
|
486
|
+
sessionKey,
|
|
487
|
+
chatId,
|
|
488
|
+
threadId = null,
|
|
489
|
+
} = {}) {
|
|
490
|
+
const prepared = await prepareSession({ sessionKey, chatId, threadId });
|
|
491
|
+
if (!prepared) return Object.freeze({ runtime: 'claude' });
|
|
492
|
+
const { runtimeConfig, catalog } = prepared;
|
|
493
|
+
const models = Array.isArray(catalog.models)
|
|
494
|
+
? catalog.models.map((entry) => Object.freeze({
|
|
495
|
+
model: entry.model,
|
|
496
|
+
displayName: entry.displayName,
|
|
497
|
+
defaultReasoningEffort: entry.defaultReasoningEffort,
|
|
498
|
+
supportedReasoningEfforts: Object.freeze([
|
|
499
|
+
...entry.supportedReasoningEfforts,
|
|
500
|
+
]),
|
|
501
|
+
}))
|
|
502
|
+
: [];
|
|
503
|
+
return Object.freeze({
|
|
504
|
+
runtime: 'codex',
|
|
505
|
+
model: runtimeConfig.model,
|
|
506
|
+
effort: runtimeConfig.effort,
|
|
507
|
+
models: Object.freeze(models),
|
|
508
|
+
efforts: Object.freeze(
|
|
509
|
+
models.find(({ model }) => model === runtimeConfig.model)
|
|
510
|
+
?.supportedReasoningEfforts ?? [],
|
|
511
|
+
),
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
function resolveReceipt(sessionKey, identity) {
|
|
516
|
+
const entry = receiptsBySession.get(sessionKey);
|
|
517
|
+
if (
|
|
518
|
+
identity?.runtime !== 'codex'
|
|
519
|
+
|| !entry
|
|
520
|
+
|| identity.spawnProfileId !== entry.receipt.spawnProfileId
|
|
521
|
+
) {
|
|
522
|
+
throw controllerError(
|
|
523
|
+
'No exact preflight receipt exists for this Codex spawn.',
|
|
524
|
+
'CODEX_PREFLIGHT_RECEIPT_MISSING',
|
|
525
|
+
'Run preflight for the exact session runtime profile before spawning.',
|
|
526
|
+
);
|
|
527
|
+
}
|
|
528
|
+
return entry.receipt;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function clientFactory({
|
|
532
|
+
sessionKey,
|
|
533
|
+
expectedStaticProfile,
|
|
534
|
+
onNotification,
|
|
535
|
+
onFault,
|
|
536
|
+
} = {}) {
|
|
537
|
+
const entry = receiptsBySession.get(sessionKey);
|
|
538
|
+
if (!entry || entry.receipt.expectedStaticProfile !== expectedStaticProfile) {
|
|
539
|
+
throw controllerError(
|
|
540
|
+
'Codex client construction did not match the preflighted profile.',
|
|
541
|
+
'CODEX_PREFLIGHT_PROFILE_MISMATCH',
|
|
542
|
+
'Discard the stale spawn and rerun exact preflight.',
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
return new orchestra.CodexAppServerClient({
|
|
546
|
+
binary: expectedStaticProfile.binary,
|
|
547
|
+
codexHome: expectedStaticProfile.codexHome,
|
|
548
|
+
cwd: expectedStaticProfile.cwd,
|
|
549
|
+
env: expectedStaticProfile.env,
|
|
550
|
+
expectedConfigSha256: expectedStaticProfile.ownedConfigSha256,
|
|
551
|
+
onNotification,
|
|
552
|
+
onFault,
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
function processReplacementReason(sessionKey, proc) {
|
|
557
|
+
const entry = receiptsBySession.get(sessionKey);
|
|
558
|
+
const record = typeof proc?.generationId === 'string'
|
|
559
|
+
? processesByGeneration.get(proc.generationId)
|
|
560
|
+
: null;
|
|
561
|
+
if (
|
|
562
|
+
!entry
|
|
563
|
+
|| !record
|
|
564
|
+
|| currentProcessBySession.get(sessionKey) !== record
|
|
565
|
+
|| record.proc !== proc
|
|
566
|
+
|| proc.sessionKey !== sessionKey
|
|
567
|
+
|| typeof entry.credentialStateFingerprint !== 'string'
|
|
568
|
+
|| typeof record.credentialStateFingerprint !== 'string'
|
|
569
|
+
|| typeof entry.catalogCapabilityFingerprint !== 'string'
|
|
570
|
+
|| typeof record.catalogCapabilityFingerprint !== 'string'
|
|
571
|
+
) {
|
|
572
|
+
throw controllerError(
|
|
573
|
+
'No exact runtime-state binding exists for this Codex process.',
|
|
574
|
+
'CODEX_PROCESS_CREDENTIAL_BINDING_MISSING',
|
|
575
|
+
'Do not reuse the process; prepare and register an exact replacement.',
|
|
576
|
+
);
|
|
577
|
+
}
|
|
578
|
+
if (
|
|
579
|
+
entry.credentialStateFingerprint
|
|
580
|
+
!== record.credentialStateFingerprint
|
|
581
|
+
) {
|
|
582
|
+
return 'credential-state-drift';
|
|
583
|
+
}
|
|
584
|
+
if (
|
|
585
|
+
entry.catalogCapabilityFingerprint
|
|
586
|
+
!== record.catalogCapabilityFingerprint
|
|
587
|
+
) {
|
|
588
|
+
return 'model-catalog-drift';
|
|
589
|
+
}
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
function requiresProcessReplacement(sessionKey, proc) {
|
|
594
|
+
return processReplacementReason(sessionKey, proc) !== null;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function registerProcess(proc) {
|
|
598
|
+
if (
|
|
599
|
+
proc?.runtime !== 'codex'
|
|
600
|
+
|| typeof proc.generationId !== 'string'
|
|
601
|
+
|| typeof proc.sessionKey !== 'string'
|
|
602
|
+
) {
|
|
603
|
+
throw new TypeError('Codex runtime controller requires a Codex process');
|
|
604
|
+
}
|
|
605
|
+
const receipt = resolveReceipt(proc.sessionKey, {
|
|
606
|
+
runtime: 'codex',
|
|
607
|
+
spawnProfileId: proc.spawnProfileId,
|
|
608
|
+
});
|
|
609
|
+
if (processesByGeneration.has(proc.generationId)) {
|
|
610
|
+
throw controllerError(
|
|
611
|
+
'Codex generation ID was reused.',
|
|
612
|
+
'CODEX_GENERATION_ID_REUSED',
|
|
613
|
+
'Discard the process and create a fresh generation.',
|
|
614
|
+
);
|
|
615
|
+
}
|
|
616
|
+
const record = {
|
|
617
|
+
proc,
|
|
618
|
+
receipt,
|
|
619
|
+
durable: false,
|
|
620
|
+
durabilityBlocked: false,
|
|
621
|
+
durabilityError: null,
|
|
622
|
+
retirementVerificationFailed: false,
|
|
623
|
+
closed: false,
|
|
624
|
+
healthyStopObserved: false,
|
|
625
|
+
retirementRequested: false,
|
|
626
|
+
retirementWaiters: [],
|
|
627
|
+
retired: false,
|
|
628
|
+
credentialStateFingerprint: receiptsBySession
|
|
629
|
+
.get(proc.sessionKey)
|
|
630
|
+
.credentialStateFingerprint,
|
|
631
|
+
catalogCapabilityFingerprint: receiptsBySession
|
|
632
|
+
.get(proc.sessionKey)
|
|
633
|
+
.catalogCapabilityFingerprint,
|
|
634
|
+
};
|
|
635
|
+
processesByGeneration.set(proc.generationId, record);
|
|
636
|
+
currentProcessBySession.set(proc.sessionKey, record);
|
|
637
|
+
proc.once?.('close', () => {
|
|
638
|
+
record.closed = true;
|
|
639
|
+
try {
|
|
640
|
+
if (!record.durable && proc.startupReleaseSafe === true) {
|
|
641
|
+
record.retired = true;
|
|
642
|
+
processesByGeneration.delete(proc.generationId);
|
|
643
|
+
forgetRetiredSession(record);
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
maybeRetire(record);
|
|
647
|
+
} catch (error) {
|
|
648
|
+
logger.error?.(
|
|
649
|
+
`[codex] durable retirement failed for ${proc.sessionKey}: ${error.message}`,
|
|
650
|
+
);
|
|
651
|
+
}
|
|
652
|
+
});
|
|
653
|
+
return proc;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
function forgetRetiredSession(record) {
|
|
657
|
+
if (currentProcessBySession.get(record.proc.sessionKey) !== record) {
|
|
658
|
+
return;
|
|
659
|
+
}
|
|
660
|
+
currentProcessBySession.delete(record.proc.sessionKey);
|
|
661
|
+
const entry = receiptsBySession.get(record.proc.sessionKey);
|
|
662
|
+
if (entry?.receipt === record.receipt) {
|
|
663
|
+
receiptsBySession.delete(record.proc.sessionKey);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
function requireCurrentDispatchOwner(sessionKey, generationId) {
|
|
668
|
+
const exactSessionKey = deliveryResultString(sessionKey, 'session key');
|
|
669
|
+
const exactGenerationId = deliveryResultString(
|
|
670
|
+
generationId,
|
|
671
|
+
'generation ID',
|
|
672
|
+
);
|
|
673
|
+
const record = processesByGeneration.get(exactGenerationId);
|
|
674
|
+
if (
|
|
675
|
+
record
|
|
676
|
+
&& currentProcessBySession.get(exactSessionKey) === record
|
|
677
|
+
&& record.proc.sessionKey === exactSessionKey
|
|
678
|
+
&& record.proc.generationId === exactGenerationId
|
|
679
|
+
&& record.durabilityBlocked
|
|
680
|
+
) {
|
|
681
|
+
throw controllerError(
|
|
682
|
+
'Codex generation is blocked by a delivery durability failure.',
|
|
683
|
+
'CODEX_DURABILITY_FAILED',
|
|
684
|
+
'Keep the exact generation fenced and recover from its durable lease.',
|
|
685
|
+
record.durabilityError,
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
if (
|
|
689
|
+
!record
|
|
690
|
+
|| currentProcessBySession.get(exactSessionKey) !== record
|
|
691
|
+
|| record.proc.sessionKey !== exactSessionKey
|
|
692
|
+
|| record.proc.generationId !== exactGenerationId
|
|
693
|
+
|| record.proc.closed
|
|
694
|
+
|| !record.durable
|
|
695
|
+
|| !(
|
|
696
|
+
record.proc.runtime === 'codex'
|
|
697
|
+
|| record.proc.backend === 'codex'
|
|
698
|
+
)
|
|
699
|
+
) {
|
|
700
|
+
throw controllerError(
|
|
701
|
+
'Codex dispatch does not belong to the current durable process.',
|
|
702
|
+
'CODEX_DISPATCH_BINDING_MISSING',
|
|
703
|
+
'Do not steer, queue, or mutate a reservation for a stale process.',
|
|
704
|
+
);
|
|
705
|
+
}
|
|
706
|
+
return {
|
|
707
|
+
record,
|
|
708
|
+
sessionKey: exactSessionKey,
|
|
709
|
+
generationId: exactGenerationId,
|
|
710
|
+
identity: requireAvailableInitialization().identity,
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
function dispatchReservationId({
|
|
715
|
+
botName,
|
|
716
|
+
telegramChatId,
|
|
717
|
+
telegramMessageId,
|
|
718
|
+
}) {
|
|
719
|
+
return `codex-dispatch:v1:${digest({
|
|
720
|
+
botName,
|
|
721
|
+
telegramChatId,
|
|
722
|
+
telegramMessageId,
|
|
723
|
+
})}`;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
function claimDispatchReservation({
|
|
727
|
+
sessionKey,
|
|
728
|
+
generationId,
|
|
729
|
+
botName,
|
|
730
|
+
telegramChatId,
|
|
731
|
+
telegramMessageId,
|
|
732
|
+
} = {}) {
|
|
733
|
+
const owner = requireCurrentDispatchOwner(sessionKey, generationId);
|
|
734
|
+
const exactBotName = deliveryResultString(botName, 'bot name');
|
|
735
|
+
const exactChatId = deliveryResultString(
|
|
736
|
+
telegramChatId,
|
|
737
|
+
'Telegram chat ID',
|
|
738
|
+
);
|
|
739
|
+
const exactMessageId = deliveryResultString(
|
|
740
|
+
telegramMessageId,
|
|
741
|
+
'Telegram message ID',
|
|
742
|
+
);
|
|
743
|
+
const reservationId = dispatchReservationId({
|
|
744
|
+
botName: exactBotName,
|
|
745
|
+
telegramChatId: exactChatId,
|
|
746
|
+
telegramMessageId: exactMessageId,
|
|
747
|
+
});
|
|
748
|
+
const result = db.claimCodexDispatchReservation({
|
|
749
|
+
reservation_id: reservationId,
|
|
750
|
+
generation_id: owner.generationId,
|
|
751
|
+
session_key: owner.sessionKey,
|
|
752
|
+
bot_name: exactBotName,
|
|
753
|
+
telegram_chat_id: exactChatId,
|
|
754
|
+
telegram_message_id: exactMessageId,
|
|
755
|
+
stable_host_id: owner.identity.stableHostId,
|
|
756
|
+
boot_session_id: owner.identity.bootSessionId,
|
|
757
|
+
ts: Date.now(),
|
|
758
|
+
});
|
|
759
|
+
return Object.freeze({
|
|
760
|
+
...result,
|
|
761
|
+
reservationId,
|
|
762
|
+
generationId: owner.generationId,
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
function finalizeAcceptedSteer({
|
|
767
|
+
sessionKey,
|
|
768
|
+
generationId,
|
|
769
|
+
reservationId,
|
|
770
|
+
steerAttemptId,
|
|
771
|
+
targetAttemptId,
|
|
772
|
+
} = {}) {
|
|
773
|
+
const owner = requireCurrentDispatchOwner(sessionKey, generationId);
|
|
774
|
+
return db.finalizeCodexAcceptedSteer({
|
|
775
|
+
reservation_id: deliveryResultString(
|
|
776
|
+
reservationId,
|
|
777
|
+
'dispatch reservation ID',
|
|
778
|
+
),
|
|
779
|
+
generation_id: owner.generationId,
|
|
780
|
+
steer_attempt_id: deliveryResultString(
|
|
781
|
+
steerAttemptId,
|
|
782
|
+
'steer attempt ID',
|
|
783
|
+
),
|
|
784
|
+
target_attempt_id: deliveryResultString(
|
|
785
|
+
targetAttemptId,
|
|
786
|
+
'target attempt ID',
|
|
787
|
+
),
|
|
788
|
+
stable_host_id: owner.identity.stableHostId,
|
|
789
|
+
boot_session_id: owner.identity.bootSessionId,
|
|
790
|
+
ts: Date.now(),
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
function markDispatchDisposition({
|
|
795
|
+
sessionKey,
|
|
796
|
+
generationId,
|
|
797
|
+
reservationId,
|
|
798
|
+
disposition,
|
|
799
|
+
} = {}) {
|
|
800
|
+
const owner = requireCurrentDispatchOwner(sessionKey, generationId);
|
|
801
|
+
if (!['queue-authorized', 'ambiguous', 'cancelled'].includes(disposition)) {
|
|
802
|
+
throw controllerError(
|
|
803
|
+
'Codex dispatch disposition is invalid.',
|
|
804
|
+
'CODEX_DISPATCH_DISPOSITION_INVALID',
|
|
805
|
+
'Use a supported durable dispatch disposition.',
|
|
806
|
+
);
|
|
807
|
+
}
|
|
808
|
+
return db.markCodexDispatchDisposition({
|
|
809
|
+
reservation_id: deliveryResultString(
|
|
810
|
+
reservationId,
|
|
811
|
+
'dispatch reservation ID',
|
|
812
|
+
),
|
|
813
|
+
generation_id: owner.generationId,
|
|
814
|
+
disposition,
|
|
815
|
+
stable_host_id: owner.identity.stableHostId,
|
|
816
|
+
boot_session_id: owner.identity.bootSessionId,
|
|
817
|
+
ts: Date.now(),
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
function settleQueuedDispatch({
|
|
822
|
+
sessionKey,
|
|
823
|
+
generationId,
|
|
824
|
+
reservationId,
|
|
825
|
+
attemptId,
|
|
826
|
+
botName,
|
|
827
|
+
telegramChatId,
|
|
828
|
+
telegramMessageId,
|
|
829
|
+
} = {}) {
|
|
830
|
+
const owner = requireCurrentDispatchOwner(sessionKey, generationId);
|
|
831
|
+
const exactBotName = deliveryResultString(botName, 'bot name');
|
|
832
|
+
const exactChatId = deliveryResultString(
|
|
833
|
+
telegramChatId,
|
|
834
|
+
'Telegram chat ID',
|
|
835
|
+
);
|
|
836
|
+
const exactMessageId = deliveryResultString(
|
|
837
|
+
telegramMessageId,
|
|
838
|
+
'Telegram message ID',
|
|
839
|
+
);
|
|
840
|
+
const exactReservationId = deliveryResultString(
|
|
841
|
+
reservationId,
|
|
842
|
+
'dispatch reservation ID',
|
|
843
|
+
);
|
|
844
|
+
if (exactReservationId !== dispatchReservationId({
|
|
845
|
+
botName: exactBotName,
|
|
846
|
+
telegramChatId: exactChatId,
|
|
847
|
+
telegramMessageId: exactMessageId,
|
|
848
|
+
})) {
|
|
849
|
+
throw controllerError(
|
|
850
|
+
'Codex queued settlement reservation identity conflicts.',
|
|
851
|
+
'CODEX_QUEUE_SETTLEMENT_CONFLICT',
|
|
852
|
+
'Do not settle a queued turn against another Telegram input.',
|
|
853
|
+
);
|
|
854
|
+
}
|
|
855
|
+
return db.settleCodexQueuedDispatch({
|
|
856
|
+
attempt_id: deliveryResultString(attemptId, 'attempt ID'),
|
|
857
|
+
generation_id: owner.generationId,
|
|
858
|
+
session_key: owner.sessionKey,
|
|
859
|
+
bot_name: exactBotName,
|
|
860
|
+
telegram_chat_id: exactChatId,
|
|
861
|
+
telegram_message_id: exactMessageId,
|
|
862
|
+
stable_host_id: owner.identity.stableHostId,
|
|
863
|
+
boot_session_id: owner.identity.bootSessionId,
|
|
864
|
+
ts: Date.now(),
|
|
865
|
+
});
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
async function settleTelegramDelivery(
|
|
869
|
+
sessionKey,
|
|
870
|
+
result,
|
|
871
|
+
{ disposition = 'delivered' } = {},
|
|
872
|
+
) {
|
|
873
|
+
const claimsCodex = (
|
|
874
|
+
result?.runtime === 'codex'
|
|
875
|
+
|| result?.backend === 'codex'
|
|
876
|
+
);
|
|
877
|
+
if (!claimsCodex) return;
|
|
878
|
+
if (!['delivered', 'failed'].includes(disposition)) {
|
|
879
|
+
throw controllerError(
|
|
880
|
+
'Codex delivery disposition is invalid.',
|
|
881
|
+
'CODEX_DELIVERY_DISPOSITION_INVALID',
|
|
882
|
+
'Use delivered or failed for the exact Telegram outcome.',
|
|
883
|
+
);
|
|
884
|
+
}
|
|
885
|
+
if (
|
|
886
|
+
(
|
|
887
|
+
result.runtime !== undefined
|
|
888
|
+
&& result.runtime !== 'codex'
|
|
889
|
+
)
|
|
890
|
+
|| (
|
|
891
|
+
result.backend !== undefined
|
|
892
|
+
&& result.backend !== 'codex'
|
|
893
|
+
)
|
|
894
|
+
) {
|
|
895
|
+
throw controllerError(
|
|
896
|
+
'Codex delivery runtime evidence is contradictory.',
|
|
897
|
+
'CODEX_DELIVERY_RESULT_INVALID',
|
|
898
|
+
'Discard the malformed result without settling durable delivery.',
|
|
899
|
+
);
|
|
900
|
+
}
|
|
901
|
+
const exactSessionKey = deliveryResultString(sessionKey, 'session key');
|
|
902
|
+
const generationId = deliveryResultString(
|
|
903
|
+
result?.generationId,
|
|
904
|
+
'generation ID',
|
|
905
|
+
);
|
|
906
|
+
const attemptId = deliveryResultString(
|
|
907
|
+
result?.attemptId,
|
|
908
|
+
'attempt ID',
|
|
909
|
+
);
|
|
910
|
+
const providerSessionId = deliveryResultString(
|
|
911
|
+
result?.providerSessionId,
|
|
912
|
+
'provider session ID',
|
|
913
|
+
);
|
|
914
|
+
const providerTurnId = deliveryResultString(
|
|
915
|
+
result?.providerTurnId,
|
|
916
|
+
'provider turn ID',
|
|
917
|
+
);
|
|
918
|
+
const record = processesByGeneration.get(generationId);
|
|
919
|
+
if (
|
|
920
|
+
record
|
|
921
|
+
&& currentProcessBySession.get(exactSessionKey) === record
|
|
922
|
+
&& record.proc.sessionKey === exactSessionKey
|
|
923
|
+
&& record.proc.generationId === generationId
|
|
924
|
+
&& record.durabilityBlocked
|
|
925
|
+
) {
|
|
926
|
+
throw record.durabilityError;
|
|
927
|
+
}
|
|
928
|
+
if (
|
|
929
|
+
!record
|
|
930
|
+
|| currentProcessBySession.get(exactSessionKey) !== record
|
|
931
|
+
|| record.proc.sessionKey !== exactSessionKey
|
|
932
|
+
|| record.proc.generationId !== generationId
|
|
933
|
+
|| !record.durable
|
|
934
|
+
|| !(
|
|
935
|
+
record.proc.runtime === 'codex'
|
|
936
|
+
|| record.proc.backend === 'codex'
|
|
937
|
+
)
|
|
938
|
+
) {
|
|
939
|
+
const stopped = retiredStopCancellations.get(generationId);
|
|
940
|
+
if (
|
|
941
|
+
disposition === 'failed'
|
|
942
|
+
&& stopped?.sessionKey === exactSessionKey
|
|
943
|
+
&& stopped.attemptId === attemptId
|
|
944
|
+
&& stopped.providerSessionId === providerSessionId
|
|
945
|
+
&& stopped.providerTurnId === providerTurnId
|
|
946
|
+
) {
|
|
947
|
+
return Object.freeze({
|
|
948
|
+
committed: true,
|
|
949
|
+
disposition: 'stop-cancelled',
|
|
950
|
+
generationId,
|
|
951
|
+
attemptId,
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
throw controllerError(
|
|
955
|
+
'Codex delivery does not belong to the current registered process.',
|
|
956
|
+
'CODEX_DELIVERY_BINDING_MISSING',
|
|
957
|
+
'Do not settle delivery for a stale or unregistered process.',
|
|
958
|
+
);
|
|
959
|
+
}
|
|
960
|
+
const attempt = db.getCodexAttempt(attemptId);
|
|
961
|
+
if (
|
|
962
|
+
record.proc.providerSessionId !== providerSessionId
|
|
963
|
+
|| !attempt
|
|
964
|
+
|| attempt.attempt_id !== attemptId
|
|
965
|
+
|| attempt.generation_id !== generationId
|
|
966
|
+
|| attempt.session_key !== exactSessionKey
|
|
967
|
+
|| attempt.thread_id !== providerSessionId
|
|
968
|
+
|| attempt.turn_id !== providerTurnId
|
|
969
|
+
) {
|
|
970
|
+
throw controllerError(
|
|
971
|
+
'Codex delivery identities do not match the registered turn attempt.',
|
|
972
|
+
'CODEX_DELIVERY_IDENTITY_MISMATCH',
|
|
973
|
+
'Discard the mismatched result without settling durable delivery.',
|
|
974
|
+
);
|
|
975
|
+
}
|
|
976
|
+
const state = requireAvailableInitialization();
|
|
977
|
+
try {
|
|
978
|
+
const retireGeneration = (
|
|
979
|
+
record.closed
|
|
980
|
+
&& record.healthyStopObserved
|
|
981
|
+
&& record.retirementRequested
|
|
982
|
+
&& !record.retirementVerificationFailed
|
|
983
|
+
&& !record.durabilityBlocked
|
|
984
|
+
);
|
|
985
|
+
const checkpoint = {
|
|
986
|
+
kind: disposition === 'delivered'
|
|
987
|
+
? 'telegram-delivery-settled'
|
|
988
|
+
: 'telegram-delivery-failed',
|
|
989
|
+
generationId: record.proc.generationId,
|
|
990
|
+
hostIdentity: state.identity.stableHostId,
|
|
991
|
+
bootSessionIdentity: state.identity.bootSessionId,
|
|
992
|
+
attemptId: attempt.attempt_id,
|
|
993
|
+
threadId: attempt.thread_id,
|
|
994
|
+
turnId: attempt.turn_id,
|
|
995
|
+
ts: Date.now(),
|
|
996
|
+
};
|
|
997
|
+
const persisted = db.recordCodexDeliveryCheckpoint({
|
|
998
|
+
checkpoint,
|
|
999
|
+
retireGeneration,
|
|
1000
|
+
});
|
|
1001
|
+
if (retireGeneration && persisted.retired !== true) {
|
|
1002
|
+
throw controllerError(
|
|
1003
|
+
'Codex delivery did not retire the verified stopped generation.',
|
|
1004
|
+
'CODEX_RETIREMENT_UNVERIFIED',
|
|
1005
|
+
'Keep the exact generation and daemon lease fenced.',
|
|
1006
|
+
);
|
|
1007
|
+
}
|
|
1008
|
+
maybeRetire(record);
|
|
1009
|
+
} catch (error) {
|
|
1010
|
+
record.durabilityBlocked = true;
|
|
1011
|
+
record.durabilityError = error;
|
|
1012
|
+
try {
|
|
1013
|
+
record.proc.blockDurability(error);
|
|
1014
|
+
} catch (fenceError) {
|
|
1015
|
+
logger.error?.(
|
|
1016
|
+
`[codex] failed to apply delivery durability fence for ${exactSessionKey}: ${fenceError.message}`,
|
|
1017
|
+
);
|
|
1018
|
+
}
|
|
1019
|
+
throw error;
|
|
1020
|
+
}
|
|
1021
|
+
return Object.freeze({
|
|
1022
|
+
committed: true,
|
|
1023
|
+
disposition,
|
|
1024
|
+
generationId,
|
|
1025
|
+
attemptId,
|
|
1026
|
+
});
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
function activateWithPreparedCheckpoint(record, payload) {
|
|
1030
|
+
if (payload.kind !== 'request-prepared') {
|
|
1031
|
+
throw controllerError(
|
|
1032
|
+
'Codex generation reached durability out of order.',
|
|
1033
|
+
'CODEX_GENERATION_NOT_DURABLE',
|
|
1034
|
+
'Quarantine the generation and inspect its first checkpoint.',
|
|
1035
|
+
);
|
|
1036
|
+
}
|
|
1037
|
+
const state = requireAvailableInitialization();
|
|
1038
|
+
const transaction = db.raw?.transaction?.(() => {
|
|
1039
|
+
db.createCodexGeneration({
|
|
1040
|
+
generation_id: record.proc.generationId,
|
|
1041
|
+
session_key: record.proc.sessionKey,
|
|
1042
|
+
thread_id: record.proc.spawnOptions?.existingSessionId ?? null,
|
|
1043
|
+
app_server_session_id: null,
|
|
1044
|
+
stable_host_id: state.identity.stableHostId,
|
|
1045
|
+
boot_session_id: state.identity.bootSessionId,
|
|
1046
|
+
ts: Date.now(),
|
|
1047
|
+
});
|
|
1048
|
+
db.recordCodexCheckpoint(payload);
|
|
1049
|
+
db.acquireCodexLease({
|
|
1050
|
+
generation_id: record.proc.generationId,
|
|
1051
|
+
stable_host_id: state.identity.stableHostId,
|
|
1052
|
+
boot_session_id: state.identity.bootSessionId,
|
|
1053
|
+
ts: Date.now(),
|
|
1054
|
+
});
|
|
1055
|
+
});
|
|
1056
|
+
if (typeof transaction !== 'function') {
|
|
1057
|
+
throw new TypeError('Codex durability requires transactional DB access');
|
|
1058
|
+
}
|
|
1059
|
+
transaction();
|
|
1060
|
+
record.durable = true;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
function maybeRetire(record) {
|
|
1064
|
+
if (
|
|
1065
|
+
record.retired
|
|
1066
|
+
|| record.durabilityBlocked
|
|
1067
|
+
|| record.retirementVerificationFailed
|
|
1068
|
+
|| !record.retirementRequested
|
|
1069
|
+
|| !record.durable
|
|
1070
|
+
|| !record.closed
|
|
1071
|
+
|| !(
|
|
1072
|
+
record.healthyStopObserved
|
|
1073
|
+
|| record.proc.startupReleaseSafe === true
|
|
1074
|
+
)
|
|
1075
|
+
) return false;
|
|
1076
|
+
try {
|
|
1077
|
+
db.markCodexGenerationRetired({
|
|
1078
|
+
generation_id: record.proc.generationId,
|
|
1079
|
+
ts: Date.now(),
|
|
1080
|
+
});
|
|
1081
|
+
record.retired = true;
|
|
1082
|
+
if (record.stopCancellation) {
|
|
1083
|
+
rememberStopCancellation(record.stopCancellation);
|
|
1084
|
+
}
|
|
1085
|
+
processesByGeneration.delete(record.proc.generationId);
|
|
1086
|
+
forgetRetiredSession(record);
|
|
1087
|
+
for (const waiter of record.retirementWaiters.splice(0)) {
|
|
1088
|
+
waiter.resolve(Object.freeze({
|
|
1089
|
+
committed: true,
|
|
1090
|
+
disposition: 'retired',
|
|
1091
|
+
}));
|
|
1092
|
+
}
|
|
1093
|
+
return true;
|
|
1094
|
+
} catch (error) {
|
|
1095
|
+
if (error?.code === 'CODEX_RETIREMENT_UNVERIFIED') return false;
|
|
1096
|
+
throw error;
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
async function verifyCodexRetirement({
|
|
1101
|
+
sessionKey,
|
|
1102
|
+
generationId,
|
|
1103
|
+
terminalStatus = null,
|
|
1104
|
+
turnId = null,
|
|
1105
|
+
signal = null,
|
|
1106
|
+
} = {}) {
|
|
1107
|
+
const exactSessionKey = deliveryResultString(sessionKey, 'session key');
|
|
1108
|
+
const exactGenerationId = deliveryResultString(
|
|
1109
|
+
generationId,
|
|
1110
|
+
'generation ID',
|
|
1111
|
+
);
|
|
1112
|
+
const record = processesByGeneration.get(exactGenerationId);
|
|
1113
|
+
if (
|
|
1114
|
+
!record
|
|
1115
|
+
|| currentProcessBySession.get(exactSessionKey) !== record
|
|
1116
|
+
|| record.proc.sessionKey !== exactSessionKey
|
|
1117
|
+
|| record.proc.generationId !== exactGenerationId
|
|
1118
|
+
|| !record.durable
|
|
1119
|
+
|| !record.closed
|
|
1120
|
+
|| !record.healthyStopObserved
|
|
1121
|
+
|| !(
|
|
1122
|
+
record.proc.runtime === 'codex'
|
|
1123
|
+
|| record.proc.backend === 'codex'
|
|
1124
|
+
)
|
|
1125
|
+
) {
|
|
1126
|
+
throw controllerError(
|
|
1127
|
+
'Codex retirement does not belong to the exact stopped generation.',
|
|
1128
|
+
'CODEX_RETIREMENT_BINDING_MISSING',
|
|
1129
|
+
'Keep the daemon ownership fence and quarantine stale retirement.',
|
|
1130
|
+
);
|
|
1131
|
+
}
|
|
1132
|
+
if (
|
|
1133
|
+
terminalStatus != null
|
|
1134
|
+
&& !['completed', 'interrupted', 'failed'].includes(terminalStatus)
|
|
1135
|
+
) {
|
|
1136
|
+
throw controllerError(
|
|
1137
|
+
'Codex retirement terminal status is invalid.',
|
|
1138
|
+
'CODEX_RETIREMENT_RESULT_INVALID',
|
|
1139
|
+
'Keep the exact generation fenced.',
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
if (turnId != null) deliveryResultString(turnId, 'turn ID');
|
|
1143
|
+
if (
|
|
1144
|
+
signal != null
|
|
1145
|
+
&& (
|
|
1146
|
+
typeof signal.aborted !== 'boolean'
|
|
1147
|
+
|| typeof signal.addEventListener !== 'function'
|
|
1148
|
+
)
|
|
1149
|
+
) {
|
|
1150
|
+
throw new TypeError('Codex retirement signal must be an AbortSignal');
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
record.retirementRequested = true;
|
|
1154
|
+
try {
|
|
1155
|
+
if (signal?.aborted) {
|
|
1156
|
+
throw controllerError(
|
|
1157
|
+
'Codex retirement verification was aborted.',
|
|
1158
|
+
'CODEX_RETIREMENT_VERIFICATION_ABORTED',
|
|
1159
|
+
'Keep the exact generation and durable lease fenced.',
|
|
1160
|
+
signal.reason,
|
|
1161
|
+
);
|
|
1162
|
+
}
|
|
1163
|
+
const state = requireAvailableInitialization();
|
|
1164
|
+
const settlement = db.settleCodexStoppedGeneration({
|
|
1165
|
+
generation_id: exactGenerationId,
|
|
1166
|
+
stable_host_id: state.identity.stableHostId,
|
|
1167
|
+
boot_session_id: state.identity.bootSessionId,
|
|
1168
|
+
ts: Date.now(),
|
|
1169
|
+
});
|
|
1170
|
+
if (settlement.disposition === 'stop-cancelled') {
|
|
1171
|
+
const attempt = db.getCodexAttempt(settlement.attemptId);
|
|
1172
|
+
if (
|
|
1173
|
+
!attempt
|
|
1174
|
+
|| attempt.generation_id !== exactGenerationId
|
|
1175
|
+
|| attempt.session_key !== exactSessionKey
|
|
1176
|
+
|| attempt.recovery_state !== 'cancelled'
|
|
1177
|
+
|| attempt.terminal_status !== 'interrupted'
|
|
1178
|
+
|| typeof attempt.thread_id !== 'string'
|
|
1179
|
+
|| typeof attempt.turn_id !== 'string'
|
|
1180
|
+
) {
|
|
1181
|
+
throw controllerError(
|
|
1182
|
+
'Codex stop cancellation lacks an exact durable attempt binding.',
|
|
1183
|
+
'CODEX_RETIREMENT_UNVERIFIED',
|
|
1184
|
+
'Keep the daemon ownership fence and inspect the stopped attempt.',
|
|
1185
|
+
);
|
|
1186
|
+
}
|
|
1187
|
+
record.stopCancellation = Object.freeze({
|
|
1188
|
+
sessionKey: exactSessionKey,
|
|
1189
|
+
generationId: exactGenerationId,
|
|
1190
|
+
attemptId: attempt.attempt_id,
|
|
1191
|
+
providerSessionId: attempt.thread_id,
|
|
1192
|
+
providerTurnId: attempt.turn_id,
|
|
1193
|
+
});
|
|
1194
|
+
}
|
|
1195
|
+
if (maybeRetire(record)) {
|
|
1196
|
+
return Object.freeze({
|
|
1197
|
+
committed: true,
|
|
1198
|
+
disposition: settlement.disposition === 'stop-cancelled'
|
|
1199
|
+
? 'stop-cancelled'
|
|
1200
|
+
: 'retired',
|
|
1201
|
+
});
|
|
1202
|
+
}
|
|
1203
|
+
if (settlement.disposition !== 'pending-delivery') {
|
|
1204
|
+
throw controllerError(
|
|
1205
|
+
'Codex durable retirement remained unresolved after disposal.',
|
|
1206
|
+
'CODEX_RETIREMENT_UNVERIFIED',
|
|
1207
|
+
'Keep the daemon ownership fence and inspect unresolved attempts.',
|
|
1208
|
+
);
|
|
1209
|
+
}
|
|
1210
|
+
return await new Promise((resolve, reject) => {
|
|
1211
|
+
const waiter = {
|
|
1212
|
+
resolve(value) {
|
|
1213
|
+
signal?.removeEventListener?.('abort', onAbort);
|
|
1214
|
+
resolve(value);
|
|
1215
|
+
},
|
|
1216
|
+
reject(error) {
|
|
1217
|
+
signal?.removeEventListener?.('abort', onAbort);
|
|
1218
|
+
reject(error);
|
|
1219
|
+
},
|
|
1220
|
+
};
|
|
1221
|
+
function onAbort() {
|
|
1222
|
+
const index = record.retirementWaiters.indexOf(waiter);
|
|
1223
|
+
if (index !== -1) record.retirementWaiters.splice(index, 1);
|
|
1224
|
+
waiter.reject(controllerError(
|
|
1225
|
+
'Codex retirement verification was aborted.',
|
|
1226
|
+
'CODEX_RETIREMENT_VERIFICATION_ABORTED',
|
|
1227
|
+
'Keep the exact generation and durable lease fenced.',
|
|
1228
|
+
signal.reason,
|
|
1229
|
+
));
|
|
1230
|
+
}
|
|
1231
|
+
record.retirementWaiters.push(waiter);
|
|
1232
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1233
|
+
if (signal?.aborted) onAbort();
|
|
1234
|
+
});
|
|
1235
|
+
} catch (error) {
|
|
1236
|
+
record.retirementVerificationFailed = true;
|
|
1237
|
+
throw error;
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
async function checkpointSink(payload) {
|
|
1242
|
+
const record = processesByGeneration.get(payload?.generationId);
|
|
1243
|
+
const state = requireAvailableInitialization();
|
|
1244
|
+
if (
|
|
1245
|
+
!record
|
|
1246
|
+
|| payload.hostIdentity !== state.identity.stableHostId
|
|
1247
|
+
|| payload.bootSessionIdentity !== state.identity.bootSessionId
|
|
1248
|
+
) {
|
|
1249
|
+
throw controllerError(
|
|
1250
|
+
'Codex checkpoint does not belong to the registered generation owner.',
|
|
1251
|
+
'CODEX_CHECKPOINT_STALE_GENERATION',
|
|
1252
|
+
'Quarantine the stale generation.',
|
|
1253
|
+
);
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
if (!record.durable) {
|
|
1257
|
+
activateWithPreparedCheckpoint(record, payload);
|
|
1258
|
+
} else {
|
|
1259
|
+
db.recordCodexCheckpoint(payload);
|
|
1260
|
+
}
|
|
1261
|
+
if (payload.kind === 'request-write-attempted') {
|
|
1262
|
+
payload.markWriteCommitted?.();
|
|
1263
|
+
}
|
|
1264
|
+
if (payload.kind === 'stop-empty-registry-observed') {
|
|
1265
|
+
record.healthyStopObserved = true;
|
|
1266
|
+
}
|
|
1267
|
+
maybeRetire(record);
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
return Object.freeze({
|
|
1271
|
+
initialize,
|
|
1272
|
+
prepareSession,
|
|
1273
|
+
resolveRuntimeView,
|
|
1274
|
+
resolveReceipt,
|
|
1275
|
+
clientFactory,
|
|
1276
|
+
requiresProcessReplacement,
|
|
1277
|
+
processReplacementReason,
|
|
1278
|
+
registerProcess,
|
|
1279
|
+
claimDispatchReservation,
|
|
1280
|
+
finalizeAcceptedSteer,
|
|
1281
|
+
markDispatchDisposition,
|
|
1282
|
+
settleQueuedDispatch,
|
|
1283
|
+
settleTelegramDelivery,
|
|
1284
|
+
verifyCodexRetirement,
|
|
1285
|
+
checkpointSink,
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
module.exports = {
|
|
1290
|
+
CodexRuntimeControllerError,
|
|
1291
|
+
createCodexRuntimeController,
|
|
1292
|
+
};
|