polygram 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +5 -3
- package/config.example.json +23 -1
- package/lib/codex/binary.js +287 -0
- package/lib/codex/delivery-finalizer.js +97 -0
- package/lib/codex/diagnostics.js +483 -0
- package/lib/codex/event-normalizer.js +50 -0
- package/lib/codex/host-identity.js +103 -0
- package/lib/codex/model-catalog.js +359 -0
- package/lib/codex/retention-scheduler.js +59 -0
- package/lib/codex/runtime-controller.js +1292 -0
- package/lib/codex/runtime-profile.js +888 -0
- package/lib/codex/spawn-context.js +87 -0
- package/lib/db/auto-resume.js +77 -9
- package/lib/db/codex-reconciliation.js +490 -0
- package/lib/db/codex-retention.js +139 -0
- package/lib/db/sessions.js +102 -1
- package/lib/db.js +3418 -19
- package/lib/error/classify.js +140 -0
- package/lib/handlers/abort.js +143 -5
- package/lib/handlers/autosteer.js +256 -1
- package/lib/handlers/codex-reconciliation.js +230 -0
- package/lib/handlers/config-callback.js +324 -89
- package/lib/handlers/config-ui.js +401 -39
- package/lib/handlers/dispatcher.js +71 -8
- package/lib/handlers/edit-correction.js +11 -1
- package/lib/handlers/edit-redelivery.js +230 -4
- package/lib/handlers/gate-inbound.js +7 -0
- package/lib/handlers/replay-disposition.js +133 -11
- package/lib/handlers/slash-commands.js +443 -37
- package/lib/ipc/client.js +22 -12
- package/lib/ipc/runtime-directory.js +208 -0
- package/lib/ipc/server.js +18 -12
- package/lib/model-costs.js +4 -3
- package/lib/pm-interface.js +33 -0
- package/lib/prompt.js +15 -9
- package/lib/rewind/rewind.js +21 -1
- package/lib/runtime-config.js +605 -0
- package/lib/sdk/build-options.js +4 -1
- package/lib/sdk/callbacks.js +46 -3
- package/lib/util/claude-session-jsonl.js +3 -2
- package/migrations/015-agent-runtime-sessions.sql +50 -0
- package/migrations/016-agent-turn-attempts.sql +211 -0
- package/migrations/017-config-runtime-changes.sql +46 -0
- package/ops/README.md +57 -0
- package/package.json +2 -2
- package/polygram.js +697 -48
- package/scripts/doctor.js +37 -8
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const {
|
|
4
|
+
resolveProviderSessionForSpawn,
|
|
5
|
+
} = require('../db/sessions');
|
|
6
|
+
|
|
7
|
+
async function buildCodexSpawnContext({
|
|
8
|
+
sessionKey,
|
|
9
|
+
chatId,
|
|
10
|
+
threadId,
|
|
11
|
+
chatConfig,
|
|
12
|
+
db,
|
|
13
|
+
pm,
|
|
14
|
+
runtimeController,
|
|
15
|
+
getSessionLabel,
|
|
16
|
+
logEvent,
|
|
17
|
+
}) {
|
|
18
|
+
const prepared = await runtimeController.prepareSession({
|
|
19
|
+
sessionKey,
|
|
20
|
+
chatId,
|
|
21
|
+
threadId,
|
|
22
|
+
});
|
|
23
|
+
const runtime = prepared.runtimeConfig;
|
|
24
|
+
let liveProcess = pm?.get(sessionKey) ?? null;
|
|
25
|
+
const replacementReason = (
|
|
26
|
+
liveProcess
|
|
27
|
+
&& !liveProcess.closed
|
|
28
|
+
) ? runtimeController.processReplacementReason(
|
|
29
|
+
sessionKey,
|
|
30
|
+
liveProcess,
|
|
31
|
+
) : null;
|
|
32
|
+
if (replacementReason) {
|
|
33
|
+
logEvent('codex-runtime-state-replacement', {
|
|
34
|
+
chat_id: chatId,
|
|
35
|
+
thread_id: threadId,
|
|
36
|
+
session_key: sessionKey,
|
|
37
|
+
generation_id: liveProcess.generationId,
|
|
38
|
+
reason: replacementReason,
|
|
39
|
+
});
|
|
40
|
+
await pm.kill(sessionKey, replacementReason);
|
|
41
|
+
liveProcess = null;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const isColdSpawn = !liveProcess || liveProcess.closed;
|
|
45
|
+
let existingSessionId = null;
|
|
46
|
+
if (isColdSpawn) {
|
|
47
|
+
const resolved = resolveProviderSessionForSpawn(db, sessionKey, {
|
|
48
|
+
runtime: 'codex',
|
|
49
|
+
backend: 'codex',
|
|
50
|
+
agent: null,
|
|
51
|
+
cwd: runtime.cwd,
|
|
52
|
+
model: runtime.model,
|
|
53
|
+
effort: runtime.effort,
|
|
54
|
+
});
|
|
55
|
+
existingSessionId = resolved.existingSessionId;
|
|
56
|
+
if (resolved.drift) {
|
|
57
|
+
logEvent('session-config-drift', {
|
|
58
|
+
chat_id: chatId,
|
|
59
|
+
thread_id: threadId,
|
|
60
|
+
session_key: sessionKey,
|
|
61
|
+
provider: 'codex',
|
|
62
|
+
fields: resolved.drift.fields,
|
|
63
|
+
before: resolved.drift.before,
|
|
64
|
+
after: resolved.drift.after,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
} else {
|
|
68
|
+
existingSessionId = liveProcess.providerSessionId ?? null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
runtime: 'codex',
|
|
73
|
+
spawnProfileId: runtime.spawnProfileId,
|
|
74
|
+
modelSettings: {
|
|
75
|
+
model: runtime.model,
|
|
76
|
+
effort: runtime.effort,
|
|
77
|
+
},
|
|
78
|
+
chatId,
|
|
79
|
+
threadId,
|
|
80
|
+
label: getSessionLabel(chatConfig, threadId),
|
|
81
|
+
existingSessionId,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
module.exports = {
|
|
86
|
+
buildCodexSpawnContext,
|
|
87
|
+
};
|
package/lib/db/auto-resume.js
CHANGED
|
@@ -76,20 +76,87 @@ function createAutoResumeTracker({ cooldownMs = DEFAULT_COOLDOWN_MS, now = Date.
|
|
|
76
76
|
};
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
/**
|
|
80
|
+
* Classify the durable provider ledger for one Codex Telegram input.
|
|
81
|
+
*
|
|
82
|
+
* A request is safe to submit automatically only when the ledger proves that
|
|
83
|
+
* no provider write occurred. Explicit cancellation is also definitely
|
|
84
|
+
* not-sent, but remains cancelled rather than becoming new work. Every other
|
|
85
|
+
* state fails closed because re-submission could duplicate text or tool
|
|
86
|
+
* effects.
|
|
87
|
+
*
|
|
88
|
+
* Linked steering inputs are never independent work. Their state follows the
|
|
89
|
+
* target turn, which is returned only for diagnostics and reconciliation.
|
|
90
|
+
*/
|
|
91
|
+
function classifyCodexRecoveryEvidence(evidence = {}) {
|
|
92
|
+
const deliveryState = evidence?.delivery_state ?? evidence?.deliveryState;
|
|
93
|
+
const recoveryState = evidence?.recovery_state ?? evidence?.recoveryState;
|
|
94
|
+
const linkedState = (
|
|
95
|
+
evidence?.linked_input_state
|
|
96
|
+
?? evidence?.linkedInputState
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
if (linkedState != null) {
|
|
100
|
+
const target = classifyCodexRecoveryEvidence({
|
|
101
|
+
delivery_state: (
|
|
102
|
+
evidence?.target_delivery_state
|
|
103
|
+
?? evidence?.targetDeliveryState
|
|
104
|
+
),
|
|
105
|
+
recovery_state: (
|
|
106
|
+
evidence?.target_recovery_state
|
|
107
|
+
?? evidence?.targetRecoveryState
|
|
108
|
+
),
|
|
109
|
+
});
|
|
110
|
+
return {
|
|
111
|
+
action: 'defer',
|
|
112
|
+
reason: 'linked-input',
|
|
113
|
+
target,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (deliveryState === 'prepared' && recoveryState === 'prepared') {
|
|
118
|
+
return {
|
|
119
|
+
action: 'recover',
|
|
120
|
+
reason: 'request-proven-not-accepted',
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
if (deliveryState === 'prepared' && recoveryState === 'cancelled') {
|
|
124
|
+
return {
|
|
125
|
+
action: 'skip',
|
|
126
|
+
reason: 'request-cancelled-before-acceptance',
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
action: 'defer',
|
|
131
|
+
reason: 'provider-acceptance-not-disproven',
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
79
135
|
/**
|
|
80
136
|
* Decide whether an error is a candidate for auto-resume.
|
|
81
137
|
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
* - NOT a boot-replay (the user typed this minutes ago and moved
|
|
88
|
-
* on; resuming now is more confusing than helpful)
|
|
89
|
-
* - NOT during shutdown (boot replay will pick it up)
|
|
138
|
+
* Claude keeps its established timeout and bridge-disconnect behavior. Codex
|
|
139
|
+
* additionally requires a typed definitely-not-sent error and matching durable
|
|
140
|
+
* evidence that no provider write occurred.
|
|
141
|
+
*
|
|
142
|
+
* All providers reject user-aborted, boot-replay, and shutdown work.
|
|
90
143
|
*/
|
|
91
|
-
function isAutoResumable({
|
|
144
|
+
function isAutoResumable({
|
|
145
|
+
error,
|
|
146
|
+
aborted,
|
|
147
|
+
replay,
|
|
148
|
+
shuttingDown,
|
|
149
|
+
provider,
|
|
150
|
+
providerRecovery,
|
|
151
|
+
}) {
|
|
92
152
|
if (aborted || replay || shuttingDown) return false;
|
|
153
|
+
if (provider === 'codex') {
|
|
154
|
+
const disposition = classifyCodexRecoveryEvidence(providerRecovery);
|
|
155
|
+
return (
|
|
156
|
+
disposition.action === 'recover'
|
|
157
|
+
&& error?.code === 'CODEX_RPC_NOT_SENT'
|
|
158
|
+
);
|
|
159
|
+
}
|
|
93
160
|
// Delivery may have happened before the contained process tree disappeared.
|
|
94
161
|
// Automatic continuation could duplicate work; a user resend starts fresh.
|
|
95
162
|
if (error?.code === 'SESSION_PROCESS_LOST') return false;
|
|
@@ -106,6 +173,7 @@ function isAutoResumable({ error, aborted, replay, shuttingDown }) {
|
|
|
106
173
|
|
|
107
174
|
module.exports = {
|
|
108
175
|
createAutoResumeTracker,
|
|
176
|
+
classifyCodexRecoveryEvidence,
|
|
109
177
|
isAutoResumable,
|
|
110
178
|
DEFAULT_COOLDOWN_MS,
|
|
111
179
|
};
|
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
function codexError(message, code) {
|
|
4
|
+
const error = new Error(message);
|
|
5
|
+
error.code = code;
|
|
6
|
+
return error;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function requiredString(value, label) {
|
|
10
|
+
if (
|
|
11
|
+
typeof value !== 'string'
|
|
12
|
+
|| value.length === 0
|
|
13
|
+
|| value.length > 512
|
|
14
|
+
) {
|
|
15
|
+
throw codexError(
|
|
16
|
+
`Codex ${label} must be a non-empty bounded string`,
|
|
17
|
+
'CODEX_PERSISTENCE_INPUT_INVALID',
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function listRecoveryAttempts(rawDb) {
|
|
24
|
+
const replayableAttemptIds = rawDb.prepare(`
|
|
25
|
+
SELECT attempt_id
|
|
26
|
+
FROM codex_turn_attempts
|
|
27
|
+
WHERE delivery_state = 'prepared'
|
|
28
|
+
AND recovery_state = 'prepared'
|
|
29
|
+
ORDER BY created_ts, attempt_id
|
|
30
|
+
`).all().map((row) => row.attempt_id);
|
|
31
|
+
const unresolved = rawDb.prepare(`
|
|
32
|
+
SELECT attempt.attempt_id,
|
|
33
|
+
EXISTS (
|
|
34
|
+
SELECT 1
|
|
35
|
+
FROM codex_reboot_releases release
|
|
36
|
+
WHERE release.stable_host_id = generation.stable_host_id
|
|
37
|
+
AND release.incident_boot_session_id = generation.boot_session_id
|
|
38
|
+
) AS reboot_released
|
|
39
|
+
FROM codex_turn_attempts attempt
|
|
40
|
+
JOIN codex_generations generation
|
|
41
|
+
ON generation.generation_id = attempt.generation_id
|
|
42
|
+
WHERE attempt.delivery_state != 'prepared'
|
|
43
|
+
AND attempt.recovery_state NOT IN ('settled', 'cancelled')
|
|
44
|
+
ORDER BY attempt.created_ts, attempt.attempt_id
|
|
45
|
+
`).all();
|
|
46
|
+
return {
|
|
47
|
+
replayableAttemptIds,
|
|
48
|
+
unresolvedAttemptIds: unresolved.map((row) => row.attempt_id),
|
|
49
|
+
blockingAttemptIds: unresolved
|
|
50
|
+
.filter((row) => !row.reboot_released)
|
|
51
|
+
.map((row) => row.attempt_id),
|
|
52
|
+
releasedAttemptIds: unresolved
|
|
53
|
+
.filter((row) => row.reboot_released)
|
|
54
|
+
.map((row) => row.attempt_id),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function publicRecoveryAttempts(attempts) {
|
|
59
|
+
return {
|
|
60
|
+
replayableAttemptIds: attempts.replayableAttemptIds,
|
|
61
|
+
unresolvedAttemptIds: attempts.unresolvedAttemptIds,
|
|
62
|
+
...(attempts.releasedAttemptIds.length > 0
|
|
63
|
+
? { releasedAttemptIds: attempts.releasedAttemptIds }
|
|
64
|
+
: {}),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const RETRY_SAFE_GENERATION_STATES = new Set([
|
|
69
|
+
'healthy-stopped',
|
|
70
|
+
'containment-failed',
|
|
71
|
+
'durability-blocked',
|
|
72
|
+
'retired',
|
|
73
|
+
]);
|
|
74
|
+
|
|
75
|
+
function listUnresolvedCodexAttempts(rawDb, input = {}) {
|
|
76
|
+
const sessionKey = input?.session_key == null
|
|
77
|
+
? null
|
|
78
|
+
: requiredString(input.session_key, 'session key');
|
|
79
|
+
return rawDb.prepare(`
|
|
80
|
+
WITH unresolved AS (
|
|
81
|
+
SELECT attempt.attempt_id,
|
|
82
|
+
attempt.generation_id,
|
|
83
|
+
attempt.session_key,
|
|
84
|
+
attempt.method,
|
|
85
|
+
attempt.telegram_source_message_id,
|
|
86
|
+
attempt.ambiguous_ts,
|
|
87
|
+
generation.state AS generation_state
|
|
88
|
+
FROM codex_turn_attempts attempt
|
|
89
|
+
JOIN codex_generations generation
|
|
90
|
+
ON generation.generation_id = attempt.generation_id
|
|
91
|
+
WHERE attempt.recovery_state = 'ambiguous'
|
|
92
|
+
AND NOT EXISTS (
|
|
93
|
+
SELECT 1
|
|
94
|
+
FROM codex_attempt_reconciliations reconciliation
|
|
95
|
+
WHERE reconciliation.attempt_id = attempt.attempt_id
|
|
96
|
+
)
|
|
97
|
+
AND (? IS NULL OR attempt.session_key = ?)
|
|
98
|
+
),
|
|
99
|
+
candidate_locations AS (
|
|
100
|
+
SELECT unresolved.attempt_id,
|
|
101
|
+
reservation.bot_name,
|
|
102
|
+
reservation.telegram_chat_id,
|
|
103
|
+
reservation.telegram_message_id,
|
|
104
|
+
0 AS source_priority,
|
|
105
|
+
reservation.created_ts AS located_ts
|
|
106
|
+
FROM unresolved
|
|
107
|
+
JOIN codex_dispatch_reservations reservation
|
|
108
|
+
ON reservation.steer_attempt_id = unresolved.attempt_id
|
|
109
|
+
OR (
|
|
110
|
+
reservation.target_attempt_id = unresolved.attempt_id
|
|
111
|
+
AND reservation.telegram_message_id
|
|
112
|
+
= unresolved.telegram_source_message_id
|
|
113
|
+
)
|
|
114
|
+
UNION ALL
|
|
115
|
+
SELECT unresolved.attempt_id,
|
|
116
|
+
selection.bot_name,
|
|
117
|
+
selection.telegram_chat_id,
|
|
118
|
+
selection.telegram_message_id,
|
|
119
|
+
1 AS source_priority,
|
|
120
|
+
selection.selected_ts AS located_ts
|
|
121
|
+
FROM unresolved
|
|
122
|
+
JOIN inbound_runtime_selections selection
|
|
123
|
+
ON selection.session_key = unresolved.session_key
|
|
124
|
+
AND selection.provider = 'codex'
|
|
125
|
+
AND selection.telegram_message_id
|
|
126
|
+
= unresolved.telegram_source_message_id
|
|
127
|
+
),
|
|
128
|
+
ranked_locations AS (
|
|
129
|
+
SELECT candidate_locations.*,
|
|
130
|
+
ROW_NUMBER() OVER (
|
|
131
|
+
PARTITION BY candidate_locations.attempt_id
|
|
132
|
+
ORDER BY candidate_locations.source_priority,
|
|
133
|
+
candidate_locations.located_ts,
|
|
134
|
+
candidate_locations.bot_name
|
|
135
|
+
) AS location_rank
|
|
136
|
+
FROM candidate_locations
|
|
137
|
+
)
|
|
138
|
+
SELECT unresolved.attempt_id,
|
|
139
|
+
unresolved.generation_id,
|
|
140
|
+
unresolved.session_key,
|
|
141
|
+
unresolved.method,
|
|
142
|
+
unresolved.telegram_source_message_id,
|
|
143
|
+
unresolved.ambiguous_ts,
|
|
144
|
+
unresolved.generation_state,
|
|
145
|
+
location.bot_name,
|
|
146
|
+
location.telegram_chat_id,
|
|
147
|
+
COALESCE(
|
|
148
|
+
location.telegram_message_id,
|
|
149
|
+
unresolved.telegram_source_message_id
|
|
150
|
+
) AS telegram_message_id,
|
|
151
|
+
message.user_id AS owner_user_id,
|
|
152
|
+
message.user AS owner_user,
|
|
153
|
+
lease.status AS containment_status,
|
|
154
|
+
lease.quarantine_reason
|
|
155
|
+
FROM unresolved
|
|
156
|
+
LEFT JOIN ranked_locations location
|
|
157
|
+
ON location.attempt_id = unresolved.attempt_id
|
|
158
|
+
AND location.location_rank = 1
|
|
159
|
+
LEFT JOIN messages message
|
|
160
|
+
ON message.direction = 'in'
|
|
161
|
+
AND message.bot_name = location.bot_name
|
|
162
|
+
AND message.chat_id = location.telegram_chat_id
|
|
163
|
+
AND CAST(message.msg_id AS TEXT) = COALESCE(
|
|
164
|
+
location.telegram_message_id,
|
|
165
|
+
unresolved.telegram_source_message_id
|
|
166
|
+
)
|
|
167
|
+
LEFT JOIN codex_daemon_lease lease
|
|
168
|
+
ON lease.singleton = 1
|
|
169
|
+
ORDER BY unresolved.ambiguous_ts,
|
|
170
|
+
unresolved.attempt_id
|
|
171
|
+
`).all(sessionKey, sessionKey);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function reconcileCodexAttempt(rawDb, input) {
|
|
175
|
+
const attemptId = requiredString(input?.attempt_id, 'attempt ID');
|
|
176
|
+
const disposition = input?.disposition;
|
|
177
|
+
if (!['incorporated', 'retry-authorized', 'dismissed'].includes(disposition)) {
|
|
178
|
+
throw codexError(
|
|
179
|
+
'Codex reconciliation disposition is invalid',
|
|
180
|
+
'CODEX_RECONCILIATION_INVALID',
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
const actor = requiredString(input?.actor, 'reconciliation actor');
|
|
184
|
+
const reason = requiredString(input?.reason, 'reconciliation reason');
|
|
185
|
+
const decidedTs = input?.ts ?? Date.now();
|
|
186
|
+
if (!Number.isSafeInteger(decidedTs) || decidedTs < 0) {
|
|
187
|
+
throw codexError(
|
|
188
|
+
'Codex reconciliation timestamp is invalid',
|
|
189
|
+
'CODEX_RECONCILIATION_INVALID',
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
let retryAttemptId = null;
|
|
193
|
+
if (disposition === 'retry-authorized') {
|
|
194
|
+
if (input?.duplicate_risk_acknowledged !== true) {
|
|
195
|
+
throw codexError(
|
|
196
|
+
'Codex retry requires an acknowledged duplicate-risk warning',
|
|
197
|
+
'CODEX_DUPLICATE_RISK_NOT_ACKNOWLEDGED',
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
retryAttemptId = requiredString(input?.retry_attempt_id, 'retry attempt ID');
|
|
201
|
+
if (retryAttemptId === attemptId) {
|
|
202
|
+
throw codexError(
|
|
203
|
+
'Codex retry must reserve a distinct attempt ID',
|
|
204
|
+
'CODEX_RECONCILIATION_INVALID',
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
} else if (input?.retry_attempt_id != null) {
|
|
208
|
+
throw codexError(
|
|
209
|
+
'Codex retry attempt ID is valid only for retry authorization',
|
|
210
|
+
'CODEX_RECONCILIATION_INVALID',
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
return rawDb.transaction(() => {
|
|
215
|
+
const attempt = rawDb.prepare(`
|
|
216
|
+
SELECT attempt.attempt_id,
|
|
217
|
+
attempt.recovery_state,
|
|
218
|
+
generation.state AS generation_state
|
|
219
|
+
FROM codex_turn_attempts attempt
|
|
220
|
+
JOIN codex_generations generation
|
|
221
|
+
ON generation.generation_id = attempt.generation_id
|
|
222
|
+
WHERE attempt.attempt_id = ?
|
|
223
|
+
`).get(attemptId);
|
|
224
|
+
if (!attempt) {
|
|
225
|
+
throw codexError(
|
|
226
|
+
'Codex reconciliation attempt does not exist',
|
|
227
|
+
'CODEX_ATTEMPT_NOT_FOUND',
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
if (attempt.recovery_state !== 'ambiguous') {
|
|
231
|
+
throw codexError(
|
|
232
|
+
'Codex reconciliation requires an ambiguous original attempt',
|
|
233
|
+
'CODEX_ATTEMPT_NOT_AMBIGUOUS',
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
if (rawDb.prepare(`
|
|
237
|
+
SELECT 1 FROM codex_attempt_reconciliations WHERE attempt_id = ?
|
|
238
|
+
`).get(attemptId)) {
|
|
239
|
+
throw codexError(
|
|
240
|
+
'Codex attempt already has a terminal reconciliation',
|
|
241
|
+
'CODEX_ATTEMPT_ALREADY_RECONCILED',
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
if (
|
|
245
|
+
retryAttemptId
|
|
246
|
+
&& !RETRY_SAFE_GENERATION_STATES.has(attempt.generation_state)
|
|
247
|
+
) {
|
|
248
|
+
throw codexError(
|
|
249
|
+
'Codex retry authorization requires the prior generation to be terminal or dead',
|
|
250
|
+
'CODEX_RETRY_GENERATION_NOT_TERMINAL',
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
rawDb.prepare(`
|
|
255
|
+
INSERT INTO codex_attempt_reconciliations (
|
|
256
|
+
attempt_id, disposition, actor, reason, decided_ts
|
|
257
|
+
) VALUES (?, ?, ?, ?, ?)
|
|
258
|
+
`).run(attemptId, disposition, actor, reason, decidedTs);
|
|
259
|
+
if (retryAttemptId) {
|
|
260
|
+
rawDb.prepare(`
|
|
261
|
+
INSERT INTO codex_retry_reservations (
|
|
262
|
+
original_attempt_id, retry_attempt_id, state, reserved_ts
|
|
263
|
+
) VALUES (?, ?, 'reserved', ?)
|
|
264
|
+
`).run(attemptId, retryAttemptId, decidedTs);
|
|
265
|
+
}
|
|
266
|
+
return {
|
|
267
|
+
attemptId,
|
|
268
|
+
disposition,
|
|
269
|
+
retryAttemptId,
|
|
270
|
+
decidedTs,
|
|
271
|
+
};
|
|
272
|
+
})();
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
function reconstructCodexRecovery(rawDb, input) {
|
|
276
|
+
let stableHostId;
|
|
277
|
+
let bootSessionId;
|
|
278
|
+
try {
|
|
279
|
+
stableHostId = requiredString(input?.stable_host_id, 'stable host identity');
|
|
280
|
+
bootSessionId = requiredString(input?.boot_session_id, 'boot-session identity');
|
|
281
|
+
} catch {
|
|
282
|
+
return {
|
|
283
|
+
status: 'quarantined',
|
|
284
|
+
reason: 'current-identity-invalid',
|
|
285
|
+
containmentReleased: false,
|
|
286
|
+
replayableAttemptIds: [],
|
|
287
|
+
unresolvedAttemptIds: [],
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
const now = input?.now ?? Date.now();
|
|
291
|
+
if (!Number.isSafeInteger(now) || now < 0) {
|
|
292
|
+
throw codexError(
|
|
293
|
+
'Codex reconstruction timestamp is invalid',
|
|
294
|
+
'CODEX_PERSISTENCE_INPUT_INVALID',
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
return rawDb.transaction(() => {
|
|
299
|
+
const attempts = listRecoveryAttempts(rawDb);
|
|
300
|
+
const identity = rawDb.prepare(`
|
|
301
|
+
SELECT stable_host_id, last_boot_session_id
|
|
302
|
+
FROM codex_runtime_identity
|
|
303
|
+
WHERE singleton = 1
|
|
304
|
+
`).get();
|
|
305
|
+
const lease = rawDb.prepare(`
|
|
306
|
+
SELECT * FROM codex_daemon_lease WHERE singleton = 1
|
|
307
|
+
`).get();
|
|
308
|
+
const generations = rawDb.prepare(`
|
|
309
|
+
SELECT COUNT(*) AS count FROM codex_generations
|
|
310
|
+
`).get().count;
|
|
311
|
+
const hasOperationalState = Boolean(
|
|
312
|
+
generations
|
|
313
|
+
|| lease
|
|
314
|
+
|| attempts.replayableAttemptIds.length
|
|
315
|
+
|| attempts.unresolvedAttemptIds.length
|
|
316
|
+
);
|
|
317
|
+
|
|
318
|
+
if (!identity) {
|
|
319
|
+
if (!hasOperationalState) {
|
|
320
|
+
rawDb.prepare(`
|
|
321
|
+
INSERT INTO codex_runtime_identity (
|
|
322
|
+
singleton, stable_host_id, last_boot_session_id,
|
|
323
|
+
established_ts, updated_ts
|
|
324
|
+
) VALUES (1, ?, ?, ?, ?)
|
|
325
|
+
`).run(stableHostId, bootSessionId, now, now);
|
|
326
|
+
return {
|
|
327
|
+
status: 'clear',
|
|
328
|
+
reason: null,
|
|
329
|
+
containmentReleased: false,
|
|
330
|
+
...publicRecoveryAttempts(attempts),
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
rawDb.prepare(`
|
|
334
|
+
INSERT INTO codex_daemon_lease (
|
|
335
|
+
singleton, generation_id, stable_host_id, boot_session_id,
|
|
336
|
+
status, quarantine_reason, updated_ts
|
|
337
|
+
) VALUES (1, NULL, ?, ?, 'quarantined',
|
|
338
|
+
'persisted-identity-missing', ?)
|
|
339
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
340
|
+
status = 'quarantined',
|
|
341
|
+
quarantine_reason = 'persisted-identity-missing',
|
|
342
|
+
updated_ts = excluded.updated_ts
|
|
343
|
+
`).run(stableHostId, bootSessionId, now);
|
|
344
|
+
return {
|
|
345
|
+
status: 'quarantined',
|
|
346
|
+
reason: 'persisted-identity-missing',
|
|
347
|
+
containmentReleased: false,
|
|
348
|
+
...publicRecoveryAttempts(attempts),
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
if (
|
|
353
|
+
typeof identity.stable_host_id !== 'string'
|
|
354
|
+
|| identity.stable_host_id.length === 0
|
|
355
|
+
|| typeof identity.last_boot_session_id !== 'string'
|
|
356
|
+
|| identity.last_boot_session_id.length === 0
|
|
357
|
+
) {
|
|
358
|
+
return {
|
|
359
|
+
status: 'quarantined',
|
|
360
|
+
reason: 'persisted-identity-corrupt',
|
|
361
|
+
containmentReleased: false,
|
|
362
|
+
...publicRecoveryAttempts(attempts),
|
|
363
|
+
};
|
|
364
|
+
}
|
|
365
|
+
if (identity.stable_host_id !== stableHostId) {
|
|
366
|
+
const incidentBoot = lease?.boot_session_id
|
|
367
|
+
?? identity.last_boot_session_id;
|
|
368
|
+
rawDb.prepare(`
|
|
369
|
+
INSERT INTO codex_daemon_lease (
|
|
370
|
+
singleton, generation_id, stable_host_id, boot_session_id,
|
|
371
|
+
status, quarantine_reason, updated_ts
|
|
372
|
+
) VALUES (1, NULL, ?, ?, 'quarantined', 'stable-host-mismatch', ?)
|
|
373
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
374
|
+
status = 'quarantined',
|
|
375
|
+
quarantine_reason = 'stable-host-mismatch',
|
|
376
|
+
updated_ts = excluded.updated_ts
|
|
377
|
+
`).run(identity.stable_host_id, incidentBoot, now);
|
|
378
|
+
return {
|
|
379
|
+
status: 'quarantined',
|
|
380
|
+
reason: 'stable-host-mismatch',
|
|
381
|
+
containmentReleased: false,
|
|
382
|
+
...publicRecoveryAttempts(attempts),
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
if (identity.last_boot_session_id !== bootSessionId) {
|
|
387
|
+
const containmentReleased = Boolean(
|
|
388
|
+
lease?.status === 'quarantined'
|
|
389
|
+
|| lease?.status === 'active'
|
|
390
|
+
|| attempts.blockingAttemptIds.length
|
|
391
|
+
);
|
|
392
|
+
rawDb.prepare(`
|
|
393
|
+
INSERT INTO codex_reboot_releases (
|
|
394
|
+
stable_host_id, incident_boot_session_id,
|
|
395
|
+
released_boot_session_id, released_ts
|
|
396
|
+
) VALUES (?, ?, ?, ?)
|
|
397
|
+
ON CONFLICT(stable_host_id, incident_boot_session_id) DO NOTHING
|
|
398
|
+
`).run(
|
|
399
|
+
stableHostId,
|
|
400
|
+
identity.last_boot_session_id,
|
|
401
|
+
bootSessionId,
|
|
402
|
+
now,
|
|
403
|
+
);
|
|
404
|
+
rawDb.prepare(`
|
|
405
|
+
UPDATE codex_runtime_identity
|
|
406
|
+
SET last_boot_session_id = ?, updated_ts = ?
|
|
407
|
+
WHERE singleton = 1
|
|
408
|
+
`).run(bootSessionId, now);
|
|
409
|
+
if (lease) {
|
|
410
|
+
rawDb.prepare(`
|
|
411
|
+
UPDATE codex_daemon_lease
|
|
412
|
+
SET generation_id = NULL,
|
|
413
|
+
stable_host_id = ?,
|
|
414
|
+
boot_session_id = ?,
|
|
415
|
+
status = 'clear',
|
|
416
|
+
quarantine_reason = NULL,
|
|
417
|
+
updated_ts = ?,
|
|
418
|
+
released_ts = ?
|
|
419
|
+
WHERE singleton = 1
|
|
420
|
+
`).run(stableHostId, bootSessionId, now, now);
|
|
421
|
+
}
|
|
422
|
+
return {
|
|
423
|
+
status: 'clear',
|
|
424
|
+
reason: null,
|
|
425
|
+
containmentReleased,
|
|
426
|
+
...publicRecoveryAttempts(listRecoveryAttempts(rawDb)),
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
if (lease?.status === 'quarantined') {
|
|
431
|
+
return {
|
|
432
|
+
status: 'quarantined',
|
|
433
|
+
reason: 'persisted-containment',
|
|
434
|
+
containmentReleased: false,
|
|
435
|
+
...publicRecoveryAttempts(attempts),
|
|
436
|
+
};
|
|
437
|
+
}
|
|
438
|
+
if (lease?.status === 'active') {
|
|
439
|
+
return {
|
|
440
|
+
status: 'quarantined',
|
|
441
|
+
reason: 'persisted-active-generation',
|
|
442
|
+
containmentReleased: false,
|
|
443
|
+
...publicRecoveryAttempts(attempts),
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
if (attempts.blockingAttemptIds.length > 0) {
|
|
447
|
+
const generation = rawDb.prepare(`
|
|
448
|
+
SELECT generation_id
|
|
449
|
+
FROM codex_turn_attempts
|
|
450
|
+
WHERE attempt_id = ?
|
|
451
|
+
`).get(attempts.blockingAttemptIds[0]);
|
|
452
|
+
rawDb.prepare(`
|
|
453
|
+
INSERT INTO codex_daemon_lease (
|
|
454
|
+
singleton, generation_id, stable_host_id, boot_session_id,
|
|
455
|
+
status, quarantine_reason, updated_ts
|
|
456
|
+
) VALUES (1, ?, ?, ?, 'quarantined', 'unresolved-codex-work', ?)
|
|
457
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
458
|
+
generation_id = excluded.generation_id,
|
|
459
|
+
status = 'quarantined',
|
|
460
|
+
quarantine_reason = 'unresolved-codex-work',
|
|
461
|
+
updated_ts = excluded.updated_ts
|
|
462
|
+
`).run(
|
|
463
|
+
generation?.generation_id ?? null,
|
|
464
|
+
stableHostId,
|
|
465
|
+
bootSessionId,
|
|
466
|
+
now,
|
|
467
|
+
);
|
|
468
|
+
return {
|
|
469
|
+
status: 'quarantined',
|
|
470
|
+
reason: 'unresolved-codex-work',
|
|
471
|
+
containmentReleased: false,
|
|
472
|
+
...publicRecoveryAttempts(attempts),
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
return {
|
|
476
|
+
status: 'clear',
|
|
477
|
+
reason: null,
|
|
478
|
+
containmentReleased: false,
|
|
479
|
+
...publicRecoveryAttempts(attempts),
|
|
480
|
+
};
|
|
481
|
+
})();
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
module.exports = {
|
|
485
|
+
listUnresolvedCodexAttempts,
|
|
486
|
+
reconcileCodexAttempt,
|
|
487
|
+
reconstructCodexRecovery,
|
|
488
|
+
codexError,
|
|
489
|
+
requiredString,
|
|
490
|
+
};
|