polygram 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +5 -0
- package/config.example.json +2 -0
- package/lib/codex/diagnostics.js +1 -0
- package/lib/codex/runtime-controller.js +81 -20
- package/lib/codex/runtime-profile.js +6 -0
- package/lib/codex/spawn-context.js +13 -4
- package/lib/db/sessions.js +35 -11
- package/lib/db.js +233 -9
- package/lib/handlers/config-callback.js +361 -5
- package/lib/handlers/config-ui.js +74 -16
- package/lib/handlers/dispatcher.js +84 -13
- package/lib/ops/clean-resume.js +358 -0
- package/lib/ops/clean-shutdown.js +84 -0
- package/lib/ops/shutdown-disposition.js +13 -4
- package/lib/process/channels-tool-dispatcher.js +12 -3
- package/lib/runtime-config.js +46 -7
- package/lib/sdk/callbacks.js +2 -0
- package/lib/telegram/api.js +71 -3
- package/lib/telegram/delivery-barrier.js +117 -0
- package/lib/telegram/process-agent-reply.js +18 -2
- package/lib/telegram/rich-edit.js +16 -5
- package/lib/telegram/rich-media.js +18 -2
- package/migrations/018-clean-restart-resume.sql +24 -0
- package/package.json +2 -2
- package/polygram.js +562 -91
|
@@ -76,6 +76,49 @@ function createDispatcher({
|
|
|
76
76
|
} = {}) {
|
|
77
77
|
// Per-session in-flight handler count.
|
|
78
78
|
const inFlightHandlers = new Map();
|
|
79
|
+
const activeHandlers = new Set();
|
|
80
|
+
|
|
81
|
+
function trackTask(task) {
|
|
82
|
+
activeHandlers.add(task);
|
|
83
|
+
task.then(
|
|
84
|
+
() => activeHandlers.delete(task),
|
|
85
|
+
() => activeHandlers.delete(task),
|
|
86
|
+
);
|
|
87
|
+
return task;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async function awaitSettlement({ timeoutMs = 30_000 } = {}) {
|
|
91
|
+
const deadline = Date.now() + timeoutMs;
|
|
92
|
+
while (activeHandlers.size > 0) {
|
|
93
|
+
const remainingMs = deadline - Date.now();
|
|
94
|
+
if (remainingMs <= 0) {
|
|
95
|
+
const error = new Error(
|
|
96
|
+
`handler settlement timed out with ${activeHandlers.size} task(s) still active`,
|
|
97
|
+
);
|
|
98
|
+
error.code = 'HANDLER_SETTLEMENT_TIMEOUT';
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
101
|
+
let timeout;
|
|
102
|
+
const timedOut = new Promise((_, reject) => {
|
|
103
|
+
timeout = setTimeout(() => {
|
|
104
|
+
const error = new Error(
|
|
105
|
+
`handler settlement timed out with ${activeHandlers.size} task(s) still active`,
|
|
106
|
+
);
|
|
107
|
+
error.code = 'HANDLER_SETTLEMENT_TIMEOUT';
|
|
108
|
+
reject(error);
|
|
109
|
+
}, remainingMs);
|
|
110
|
+
timeout.unref?.();
|
|
111
|
+
});
|
|
112
|
+
try {
|
|
113
|
+
await Promise.race([
|
|
114
|
+
Promise.allSettled([...activeHandlers]),
|
|
115
|
+
timedOut,
|
|
116
|
+
]);
|
|
117
|
+
} finally {
|
|
118
|
+
clearTimeout(timeout);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
79
122
|
|
|
80
123
|
function queueWarnThreshold() {
|
|
81
124
|
const v = Number(config.bot?.queueWarnThreshold);
|
|
@@ -111,7 +154,12 @@ function createDispatcher({
|
|
|
111
154
|
: '🔁 Auto-resuming after timeout — continuing where the previous turn left off.',
|
|
112
155
|
reply_parameters: { message_id: originalMsg.message_id },
|
|
113
156
|
...(threadId && { message_thread_id: threadId }),
|
|
114
|
-
}, {
|
|
157
|
+
}, {
|
|
158
|
+
source: 'auto-resume-indicator',
|
|
159
|
+
botName,
|
|
160
|
+
sessionKey,
|
|
161
|
+
sourceMsgId: originalMsg.message_id,
|
|
162
|
+
}).catch((sendErr) => {
|
|
115
163
|
logger.error?.(`[${sessionKey}] auto-resume indicator send failed: ${sendErr.message}`);
|
|
116
164
|
});
|
|
117
165
|
|
|
@@ -204,7 +252,12 @@ function createDispatcher({
|
|
|
204
252
|
threadId,
|
|
205
253
|
chunks,
|
|
206
254
|
replyToMessageId: originalMsg.message_id,
|
|
207
|
-
meta: {
|
|
255
|
+
meta: {
|
|
256
|
+
source: 'auto-resume-reply',
|
|
257
|
+
botName,
|
|
258
|
+
sessionKey,
|
|
259
|
+
sourceMsgId: originalMsg.message_id,
|
|
260
|
+
},
|
|
208
261
|
logger: { error: (m) => logger.error?.(`[${sessionKey}] auto-resume deliver: ${m}`) },
|
|
209
262
|
});
|
|
210
263
|
|
|
@@ -221,7 +274,7 @@ function createDispatcher({
|
|
|
221
274
|
in_flight: count, threshold: warnAt,
|
|
222
275
|
});
|
|
223
276
|
}
|
|
224
|
-
handleMessage(sessionKey, chatId, msg, bot).catch((err) => {
|
|
277
|
+
const task = handleMessage(sessionKey, chatId, msg, bot).catch(async (err) => {
|
|
225
278
|
const wasAborted = abortGrace.isRecent(sessionKey);
|
|
226
279
|
const isReplay = msg._isReplay === true;
|
|
227
280
|
let oomShutdown = false;
|
|
@@ -349,7 +402,12 @@ function createDispatcher({
|
|
|
349
402
|
chat_id: chatId,
|
|
350
403
|
text: '⚠️ This turn was interrupted and didn\'t complete on retry — please rephrase or simplify, or split into smaller steps.',
|
|
351
404
|
reply_parameters: { message_id: msg.message_id },
|
|
352
|
-
}, {
|
|
405
|
+
}, {
|
|
406
|
+
source: 'error-reply',
|
|
407
|
+
botName,
|
|
408
|
+
sessionKey,
|
|
409
|
+
sourceMsgId: msg.message_id,
|
|
410
|
+
}).catch((replyErr) => {
|
|
353
411
|
logger.error?.(`[${sessionKey}] failed to send replay-failure reply: ${replyErr.message}`);
|
|
354
412
|
});
|
|
355
413
|
}
|
|
@@ -404,11 +462,12 @@ function createDispatcher({
|
|
|
404
462
|
// best-effort timer so a pending retry can't pin the daemon alive
|
|
405
463
|
// (the Telegram long-poll already keeps the loop running).
|
|
406
464
|
const retryMsg = { ...msg, _startupRetried: true };
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
)
|
|
411
|
-
return;
|
|
465
|
+
await new Promise((resolve) => {
|
|
466
|
+
const timer = setTimeout(resolve, startupRetryDelayMs);
|
|
467
|
+
timer.unref?.();
|
|
468
|
+
});
|
|
469
|
+
if (getIsShuttingDown()) return;
|
|
470
|
+
return dispatchHandleMessage(sessionKey, chatId, retryMsg, bot);
|
|
412
471
|
}
|
|
413
472
|
// rc.54: auto-resume on 300s no-activity timeout. The
|
|
414
473
|
// resume turn itself runs through sendToProcess directly
|
|
@@ -427,7 +486,7 @@ function createDispatcher({
|
|
|
427
486
|
chat_id: chatId, session_key: sessionKey, msg_id: msg.message_id,
|
|
428
487
|
original_error: err.message?.slice(0, 200),
|
|
429
488
|
});
|
|
430
|
-
attemptAutoResume(sessionKey, chatId, msg, bot, {
|
|
489
|
+
return attemptAutoResume(sessionKey, chatId, msg, bot, {
|
|
431
490
|
provider,
|
|
432
491
|
providerRecovery,
|
|
433
492
|
})
|
|
@@ -471,10 +530,14 @@ function createDispatcher({
|
|
|
471
530
|
tg(bot, 'sendMessage', {
|
|
472
531
|
chat_id: chatId, text: fallbackText,
|
|
473
532
|
reply_parameters: { message_id: msg.message_id },
|
|
474
|
-
}, {
|
|
533
|
+
}, {
|
|
534
|
+
source: 'error-reply',
|
|
535
|
+
botName,
|
|
536
|
+
sessionKey,
|
|
537
|
+
sourceMsgId: msg.message_id,
|
|
538
|
+
}).catch(() => {});
|
|
475
539
|
}
|
|
476
540
|
});
|
|
477
|
-
return;
|
|
478
541
|
}
|
|
479
542
|
// 0.7.7: errorReplyText may return null (suppress reply
|
|
480
543
|
// signal — INTERRUPTED inside abort grace).
|
|
@@ -486,7 +549,12 @@ function createDispatcher({
|
|
|
486
549
|
chat_id: chatId,
|
|
487
550
|
text: replyText,
|
|
488
551
|
reply_parameters: { message_id: msg.message_id },
|
|
489
|
-
}, {
|
|
552
|
+
}, {
|
|
553
|
+
source: 'error-reply',
|
|
554
|
+
botName,
|
|
555
|
+
sessionKey,
|
|
556
|
+
sourceMsgId: msg.message_id,
|
|
557
|
+
}).catch((replyErr) => {
|
|
490
558
|
logger.error?.(`[${sessionKey}] failed to send error reply: ${replyErr.message}`);
|
|
491
559
|
});
|
|
492
560
|
}
|
|
@@ -496,6 +564,7 @@ function createDispatcher({
|
|
|
496
564
|
if (n <= 0) inFlightHandlers.delete(sessionKey);
|
|
497
565
|
else inFlightHandlers.set(sessionKey, n);
|
|
498
566
|
});
|
|
567
|
+
return trackTask(task);
|
|
499
568
|
}
|
|
500
569
|
|
|
501
570
|
return {
|
|
@@ -504,6 +573,8 @@ function createDispatcher({
|
|
|
504
573
|
errorReplyText,
|
|
505
574
|
queueWarnThreshold,
|
|
506
575
|
inFlightHandlers, // exposed so polygram.js can introspect for shutdown drain
|
|
576
|
+
awaitSettlement,
|
|
577
|
+
trackTask,
|
|
507
578
|
};
|
|
508
579
|
}
|
|
509
580
|
|
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const EXPECTED_NAMESPACE = 'claude:channels';
|
|
4
|
+
const CLEAN_RESUME_POLICY_VERSION = 1;
|
|
5
|
+
|
|
6
|
+
function recoveryNoticeText({ ambiguous = false } = {}) {
|
|
7
|
+
if (ambiguous) {
|
|
8
|
+
return '↺ Restart recovery could not confirm whether the answer was delivered. I will not retry it automatically; send the message again if you still need a reply.';
|
|
9
|
+
}
|
|
10
|
+
return '↺ Restarted — I couldn\'t safely resume this message. If it still needs a reply, send it again.';
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function attestsExactResume(attestation, expectedSessionId) {
|
|
14
|
+
return attestation?.namespace === EXPECTED_NAMESPACE
|
|
15
|
+
&& attestation.sessionId === expectedSessionId
|
|
16
|
+
&& attestation.resumed === true
|
|
17
|
+
&& attestation.freshFallback === false;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function validateStrictResumeSpawn({
|
|
21
|
+
strictResume,
|
|
22
|
+
promptBackend,
|
|
23
|
+
liveProcess,
|
|
24
|
+
runtime,
|
|
25
|
+
resolved,
|
|
26
|
+
} = {}) {
|
|
27
|
+
if (promptBackend === 'codex') {
|
|
28
|
+
const error = new Error('strict Claude resume cannot cross a backend change');
|
|
29
|
+
error.code = 'CLEAN_RESUME_CONFIG_DRIFT';
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
if (liveProcess && !liveProcess.closed) {
|
|
33
|
+
const error = new Error('strict clean resume requires an unspawned session');
|
|
34
|
+
error.code = 'CLEAN_RESUME_SESSION_ALREADY_SPAWNED';
|
|
35
|
+
throw error;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const driftFields = ['agent', 'cwd'].filter(
|
|
39
|
+
(field) => (runtime?.[field] || null) !== (resolved?.[field] || null),
|
|
40
|
+
);
|
|
41
|
+
if (
|
|
42
|
+
!runtime
|
|
43
|
+
|| runtime.generation_id !== strictResume?.expectedGenerationId
|
|
44
|
+
|| runtime.provider_session_id !== strictResume?.expectedSessionId
|
|
45
|
+
|| !['cli', 'channels'].includes(runtime.pm_backend)
|
|
46
|
+
|| !['cli', 'channels'].includes(resolved?.backend)
|
|
47
|
+
|| driftFields.length > 0
|
|
48
|
+
) {
|
|
49
|
+
const error = new Error('strict clean resume session identity or config changed');
|
|
50
|
+
error.code = driftFields.length > 0
|
|
51
|
+
? 'CLEAN_RESUME_CONFIG_DRIFT'
|
|
52
|
+
: 'CLEAN_RESUME_SESSION_CHANGED';
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
return strictResume.expectedSessionId;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function createCleanResumeTurnContext({ sourceMsgId, threadId = null } = {}) {
|
|
59
|
+
return {
|
|
60
|
+
streamer: {
|
|
61
|
+
onChunk: async () => {},
|
|
62
|
+
forceNewMessage: () => {},
|
|
63
|
+
finalize: async () => ({ streamed: false }),
|
|
64
|
+
flushDraft: async () => {},
|
|
65
|
+
discard: async () => {},
|
|
66
|
+
},
|
|
67
|
+
reactor: {
|
|
68
|
+
setState: () => {},
|
|
69
|
+
heartbeat: () => {},
|
|
70
|
+
clear: async () => {},
|
|
71
|
+
stop: () => {},
|
|
72
|
+
},
|
|
73
|
+
sourceMsgId,
|
|
74
|
+
threadId,
|
|
75
|
+
onFirstStream: () => {},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function executeCleanResumeClaim(claim, {
|
|
80
|
+
enabled = false,
|
|
81
|
+
loadSource,
|
|
82
|
+
sessionKeyForSource,
|
|
83
|
+
loadRuntimeSession,
|
|
84
|
+
resolveStrictSpawnContext,
|
|
85
|
+
strictSpawn,
|
|
86
|
+
discardSpawn,
|
|
87
|
+
sendContinue,
|
|
88
|
+
deliverResult,
|
|
89
|
+
sendNotice,
|
|
90
|
+
complete,
|
|
91
|
+
logEvent = () => {},
|
|
92
|
+
onReady = () => {},
|
|
93
|
+
} = {}) {
|
|
94
|
+
let ready = false;
|
|
95
|
+
const markReady = () => {
|
|
96
|
+
if (ready) return;
|
|
97
|
+
ready = true;
|
|
98
|
+
onReady();
|
|
99
|
+
};
|
|
100
|
+
const eventDetail = {
|
|
101
|
+
bot: claim?.bot_name ?? null,
|
|
102
|
+
session_key: claim?.session_key ?? null,
|
|
103
|
+
source_message_id: claim?.source_message_id ?? null,
|
|
104
|
+
policy_version: claim?.policy_version ?? null,
|
|
105
|
+
};
|
|
106
|
+
const source = typeof loadSource === 'function'
|
|
107
|
+
? loadSource(claim.source_message_id)
|
|
108
|
+
: null;
|
|
109
|
+
|
|
110
|
+
async function fallback(reason, { ambiguous = false } = {}) {
|
|
111
|
+
markReady();
|
|
112
|
+
const text = recoveryNoticeText({ ambiguous });
|
|
113
|
+
await sendNotice({
|
|
114
|
+
claim,
|
|
115
|
+
source,
|
|
116
|
+
text,
|
|
117
|
+
ambiguous,
|
|
118
|
+
reason,
|
|
119
|
+
});
|
|
120
|
+
await complete({
|
|
121
|
+
sourceMessageId: claim.source_message_id,
|
|
122
|
+
status: 'replay-skipped',
|
|
123
|
+
});
|
|
124
|
+
logEvent('clean-resume-fallback', {
|
|
125
|
+
...eventDetail,
|
|
126
|
+
reason,
|
|
127
|
+
ambiguous: ambiguous || undefined,
|
|
128
|
+
});
|
|
129
|
+
return { status: 'replay-skipped', reason };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function rejected(reason) {
|
|
133
|
+
logEvent('clean-resume-rejected', {
|
|
134
|
+
...eventDetail,
|
|
135
|
+
reason,
|
|
136
|
+
});
|
|
137
|
+
return fallback(reason);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function failed(reason, { ambiguous = false } = {}) {
|
|
141
|
+
logEvent('clean-resume-failed', {
|
|
142
|
+
...eventDetail,
|
|
143
|
+
reason,
|
|
144
|
+
ambiguous: ambiguous || undefined,
|
|
145
|
+
});
|
|
146
|
+
return fallback(reason, { ambiguous });
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (!enabled) return fallback('rollout-disabled');
|
|
150
|
+
if (claim?.executable !== true) {
|
|
151
|
+
return fallback(claim?.reason || 'claim-not-executable');
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
if (
|
|
155
|
+
!source
|
|
156
|
+
|| source.id !== claim.source_message_id
|
|
157
|
+
|| source.bot_name !== claim.bot_name
|
|
158
|
+
|| source.handler_status !== 'resume-attempted'
|
|
159
|
+
|| sessionKeyForSource(source) !== claim.session_key
|
|
160
|
+
) {
|
|
161
|
+
return fallback('source-message-mismatch');
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const runtime = loadRuntimeSession(claim.session_key);
|
|
165
|
+
if (
|
|
166
|
+
!runtime
|
|
167
|
+
|| runtime.namespace !== EXPECTED_NAMESPACE
|
|
168
|
+
|| runtime.generation_id !== claim.session_generation_id
|
|
169
|
+
|| typeof runtime.provider_session_id !== 'string'
|
|
170
|
+
|| runtime.provider_session_id.length === 0
|
|
171
|
+
|| !['cli', 'channels'].includes(runtime.pm_backend)
|
|
172
|
+
) {
|
|
173
|
+
return rejected(
|
|
174
|
+
runtime?.generation_id !== claim.session_generation_id
|
|
175
|
+
? 'session-generation-replaced'
|
|
176
|
+
: 'session-runtime-incompatible',
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
let resolved;
|
|
181
|
+
try {
|
|
182
|
+
resolved = await resolveStrictSpawnContext({
|
|
183
|
+
claim,
|
|
184
|
+
source,
|
|
185
|
+
runtime,
|
|
186
|
+
});
|
|
187
|
+
} catch (error) {
|
|
188
|
+
return rejected(error?.code || 'session-config-drift');
|
|
189
|
+
}
|
|
190
|
+
if (!resolved?.ok) {
|
|
191
|
+
return rejected(resolved?.reason || 'session-config-drift');
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
let spawned;
|
|
195
|
+
try {
|
|
196
|
+
spawned = await strictSpawn({
|
|
197
|
+
sessionKey: claim.session_key,
|
|
198
|
+
context: resolved.context,
|
|
199
|
+
});
|
|
200
|
+
} catch (error) {
|
|
201
|
+
return rejected(error?.code || 'strict-resume-failed');
|
|
202
|
+
}
|
|
203
|
+
if (
|
|
204
|
+
!spawned?.process
|
|
205
|
+
|| !attestsExactResume(spawned.attestation, runtime.provider_session_id)
|
|
206
|
+
) {
|
|
207
|
+
if (spawned?.process) {
|
|
208
|
+
if (typeof discardSpawn !== 'function') {
|
|
209
|
+
throw new TypeError('clean resume rejected spawn requires exact-process cleanup');
|
|
210
|
+
}
|
|
211
|
+
await discardSpawn({
|
|
212
|
+
sessionKey: claim.session_key,
|
|
213
|
+
process: spawned.process,
|
|
214
|
+
reason: 'resume-attestation-mismatch',
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
return rejected('resume-attestation-mismatch');
|
|
218
|
+
}
|
|
219
|
+
logEvent('clean-resume-attested', eventDetail);
|
|
220
|
+
|
|
221
|
+
let result;
|
|
222
|
+
try {
|
|
223
|
+
result = await sendContinue({
|
|
224
|
+
sessionKey: claim.session_key,
|
|
225
|
+
text: 'continue',
|
|
226
|
+
sourceMsgId: source.msg_id,
|
|
227
|
+
expectedProcess: spawned.process,
|
|
228
|
+
onDispatched: markReady,
|
|
229
|
+
});
|
|
230
|
+
markReady();
|
|
231
|
+
} catch (error) {
|
|
232
|
+
markReady();
|
|
233
|
+
const ambiguous = error?.deliveryAmbiguous === true;
|
|
234
|
+
return failed(
|
|
235
|
+
ambiguous ? 'continuation-delivery-ambiguous' : 'continuation-failed',
|
|
236
|
+
{ ambiguous },
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (result?.deliveryAmbiguous === true) {
|
|
241
|
+
return failed('continuation-delivery-ambiguous', { ambiguous: true });
|
|
242
|
+
}
|
|
243
|
+
if (result?.error) {
|
|
244
|
+
return failed('continuation-failed');
|
|
245
|
+
}
|
|
246
|
+
if (!result?.alreadyDelivered) {
|
|
247
|
+
if (typeof result?.text !== 'string' || result.text.trim().length === 0) {
|
|
248
|
+
return failed('continuation-empty');
|
|
249
|
+
}
|
|
250
|
+
try {
|
|
251
|
+
const delivered = await deliverResult({
|
|
252
|
+
claim,
|
|
253
|
+
source,
|
|
254
|
+
text: result.text,
|
|
255
|
+
result,
|
|
256
|
+
});
|
|
257
|
+
if (delivered?.ok !== true) {
|
|
258
|
+
return failed(
|
|
259
|
+
delivered?.ambiguous
|
|
260
|
+
? 'continuation-delivery-ambiguous'
|
|
261
|
+
: 'continuation-delivery-failed',
|
|
262
|
+
{ ambiguous: delivered?.ambiguous === true },
|
|
263
|
+
);
|
|
264
|
+
}
|
|
265
|
+
} catch (error) {
|
|
266
|
+
const ambiguous = error?.deliveryAmbiguous === true;
|
|
267
|
+
return failed(
|
|
268
|
+
ambiguous ? 'continuation-delivery-ambiguous' : 'continuation-delivery-failed',
|
|
269
|
+
{ ambiguous },
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
await complete({
|
|
275
|
+
sourceMessageId: claim.source_message_id,
|
|
276
|
+
status: 'replied',
|
|
277
|
+
});
|
|
278
|
+
logEvent('clean-resume-succeeded', eventDetail);
|
|
279
|
+
return { status: 'replied', reason: null };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
async function startCleanRestartRecovery({
|
|
283
|
+
db,
|
|
284
|
+
botName,
|
|
285
|
+
maxAgeMs,
|
|
286
|
+
olderThanMs,
|
|
287
|
+
supportedPolicyVersions = [CLEAN_RESUME_POLICY_VERSION],
|
|
288
|
+
sessionKeyForSource,
|
|
289
|
+
executeClaim,
|
|
290
|
+
trackTask = (task) => task,
|
|
291
|
+
onClaim = () => {},
|
|
292
|
+
onTaskError = () => {},
|
|
293
|
+
} = {}) {
|
|
294
|
+
const recovery = db.claimCleanRestartRecovery({
|
|
295
|
+
botName,
|
|
296
|
+
maxAgeMs,
|
|
297
|
+
olderThanMs,
|
|
298
|
+
supportedPolicyVersions,
|
|
299
|
+
});
|
|
300
|
+
const claims = [
|
|
301
|
+
...(recovery.claims || []),
|
|
302
|
+
...(recovery.stranded || []).map((row) => ({
|
|
303
|
+
bot_name: botName,
|
|
304
|
+
session_key: sessionKeyForSource(row),
|
|
305
|
+
session_generation_id: null,
|
|
306
|
+
source_message_id: row.id,
|
|
307
|
+
policy_version: CLEAN_RESUME_POLICY_VERSION,
|
|
308
|
+
executable: false,
|
|
309
|
+
reason: 'stranded-resume-attempt',
|
|
310
|
+
})),
|
|
311
|
+
];
|
|
312
|
+
for (let index = 0; index < claims.length; index += 1) {
|
|
313
|
+
onClaim(index < (recovery.claims || []).length ? 'claimed' : 'stranded', claims[index]);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
const launches = claims.map((claim) => {
|
|
317
|
+
let releaseReady;
|
|
318
|
+
let rejectReady;
|
|
319
|
+
let ready = false;
|
|
320
|
+
const readiness = new Promise((resolve, reject) => {
|
|
321
|
+
rejectReady = reject;
|
|
322
|
+
releaseReady = () => {
|
|
323
|
+
if (ready) return;
|
|
324
|
+
ready = true;
|
|
325
|
+
resolve();
|
|
326
|
+
};
|
|
327
|
+
});
|
|
328
|
+
const task = Promise.resolve()
|
|
329
|
+
.then(() => executeClaim(claim, { onReady: releaseReady }))
|
|
330
|
+
.catch((error) => {
|
|
331
|
+
try { onTaskError(error, claim); } catch {}
|
|
332
|
+
if (!ready) {
|
|
333
|
+
ready = true;
|
|
334
|
+
rejectReady(error);
|
|
335
|
+
}
|
|
336
|
+
})
|
|
337
|
+
.finally(releaseReady);
|
|
338
|
+
trackTask(task);
|
|
339
|
+
return { task, readiness };
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
await Promise.all(launches.map(({ readiness }) => readiness));
|
|
343
|
+
return {
|
|
344
|
+
clean: recovery.clean,
|
|
345
|
+
claims,
|
|
346
|
+
tasks: launches.map(({ task }) => task),
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
module.exports = {
|
|
351
|
+
CLEAN_RESUME_POLICY_VERSION,
|
|
352
|
+
executeCleanResumeClaim,
|
|
353
|
+
recoveryNoticeText,
|
|
354
|
+
attestsExactResume,
|
|
355
|
+
createCleanResumeTurnContext,
|
|
356
|
+
startCleanRestartRecovery,
|
|
357
|
+
validateStrictResumeSpawn,
|
|
358
|
+
};
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
async function prepareCleanRetirement({
|
|
4
|
+
pm,
|
|
5
|
+
deliveryBarrier,
|
|
6
|
+
awaitHandlerSettlement,
|
|
7
|
+
settlementTimeoutMs = 30_000,
|
|
8
|
+
} = {}) {
|
|
9
|
+
if (typeof deliveryBarrier?.fenceAndDrain !== 'function'
|
|
10
|
+
|| typeof deliveryBarrier?.inspect !== 'function') {
|
|
11
|
+
throw new TypeError('clean shutdown requires a delivery barrier');
|
|
12
|
+
}
|
|
13
|
+
if (typeof pm?.retireForCleanRestart !== 'function') {
|
|
14
|
+
throw new TypeError('ProcessManager.retireForCleanRestart is required');
|
|
15
|
+
}
|
|
16
|
+
if (typeof awaitHandlerSettlement !== 'function') {
|
|
17
|
+
throw new TypeError('clean shutdown requires handler settlement');
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const deliveryDrain = deliveryBarrier.fenceAndDrain();
|
|
21
|
+
const processRetirement = pm.retireForCleanRestart({
|
|
22
|
+
getDeliveryEvidence: (sessionKey, sourceMsgId) => (
|
|
23
|
+
deliveryBarrier.inspect(sessionKey, sourceMsgId)
|
|
24
|
+
),
|
|
25
|
+
});
|
|
26
|
+
const [, snapshots] = await Promise.all([
|
|
27
|
+
deliveryDrain,
|
|
28
|
+
processRetirement,
|
|
29
|
+
]);
|
|
30
|
+
await awaitHandlerSettlement({ timeoutMs: settlementTimeoutMs });
|
|
31
|
+
return { snapshots: Array.isArray(snapshots) ? snapshots : [] };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function buildResumeIntents({
|
|
35
|
+
snapshots = [],
|
|
36
|
+
resolveSourceMessageId,
|
|
37
|
+
policyVersion = 1,
|
|
38
|
+
} = {}) {
|
|
39
|
+
if (typeof resolveSourceMessageId !== 'function') {
|
|
40
|
+
throw new TypeError('retirement source resolver is required');
|
|
41
|
+
}
|
|
42
|
+
const seenSessions = new Set();
|
|
43
|
+
const seenSources = new Set();
|
|
44
|
+
const resumeIntents = [];
|
|
45
|
+
const classified = snapshots.map((snapshot) => {
|
|
46
|
+
const result = {
|
|
47
|
+
...snapshot,
|
|
48
|
+
eligible: snapshot?.eligible === true,
|
|
49
|
+
reason: snapshot?.reason ?? null,
|
|
50
|
+
};
|
|
51
|
+
if (!result.eligible) return result;
|
|
52
|
+
if (seenSessions.has(result.sessionKey)) {
|
|
53
|
+
throw new Error(`duplicate retirement snapshot for ${result.sessionKey}`);
|
|
54
|
+
}
|
|
55
|
+
seenSessions.add(result.sessionKey);
|
|
56
|
+
const sourceMessageId = resolveSourceMessageId(result);
|
|
57
|
+
if (!Number.isSafeInteger(sourceMessageId) || sourceMessageId <= 0) {
|
|
58
|
+
return {
|
|
59
|
+
...result,
|
|
60
|
+
eligible: false,
|
|
61
|
+
reason: 'source-message-missing',
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
if (seenSources.has(sourceMessageId)) {
|
|
65
|
+
throw new Error(`duplicate retirement source ${sourceMessageId}`);
|
|
66
|
+
}
|
|
67
|
+
seenSources.add(sourceMessageId);
|
|
68
|
+
resumeIntents.push({
|
|
69
|
+
sessionKey: result.sessionKey,
|
|
70
|
+
sourceMessageId,
|
|
71
|
+
policyVersion,
|
|
72
|
+
});
|
|
73
|
+
return {
|
|
74
|
+
...result,
|
|
75
|
+
sourceMessageId,
|
|
76
|
+
};
|
|
77
|
+
});
|
|
78
|
+
return { resumeIntents, snapshots: classified };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = {
|
|
82
|
+
prepareCleanRetirement,
|
|
83
|
+
buildResumeIntents,
|
|
84
|
+
};
|
|
@@ -20,15 +20,24 @@ function persistShutdownDisposition({
|
|
|
20
20
|
now = Date.now(),
|
|
21
21
|
since,
|
|
22
22
|
observation,
|
|
23
|
+
resumeIntents = [],
|
|
24
|
+
forceCrashReason = null,
|
|
23
25
|
} = {}) {
|
|
24
26
|
const oomDetected = observation?.detected === true;
|
|
25
|
-
const
|
|
27
|
+
const crashLike = oomDetected || forceCrashReason != null;
|
|
28
|
+
const result = crashLike
|
|
26
29
|
? db.recordCrashShutdown({ botName, now, since })
|
|
27
|
-
: db.recordCleanShutdown({
|
|
30
|
+
: db.recordCleanShutdown({
|
|
31
|
+
botName,
|
|
32
|
+
now,
|
|
33
|
+
since,
|
|
34
|
+
resumeIntents,
|
|
35
|
+
});
|
|
28
36
|
return {
|
|
29
|
-
clean: !
|
|
30
|
-
shutdownReason: shutdownReasonFor(observation),
|
|
37
|
+
clean: !crashLike,
|
|
38
|
+
shutdownReason: forceCrashReason || shutdownReasonFor(observation),
|
|
31
39
|
replayMarked: result?.replayMarked ?? 0,
|
|
40
|
+
...(!crashLike ? { intentsRecorded: result?.intentsRecorded ?? 0 } : {}),
|
|
32
41
|
...(oomDetected ? { oomKillDelta: String(observation.delta) } : {}),
|
|
33
42
|
};
|
|
34
43
|
}
|
|
@@ -282,7 +282,12 @@ function createChannelsToolDispatcher({
|
|
|
282
282
|
} else {
|
|
283
283
|
const params = { chat_id: chatId, message_id: messageId, text: body };
|
|
284
284
|
if (threadId) params.message_thread_id = threadId;
|
|
285
|
-
await send(bot, 'editMessageText', params, {
|
|
285
|
+
await send(bot, 'editMessageText', params, {
|
|
286
|
+
source: 'channels-tool-dispatcher',
|
|
287
|
+
sessionKey,
|
|
288
|
+
sourceMsgId,
|
|
289
|
+
toolName,
|
|
290
|
+
});
|
|
286
291
|
}
|
|
287
292
|
rememberOwned(sessionKey, messageId); // keep ownership across re-edits
|
|
288
293
|
// The bubble now says something the stored row does not. Persisting is
|
|
@@ -388,7 +393,7 @@ function createChannelsToolDispatcher({
|
|
|
388
393
|
replyToMessageId: sourceMsgId || null,
|
|
389
394
|
applyReactions: sourceMsgId != null,
|
|
390
395
|
source: 'channels-tool-dispatcher',
|
|
391
|
-
meta: { sessionKey, toolName },
|
|
396
|
+
meta: { sessionKey, sourceMsgId, toolName },
|
|
392
397
|
parseResponse,
|
|
393
398
|
sanitizeAssistantReply,
|
|
394
399
|
chunkMarkdownText: chunkText,
|
|
@@ -481,7 +486,11 @@ function createChannelsToolDispatcher({
|
|
|
481
486
|
[fieldName]: { source: check.resolved, filename: path.basename(check.resolved) },
|
|
482
487
|
};
|
|
483
488
|
if (threadId) params.message_thread_id = threadId;
|
|
484
|
-
await send(bot, method, params, {
|
|
489
|
+
await send(bot, method, params, {
|
|
490
|
+
source: 'channels-tool-dispatcher',
|
|
491
|
+
sessionKey,
|
|
492
|
+
sourceMsgId,
|
|
493
|
+
});
|
|
485
494
|
} catch (err) {
|
|
486
495
|
logger.warn?.(`[channels-tool-dispatcher] file attach failed for ${check.resolved}: ${err.message}`);
|
|
487
496
|
failedAttachments.push({ path: filePath, error: err.message });
|