tokenmaxxing 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DESIGN.md +2 -30
- package/LICENSE +21 -0
- package/README.md +1 -2
- package/agent-plugin/bin/tokenmaxxing-mcp +7 -0
- package/agent-plugin/mcp.json +10 -0
- package/agent-plugin/plugin.json +14 -0
- package/agent-plugin/skills/codex-pool/SKILL.md +23 -0
- package/agent-plugin/skills/codex-pool/references/codex.md +5 -0
- package/agent-plugin/skills/credentials-hygiene/SKILL.md +26 -0
- package/agent-plugin/skills/credentials-hygiene/references/credentials.md +6 -0
- package/agent-plugin/skills/doctor-diagnostics/SKILL.md +26 -0
- package/agent-plugin/skills/doctor-diagnostics/references/troubleshooting.md +5 -0
- package/agent-plugin/skills/pool-status/SKILL.md +27 -0
- package/agent-plugin/skills/pool-status/references/commands.md +8 -0
- package/agent-plugin/skills/safe-contribution/SKILL.md +27 -0
- package/agent-plugin/skills/safe-contribution/references/ship.md +5 -0
- package/agent-plugin/skills/sdk-pairing/SKILL.md +33 -0
- package/agent-plugin/skills/sdk-pairing/references/sdk.md +6 -0
- package/agent-plugin/skills/switching-policy/SKILL.md +29 -0
- package/agent-plugin/skills/switching-policy/references/policy.md +7 -0
- package/package.json +3 -5
- package/src/entries/mcp.ts +288 -0
- package/src/lib/decide.ts +2 -4
- package/src/lib/lock.ts +3 -7
- package/src/lib/log.ts +8 -11
- package/src/lib/paths.ts +0 -9
- package/src/lib/usage.ts +6 -5
- package/src/main.ts +1 -6
- package/src/cli/serve.ts +0 -1790
- package/src/lib/slackbridge.ts +0 -1363
- package/src/lib/slackstate.ts +0 -352
- package/src/lib/slackstream.ts +0 -300
- package/src/serve-plugin/.claude-plugin/plugin.json +0 -4
- package/src/serve-plugin/skills/ask-the-user/SKILL.md +0 -41
- package/src/serve-plugin/skills/serve-session/SKILL.md +0 -50
package/src/lib/slackbridge.ts
DELETED
|
@@ -1,1363 +0,0 @@
|
|
|
1
|
-
// The Slack->claude relay. One claude turn per Slack message via the Agent SDK
|
|
2
|
-
// (re-query with resume, never a persistent streaming query: the SDK subprocess
|
|
3
|
-
// reads credentials at spawn, so per-turn spawns are what let the pool decision
|
|
4
|
-
// pick the freshest account at every boundary and let the daemon restart
|
|
5
|
-
// without losing threads). Verified against @anthropic-ai/claude-agent-sdk
|
|
6
|
-
// 0.3.214 and code.claude.com/docs 2026-07-18; both change monthly.
|
|
7
|
-
|
|
8
|
-
import { spawn } from "node:child_process";
|
|
9
|
-
import { join } from "node:path";
|
|
10
|
-
import { z } from "zod";
|
|
11
|
-
import { delay } from "es-toolkit";
|
|
12
|
-
import { createSdkMcpServer, query, tool, type SDKUserMessage, type SpawnOptions } from "@anthropic-ai/claude-agent-sdk";
|
|
13
|
-
import { StreamingMarkdownRenderer, type StreamChunk } from "chat";
|
|
14
|
-
import { ensureBestAccount, pooledOptions, stopHookCheck, type SwapDecision } from "../sdk.ts";
|
|
15
|
-
import { POST_SWAP_COOLDOWN_MS } from "./decide.ts";
|
|
16
|
-
import { readOAuthAccount } from "./claudejson.ts";
|
|
17
|
-
import { http, safeErrorDetail } from "./http.ts";
|
|
18
|
-
import { loadLastSwapAt } from "./state.ts";
|
|
19
|
-
import { fmtResetShort, recordObservedLimit } from "./usage.ts";
|
|
20
|
-
import { deleteSlackThread, type SlackLink } from "./slackstate.ts";
|
|
21
|
-
import { agentEventChunks, newStreamMapState } from "./slackstream.ts";
|
|
22
|
-
import { log } from "./log.ts";
|
|
23
|
-
|
|
24
|
-
/** With systemPrompt omitted the SDK runs a MINIMAL system prompt (the
|
|
25
|
-
* claude_code preset is opt-in since SDK 0.1.0, re-verified for 0.3.214
|
|
26
|
-
* 2026-07-18), so this small standalone prompt replaces nothing. It exists
|
|
27
|
-
* because a relayed model once answered with a literal "<br>": Slack renders
|
|
28
|
-
* markdown, never HTML. */
|
|
29
|
-
const SLACK_SYSTEM_PROMPT =
|
|
30
|
-
"Your replies are relayed into a Slack thread and render as Slack-flavored markdown. Write plain markdown only - never HTML tags such as <br> (use real line breaks).";
|
|
31
|
-
|
|
32
|
-
export const TurnOutcomeSchema = z.object({
|
|
33
|
-
sessionId: z.string().nullable(),
|
|
34
|
-
failed: z.boolean(),
|
|
35
|
-
/** the FAILED turn hit a usage/rate limit (or the pool was depleted before
|
|
36
|
-
* it could spawn). Only an errored result is ever limit-classified. */
|
|
37
|
-
rateLimited: z.boolean(),
|
|
38
|
-
/** the model called finish_thread this turn: garbage-collect after the turn. */
|
|
39
|
-
finish: z.boolean(),
|
|
40
|
-
/** the model called need_attention this turn (it asked the user for a
|
|
41
|
-
* decision): the daemon marks the thread waiting - question-mark status
|
|
42
|
-
* reaction, a one-time nudge if the user stays quiet, and an asked user's
|
|
43
|
-
* reaction relays as their answer. */
|
|
44
|
-
attention: z.boolean(),
|
|
45
|
-
/** relayThread posted a TERMINAL drop notice for this message ("dropped;
|
|
46
|
-
* re-send it"): a drain must NOT presume a killed child and retain the
|
|
47
|
-
* resume marker, or startup replays work the user was told to resend. */
|
|
48
|
-
announcedDrop: z.boolean(),
|
|
49
|
-
/** the LAST attempt's claude child ran to a SUCCESSFUL result: the work is
|
|
50
|
-
* done even if Slack delivery later failed (textLost sets failed for the
|
|
51
|
-
* operator's benefit). A drain must not read that delivery failure as a
|
|
52
|
-
* killed child and re-run completed work (adversarial-review catch). */
|
|
53
|
-
resultReceived: z.boolean(),
|
|
54
|
-
/** epoch ms when the pool is expected usable again: set when the turn ended
|
|
55
|
-
* at a usage limit whose recovery time is known. The caller keeps the
|
|
56
|
-
* thread's activeTurn marker with resumeAt and the daemon resumes the turn
|
|
57
|
-
* itself, instead of the old "re-send it once the pool recovers" drop. */
|
|
58
|
-
deferUntil: z.number().nullable(),
|
|
59
|
-
/** a steered follow-up's own drained turn FAILED after the primary turn
|
|
60
|
-
* already succeeded (success stays sticky, the loss is announced
|
|
61
|
-
* in-thread): the caller settles the steered messages' reactions as
|
|
62
|
-
* failed, never as done - a lost instruction must not read green.
|
|
63
|
-
* Attribution is per-turn, not per-message (steer() carries text only),
|
|
64
|
-
* so when several messages were steered a successfully folded one can
|
|
65
|
-
* read failed too - accepted: a false re-send ask beats a false green. */
|
|
66
|
-
steerLost: z.boolean(),
|
|
67
|
-
});
|
|
68
|
-
export type TurnOutcome = z.infer<typeof TurnOutcomeSchema>;
|
|
69
|
-
|
|
70
|
-
// ---- depleted-pool recovery policy (pure, unit-tested) ---------------------
|
|
71
|
-
|
|
72
|
-
/** TOTAL parking budget for one Slack message (a single deadline across all
|
|
73
|
-
* its parks, not per park): the parked handler holds the thread's queue slot,
|
|
74
|
-
* so recovery further out gets an honest drop notice instead of a hostage
|
|
75
|
-
* handler, and the daemon's queue-entry TTL is sized to outlast a full
|
|
76
|
-
* park + turn so follow-ups fold instead of silently expiring. */
|
|
77
|
-
export const PARK_MAX_MS = 840_000;
|
|
78
|
-
/** spawn slightly after the reset passes, never right on the boundary. */
|
|
79
|
-
const PARK_GRACE_MS = 5_000;
|
|
80
|
-
/** post-limit short retry: one beat for the pool to observe the limit and
|
|
81
|
-
* swap (slaude's parkShortRetry - a successful swap makes it invisible). */
|
|
82
|
-
const RETRY_DELAY_MS = 10_000;
|
|
83
|
-
/** parks + retries per Slack message; keeps a stale usage cache from looping
|
|
84
|
-
* a thread forever. */
|
|
85
|
-
export const MAX_RECOVERIES = 3;
|
|
86
|
-
/** extra spawns for a NON-limit failure whose child never completed (crash,
|
|
87
|
-
* API blip, errored result): each resumes the same session, so a retry
|
|
88
|
-
* continues the turn instead of re-running it. Separate from MAX_RECOVERIES
|
|
89
|
-
* on purpose - that budget bounds limit-driven parking, this one bounds
|
|
90
|
-
* quota spent chasing a possibly-permanent error. */
|
|
91
|
-
export const MAX_TRANSIENT_RETRIES = 2;
|
|
92
|
-
/** Slack expires a native stream SERVER-SIDE on undocumented timers
|
|
93
|
-
* (Slack-maintainer-confirmed in slackapi/python-slack-sdk#1859: idle
|
|
94
|
-
* around 30s, total lifetime around 300s measured), and an expired stream
|
|
95
|
-
* freezes in the Slack client as a grey "Something went wrong" pill - the
|
|
96
|
-
* exact 2026-07-27 report. The salvage path recovers the CONTENT but
|
|
97
|
-
* cannot un-freeze the pill, so the fix is to never let Slack expire a
|
|
98
|
-
* stream we own: rotate the open segment - a clean end() that the adapter
|
|
99
|
-
* finishes with a proper stream stop - before either timer can fire, and
|
|
100
|
-
* let the next chunk open a fresh message, the same flow pushText's size
|
|
101
|
-
* splits already use. Thresholds sit well inside Slack's observed margins;
|
|
102
|
-
* read at every arm (a mutable object, the file's test seam pattern). */
|
|
103
|
-
export const SEGMENT_ROTATION = { idleMs: 20_000, maxAgeMs: 240_000 };
|
|
104
|
-
|
|
105
|
-
const ParkPlanSchema = z.union([
|
|
106
|
-
z.object({ kind: z.literal("proceed") }),
|
|
107
|
-
z.object({ kind: z.literal("park"), wakeAt: z.number() }),
|
|
108
|
-
z.object({ kind: z.literal("defer"), resumeAt: z.number() }),
|
|
109
|
-
z.object({ kind: z.literal("drop") }),
|
|
110
|
-
]);
|
|
111
|
-
export type ParkPlan = z.infer<typeof ParkPlanSchema>;
|
|
112
|
-
|
|
113
|
-
/** What to do with a spawn-boundary switch decision: proceed on a usable pool;
|
|
114
|
-
* park in-handler until the soonest recovery when it lands inside the
|
|
115
|
-
* message's one shared deadline; DEFER when recovery is known but further
|
|
116
|
-
* out (or the in-handler recovery budget is spent): the handler releases the
|
|
117
|
-
* queue slot and the daemon resumes the turn from its durable marker once
|
|
118
|
-
* the pool recovers (2026-07-20 incident: dropped messages sat dead for
|
|
119
|
-
* hours after the pool recovered until the user re-sent them by hand -
|
|
120
|
-
* superseding the older drop-instead-of-promise rule, whose premise was that
|
|
121
|
-
* a will-resume promise could not be kept; the marker + scheduler + startup
|
|
122
|
-
* scan make it durable). Only an UNKNOWN recovery time still drops honestly.
|
|
123
|
-
* The deadline is fixed when the message's relay starts, so chained parks
|
|
124
|
-
* can never hold the queue slot longer than PARK_MAX_MS in total. */
|
|
125
|
-
export function parkPlan(input: { decision: SwapDecision; recoveries: number; deadline: number }): ParkPlan {
|
|
126
|
-
const depleted = input.decision.reason === "all-depleted" || input.decision.reason === "depleted-wait";
|
|
127
|
-
if (!depleted) return { kind: "proceed" };
|
|
128
|
-
const wake = input.decision.waitUntil ?? null;
|
|
129
|
-
if (wake == null) return { kind: "drop" };
|
|
130
|
-
// the grace counts against the deadline too: the promised total hold is
|
|
131
|
-
// exact, not deadline-plus-grace (review catch, PR #18).
|
|
132
|
-
if (wake + PARK_GRACE_MS > input.deadline || input.recoveries >= MAX_RECOVERIES) {
|
|
133
|
-
return { kind: "defer", resumeAt: wake + PARK_GRACE_MS };
|
|
134
|
-
}
|
|
135
|
-
return { kind: "park", wakeAt: wake + PARK_GRACE_MS };
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
/** Phrases claude's ERRORED results carry at a usage/rate limit (ported from
|
|
139
|
-
* slaude's battle-tested set). Checked only against errored results: a
|
|
140
|
-
* successful answer that merely discusses usage limits (routine in this
|
|
141
|
-
* repo's own threads) must never be discarded and re-run. */
|
|
142
|
-
const RATE_LIMIT_PHRASES = [
|
|
143
|
-
"usage limit reached",
|
|
144
|
-
"rate limit reached",
|
|
145
|
-
"rate limit exceeded",
|
|
146
|
-
"rate limit hit",
|
|
147
|
-
"hit your usage limit",
|
|
148
|
-
"hit your weekly limit",
|
|
149
|
-
"limit will reset",
|
|
150
|
-
"5-hour limit",
|
|
151
|
-
"out of extra usage",
|
|
152
|
-
];
|
|
153
|
-
|
|
154
|
-
export function isRateLimitText(input: { text: string }): boolean {
|
|
155
|
-
const lower = input.text.toLowerCase();
|
|
156
|
-
return RATE_LIMIT_PHRASES.some((phrase) => lower.includes(phrase));
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
/** The CLI puts an errored result's reason in `result` even on non-success
|
|
160
|
-
* subtypes (the exact shape PingResultSchema in usage.ts handles), while the
|
|
161
|
-
* SDK's error type declares only `errors` - so limit text is gathered
|
|
162
|
-
* loosely from both fields. */
|
|
163
|
-
const ResultTextSchema = z.looseObject({
|
|
164
|
-
result: z.string().optional(),
|
|
165
|
-
errors: z.array(z.string()).optional(),
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
function erroredResultText(message: unknown): string {
|
|
169
|
-
const parsed = ResultTextSchema.safeParse(message);
|
|
170
|
-
if (!parsed.success) return "";
|
|
171
|
-
return [parsed.data.result, ...(parsed.data.errors ?? [])]
|
|
172
|
-
.filter((t): t is string => t != null && t !== "")
|
|
173
|
-
.join("\n");
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
// ---- workspace identity ----------------------------------------------------
|
|
177
|
-
|
|
178
|
-
const AuthTestSchema = z.looseObject({
|
|
179
|
-
ok: z.boolean(),
|
|
180
|
-
team_id: z.string().optional(),
|
|
181
|
-
error: z.string().optional(),
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
/** auth.test: the home workspace (team) id the bot token belongs to - the
|
|
185
|
-
* reference `isOutsideAuthor` compares message origins against. Errors carry
|
|
186
|
-
* the Slack error code only, never the token. */
|
|
187
|
-
export async function fetchWorkspaceTeamId(input: { botToken: string }): Promise<string> {
|
|
188
|
-
const res = await http
|
|
189
|
-
.post("https://slack.com/api/auth.test", {
|
|
190
|
-
headers: { authorization: `Bearer ${input.botToken}` },
|
|
191
|
-
})
|
|
192
|
-
.catch((e: unknown) => {
|
|
193
|
-
// a thrown ky error (timeout, network) carries its Request with the
|
|
194
|
-
// Authorization header - rethrow message-only so no caller can ever
|
|
195
|
-
// log the token.
|
|
196
|
-
throw new Error(`Slack auth.test failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
197
|
-
});
|
|
198
|
-
const text = await res.text();
|
|
199
|
-
if (!res.ok) throw new Error(`Slack auth.test failed: HTTP ${res.status} (${safeErrorDetail({ text })})`);
|
|
200
|
-
const body: unknown = (() => {
|
|
201
|
-
try {
|
|
202
|
-
return JSON.parse(text);
|
|
203
|
-
} catch {
|
|
204
|
-
return null;
|
|
205
|
-
}
|
|
206
|
-
})();
|
|
207
|
-
const parsed = AuthTestSchema.safeParse(body);
|
|
208
|
-
if (!parsed.success || !parsed.data.ok || !parsed.data.team_id) {
|
|
209
|
-
throw new Error(
|
|
210
|
-
`Slack auth.test failed: ${parsed.success ? (parsed.data.error ?? "no team_id in response") : "unrecognized response"}`,
|
|
211
|
-
);
|
|
212
|
-
}
|
|
213
|
-
return parsed.data.team_id;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
/** The serve plugin shipped inside the package (src/serve-plugin/): skills
|
|
217
|
-
* that teach a relayed session how to behave in a Slack thread, loaded per
|
|
218
|
-
* turn via the SDK's local-plugin option and namespaced `tokenmaxxing:...`. */
|
|
219
|
-
const SERVE_PLUGIN_DIR = join(import.meta.dir, "..", "serve-plugin");
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* The per-turn context a UserPromptSubmit hook injects. The skills are static
|
|
223
|
-
* files, so the one dynamic fact they cannot carry - WHO asked - rides in
|
|
224
|
-
* here as the requester's raw mention token (`<@U...>` passes verbatim
|
|
225
|
-
* through the streamed markdown_text path, and the post-and-edit fallback's
|
|
226
|
-
* finalize leaves an already-formed mention intact). Wording is load-bearing:
|
|
227
|
-
* the ask-the-user skill points at the "Slack relay context" note.
|
|
228
|
-
*/
|
|
229
|
-
export function serveTurnContext(input: { requesterIds: string[] }): string {
|
|
230
|
-
const tokens = input.requesterIds.map((id) => `<@${id}>`);
|
|
231
|
-
let requester = "The requesting user is unknown this turn, so no mention token is available.";
|
|
232
|
-
if (tokens.length === 1) requester = `The requesting user's Slack mention token is ${tokens[0]}; include it literally in reply text to notify them.`;
|
|
233
|
-
if (tokens.length > 1) requester = `This turn folds messages from several users; their Slack mention tokens are ${tokens.join(" ")}. Include the relevant user's token literally in reply text to notify them.`;
|
|
234
|
-
return [
|
|
235
|
-
"Slack relay context: this session is relayed into a Slack thread by tokenmaxxing serve, and your reply posts back into the thread.",
|
|
236
|
-
requester,
|
|
237
|
-
"When you need the user's decision, approval, or input, follow the tokenmaxxing:ask-the-user skill (tag them, ask, end the turn).",
|
|
238
|
-
"The tokenmaxxing:serve-session skill explains how this session runs.",
|
|
239
|
-
].join(" ");
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const SegmentChunkSchema = z.union([z.string(), z.custom<StreamChunk>()]);
|
|
243
|
-
type SegmentChunk = z.infer<typeof SegmentChunkSchema>;
|
|
244
|
-
|
|
245
|
-
/** The exact user-message shape the SDK's own string-prompt path writes to the
|
|
246
|
-
* child's stdin (verified in @anthropic-ai/claude-agent-sdk 0.3.214: the SDK
|
|
247
|
-
* serializes yielded messages verbatim, adding nothing). No uuid on purpose:
|
|
248
|
-
* the CLI dedupes stream-json user messages by uuid and silently swallows a
|
|
249
|
-
* reused one, so a uuid derived from a Slack message ts would eat retries.
|
|
250
|
-
* No priority either: the default "next" folds the message into the running
|
|
251
|
-
* turn at the next tool boundary, while "now" is an undocumented hard
|
|
252
|
-
* interrupt that would abort the turn mid-tool (both verified in the claude
|
|
253
|
-
* 2.1.220 binary). */
|
|
254
|
-
function steerUserMessage(text: string): SDKUserMessage {
|
|
255
|
-
return { type: "user", session_id: "", message: { role: "user", content: [{ type: "text", text }] }, parent_tool_use_id: null };
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
/** A hand-pushed async iterable of SDK user messages: the streaming-input
|
|
259
|
-
* prompt for one query attempt. The initial prompt is pushed before query()
|
|
260
|
-
* and steered messages join mid-turn; end() closes the child's stdin (the
|
|
261
|
-
* SDK ends the stream when the iterable finishes). */
|
|
262
|
-
function pushableMessages(): {
|
|
263
|
-
iterable: AsyncIterable<SDKUserMessage>;
|
|
264
|
-
push: (m: SDKUserMessage) => void;
|
|
265
|
-
end: () => void;
|
|
266
|
-
} {
|
|
267
|
-
const queued: SDKUserMessage[] = [];
|
|
268
|
-
let cursor = 0;
|
|
269
|
-
let done = false;
|
|
270
|
-
let notify: (() => void) | null = null;
|
|
271
|
-
return {
|
|
272
|
-
push(m) {
|
|
273
|
-
queued.push(m);
|
|
274
|
-
notify?.();
|
|
275
|
-
},
|
|
276
|
-
end() {
|
|
277
|
-
done = true;
|
|
278
|
-
notify?.();
|
|
279
|
-
},
|
|
280
|
-
iterable: {
|
|
281
|
-
async *[Symbol.asyncIterator]() {
|
|
282
|
-
while (true) {
|
|
283
|
-
while (cursor < queued.length) {
|
|
284
|
-
const next = queued[cursor]!;
|
|
285
|
-
cursor += 1;
|
|
286
|
-
yield next;
|
|
287
|
-
}
|
|
288
|
-
if (done) return;
|
|
289
|
-
await new Promise<void>((resolve) => {
|
|
290
|
-
notify = resolve;
|
|
291
|
-
});
|
|
292
|
-
notify = null;
|
|
293
|
-
}
|
|
294
|
-
},
|
|
295
|
-
},
|
|
296
|
-
};
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
/** A hand-pushed async iterable: relayThread feeds one of these per Slack
|
|
300
|
-
* message segment while thread.post concurrently drains it. */
|
|
301
|
-
function pushableStream(): {
|
|
302
|
-
iterable: AsyncIterable<SegmentChunk>;
|
|
303
|
-
push: (chunk: SegmentChunk) => void;
|
|
304
|
-
end: () => void;
|
|
305
|
-
ledger: () => { chunks: SegmentChunk[]; confirmed: number };
|
|
306
|
-
} {
|
|
307
|
-
const chunks: SegmentChunk[] = [];
|
|
308
|
-
let cursor = 0;
|
|
309
|
-
let confirmed = 0;
|
|
310
|
-
let done = false;
|
|
311
|
-
let notify: (() => void) | null = null;
|
|
312
|
-
return {
|
|
313
|
-
push(chunk) {
|
|
314
|
-
chunks.push(chunk);
|
|
315
|
-
notify?.();
|
|
316
|
-
},
|
|
317
|
-
end() {
|
|
318
|
-
done = true;
|
|
319
|
-
notify?.();
|
|
320
|
-
},
|
|
321
|
-
/** Everything ever pushed plus how much of it the consumer PROVED
|
|
322
|
-
* delivered. The Slack adapter pulls one chunk, awaits its Slack append,
|
|
323
|
-
* then pulls the next (verified in @chat-adapter/slack 4.34.0 stream()),
|
|
324
|
-
* so each pull confirms the append for the previous chunk landed; a
|
|
325
|
-
* consumer that dies mid-append unwinds the for-await with an implicit
|
|
326
|
-
* return() at the yield, leaving that chunk and everything after it
|
|
327
|
-
* unconfirmed. */
|
|
328
|
-
ledger() {
|
|
329
|
-
return { chunks: [...chunks], confirmed };
|
|
330
|
-
},
|
|
331
|
-
iterable: {
|
|
332
|
-
async *[Symbol.asyncIterator]() {
|
|
333
|
-
while (true) {
|
|
334
|
-
while (cursor < chunks.length) {
|
|
335
|
-
const next = chunks[cursor]!;
|
|
336
|
-
cursor += 1;
|
|
337
|
-
yield next;
|
|
338
|
-
confirmed = cursor;
|
|
339
|
-
}
|
|
340
|
-
if (done) return;
|
|
341
|
-
await new Promise<void>((resolve) => {
|
|
342
|
-
notify = resolve;
|
|
343
|
-
});
|
|
344
|
-
notify = null;
|
|
345
|
-
}
|
|
346
|
-
},
|
|
347
|
-
},
|
|
348
|
-
};
|
|
349
|
-
}
|
|
350
|
-
|
|
351
|
-
/** Slack rejects an over-long message with msg_too_long, and NOTHING in chat
|
|
352
|
-
* 4.34.0 or the Slack adapter bounds, truncates, or splits reply text
|
|
353
|
-
* (verified in-source 2026-07-20): a natively streamed message accumulates
|
|
354
|
-
* server-side toward the 12,000-char markdown_text envelope (docs.slack.dev
|
|
355
|
-
* documents that limit on chat.postMessage/update and all three streaming
|
|
356
|
-
* methods), the post-and-edit fallback re-sends the FULL accumulated text as
|
|
357
|
-
* markdown_text on every edit, and once anything rendered natively a failed
|
|
358
|
-
* append REJECTS the whole thread.post - the reply dies (live incident
|
|
359
|
-
* 2026-07-20, three failed turns). relayThread therefore splits reply text
|
|
360
|
-
* across Slack messages BEFORE the cap; the 2,000-char margin absorbs the
|
|
361
|
-
* renderer's markdown normalization and mention-linkification expansion.
|
|
362
|
-
* Tradeoff (accepted): the adapter-internal plain-text fallback edits via
|
|
363
|
-
* chat.update `text` (hard 4,000-char cap), but it only engages when the
|
|
364
|
-
* workspace refused native streaming outright - splitting every normal reply
|
|
365
|
-
* 3x tighter to cover that never-hit path is worse than the residual risk. */
|
|
366
|
-
export const SEGMENT_TEXT_MAX = 10_000;
|
|
367
|
-
|
|
368
|
-
/** Full permission names of the in-process tools (mcp__<server>__<tool>):
|
|
369
|
-
* they must be in allowedTools, because no one can answer a permission
|
|
370
|
-
* prompt through Slack. */
|
|
371
|
-
const FINISH_THREAD_TOOL = "mcp__tokenmaxxing__finish_thread";
|
|
372
|
-
const NEED_ATTENTION_TOOL = "mcp__tokenmaxxing__need_attention";
|
|
373
|
-
|
|
374
|
-
/** The per-turn in-process MCP server exposing finish_thread and
|
|
375
|
-
* need_attention. The handlers run in the daemon process, but they must NOT
|
|
376
|
-
* act inline: the claude subprocess is still mid-turn and segments are still
|
|
377
|
-
* streaming to Slack, so each only records the request and the daemon acts
|
|
378
|
-
* after the turn ends (serve.ts). alwaysLoad keeps the tools visible in the
|
|
379
|
-
* prompt instead of deferred behind tool search: they have to be in view at
|
|
380
|
-
* the exact moment the user says the work is done or the model hits a fork. */
|
|
381
|
-
function serveToolServer(input: { onFinish: () => void; onAttention: () => void }) {
|
|
382
|
-
return createSdkMcpServer({
|
|
383
|
-
name: "tokenmaxxing",
|
|
384
|
-
alwaysLoad: true,
|
|
385
|
-
tools: [
|
|
386
|
-
tool(
|
|
387
|
-
"finish_thread",
|
|
388
|
-
"Close out this Slack thread when the user clearly states the work is finished (shipped, done, clean this up) and wants the thread closed. After this turn ends the daemon drops the thread's session record, unsubscribes, and posts a confirmation; the repo checkout and everything in it are untouched. Do not call this for a merely answered question - only for an explicit wrap-up.",
|
|
389
|
-
{},
|
|
390
|
-
async () => {
|
|
391
|
-
input.onFinish();
|
|
392
|
-
return { content: [{ type: "text", text: "close-out scheduled - it runs right after this turn ends and posts its own confirmation; just acknowledge the wrap-up now" }] };
|
|
393
|
-
},
|
|
394
|
-
),
|
|
395
|
-
tool(
|
|
396
|
-
"need_attention",
|
|
397
|
-
"Flag this Slack thread as waiting on the requesting user. Call it in the same turn in which you ask them for a decision, approval, or missing information (the ask-the-user skill), then ask in your reply text with their mention token and end the turn. After the turn ends the daemon marks the thread attention-needed (a question-mark reaction on the triggering message) and tags the user once more if they stay quiet; a reaction from them, such as a thumbs up, relays back to you as their answer. Do not call this for rhetorical questions or ordinary replies.",
|
|
398
|
-
{},
|
|
399
|
-
async () => {
|
|
400
|
-
input.onAttention();
|
|
401
|
-
return { content: [{ type: "text", text: "attention flagged - after this turn ends the daemon marks the thread as waiting on the user and nudges them if they stay quiet; ask your question in the reply text with their mention token, then end the turn" }] };
|
|
402
|
-
},
|
|
403
|
-
),
|
|
404
|
-
],
|
|
405
|
-
});
|
|
406
|
-
}
|
|
407
|
-
|
|
408
|
-
const CleanupOutcomeSchema = z.object({
|
|
409
|
-
/** the thread's state is gone; a fresh @mention starts a new session. */
|
|
410
|
-
removed: z.boolean(),
|
|
411
|
-
message: z.string(),
|
|
412
|
-
});
|
|
413
|
-
export type CleanupOutcome = z.infer<typeof CleanupOutcomeSchema>;
|
|
414
|
-
|
|
415
|
-
/**
|
|
416
|
-
* Close out a finished thread. Threads run IN the linked repo checkout (no
|
|
417
|
-
* per-thread worktree or branch since #14), so there is nothing on disk to
|
|
418
|
-
* collect: dropping the slack-threads record is the whole cleanup, and the
|
|
419
|
-
* shared checkout is never touched. The worktree-era residue gate and branch
|
|
420
|
-
* archiving died with the worktrees themselves.
|
|
421
|
-
*/
|
|
422
|
-
export function cleanupThread(input: { threadId: string }): CleanupOutcome {
|
|
423
|
-
deleteSlackThread(input.threadId);
|
|
424
|
-
return { removed: true, message: "thread finished - session closed; a fresh @mention here starts a new one" };
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
/** Live detached process-group leader pids: one process-exit hook SIGTERMs
|
|
428
|
-
* them all, so a forced daemon exit (second signal, drain timeout) cannot
|
|
429
|
-
* leak claude's tool subprocesses. */
|
|
430
|
-
const liveGroups = new Set<number>();
|
|
431
|
-
let groupExitHookArmed = false;
|
|
432
|
-
|
|
433
|
-
/** Exported for serve's orphan reaping: a daemon killed uncatchably (SIGKILL,
|
|
434
|
-
* crash) never runs the exit hook below, so the next generation must be able
|
|
435
|
-
* to terminate a surviving detached group before resuming its turn. */
|
|
436
|
-
export function killGroup(pid: number, signal: "SIGTERM" | "SIGKILL" = "SIGTERM"): void {
|
|
437
|
-
try {
|
|
438
|
-
process.kill(-pid, signal);
|
|
439
|
-
} catch (e) {
|
|
440
|
-
// ESRCH = the group is already gone, which is the state we wanted;
|
|
441
|
-
// anything else (EPERM, a bad pid) must surface, not silently leak.
|
|
442
|
-
if (!(e instanceof Error && "code" in e && e.code === "ESRCH")) throw e;
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
/**
|
|
447
|
-
* Spawns the claude child in its OWN process group (post-0.19.1 review catch):
|
|
448
|
-
* a terminal Ctrl-C delivers SIGINT to the whole foreground group, so a
|
|
449
|
-
* non-detached child died at the same instant the daemon's drain started and
|
|
450
|
-
* the drain could never preserve the in-flight turn. Detached, only the daemon
|
|
451
|
-
* receives the terminal signal. Two consequences the review on PR #16 caught:
|
|
452
|
-
* the SDK's SpawnedProcess contract consumes only stdin/stdout, so stderr must
|
|
453
|
-
* be ignored outright (a piped-but-never-read stderr fills and blocks a chatty
|
|
454
|
-
* child; exit errors lose the stderr tail, an accepted cost of turn survival),
|
|
455
|
-
* and the SDK's abort path kills the lone PID, so the forwarded abort signal
|
|
456
|
-
* and a process-exit hook SIGTERM the whole detached group instead - claude's
|
|
457
|
-
* tool subprocesses must not outlive the daemon or the turn.
|
|
458
|
-
*/
|
|
459
|
-
export function detachedClaudeSpawn(options: SpawnOptions) {
|
|
460
|
-
const stdio: ["pipe", "pipe", "ignore"] = ["pipe", "pipe", "ignore"];
|
|
461
|
-
const child = spawn(options.command, options.args, {
|
|
462
|
-
cwd: options.cwd,
|
|
463
|
-
env: options.env,
|
|
464
|
-
stdio,
|
|
465
|
-
detached: true,
|
|
466
|
-
});
|
|
467
|
-
if (!groupExitHookArmed) {
|
|
468
|
-
groupExitHookArmed = true;
|
|
469
|
-
process.once("exit", () => {
|
|
470
|
-
for (const pid of liveGroups) killGroup(pid);
|
|
471
|
-
});
|
|
472
|
-
}
|
|
473
|
-
if (child.pid !== undefined) {
|
|
474
|
-
const pid = child.pid;
|
|
475
|
-
liveGroups.add(pid);
|
|
476
|
-
const onAbort = () => killGroup(pid);
|
|
477
|
-
// an already-aborted signal never fires "abort" again (cubic review
|
|
478
|
-
// catch): a cancellation racing the spawn must still kill the group.
|
|
479
|
-
if (options.signal?.aborted) onAbort();
|
|
480
|
-
else options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
481
|
-
child.once("exit", () => {
|
|
482
|
-
liveGroups.delete(pid);
|
|
483
|
-
options.signal?.removeEventListener("abort", onAbort);
|
|
484
|
-
});
|
|
485
|
-
}
|
|
486
|
-
return child;
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
/**
|
|
490
|
-
* One claude turn relayed into a Slack thread as ONE streamed message: reply
|
|
491
|
-
* text streams natively and thinking/tool calls stream as task_update cards
|
|
492
|
-
* (see slackstream.ts) that Slack groups into a single collapsible plan block
|
|
493
|
-
* (user ask 2026-07-20: "squash them into one dropdown"; the serve edge wraps
|
|
494
|
-
* each posted segment in a StreamingPlan with groupTasks "plan", superseding
|
|
495
|
-
* the 2026-07-18 separate-messages-around-tool-runs shape). Segments still
|
|
496
|
-
* exist as the posting machinery: recovery notices post as their own
|
|
497
|
-
* messages, a rejected post opens a fresh one (so an over-long turn that
|
|
498
|
-
* trips Slack's message cap degrades to a follow-on message instead of
|
|
499
|
-
* vanishing), and segments post strictly in order: the next opens only after
|
|
500
|
-
* the previous post resolves. Never throws: a failure posts a short
|
|
501
|
-
* diagnostic line and sets outcome.failed (the daemon must keep serving other
|
|
502
|
-
* threads). Error text is message-only - a raw error body could echo request
|
|
503
|
-
* material.
|
|
504
|
-
*
|
|
505
|
-
* Depleted-pool recovery (ported from slaude at its shutdown, reshaped around
|
|
506
|
-
* the pool): the spawn-boundary switch decision is CONSUMED, not discarded -
|
|
507
|
-
* a depleted pool parks BEFORE a doomed spawn burns a failed turn, with an
|
|
508
|
-
* honest in-thread notice either way; a mid-turn limit the cached pool state
|
|
509
|
-
* did not predict is persisted (recordObservedLimit) and retried silently into
|
|
510
|
-
* the same session. Total in-handler parking is bounded by one shared
|
|
511
|
-
* PARK_MAX_MS deadline plus MAX_RECOVERIES; a limit whose recovery lands
|
|
512
|
-
* beyond that budget DEFERS instead of dropping (outcome.deferUntil): the
|
|
513
|
-
* caller keeps the thread's durable marker with resumeAt and the daemon
|
|
514
|
-
* resumes the turn itself once the pool recovers (2026-07-20 incident).
|
|
515
|
-
* Only an unknown recovery time still drops, and every drop the relay itself
|
|
516
|
-
* performs is announced in-thread (a queue-entry TTL expiry upstream is the
|
|
517
|
-
* one drop it cannot see).
|
|
518
|
-
*/
|
|
519
|
-
export async function relayThread(input: {
|
|
520
|
-
cwd: string;
|
|
521
|
-
sessionId: string | null;
|
|
522
|
-
prompt: string;
|
|
523
|
-
/** bare Slack user id (U...) of the triggering message's author. */
|
|
524
|
-
requesterIds: string[];
|
|
525
|
-
link: SlackLink;
|
|
526
|
-
post: (m: AsyncIterable<SegmentChunk>) => Promise<unknown>;
|
|
527
|
-
/** fires the moment an init message assigns a session id the caller has not
|
|
528
|
-
* persisted yet, so a first-turn kill stays resumable (2026-07-18
|
|
529
|
-
* incident: a restart killed a first turn and the thread record kept
|
|
530
|
-
* sessionId null, stranding the session). Retries resume the same session,
|
|
531
|
-
* so re-fires only on an actual id change. */
|
|
532
|
-
onSessionId?: (sessionId: string) => void;
|
|
533
|
-
/** fires with the DETACHED claude child's pid (= its process-group id) the
|
|
534
|
-
* moment it spawns - once per spawn, so a retry's fresh child replaces the
|
|
535
|
-
* previous pid - so the caller can persist it into the activeTurn marker:
|
|
536
|
-
* a daemon death that skips the exit hook (SIGKILL, crash) leaves that
|
|
537
|
-
* group alive, and the next generation must find and reap it before
|
|
538
|
-
* resuming the turn. */
|
|
539
|
-
onSpawn?: (pid: number) => void;
|
|
540
|
-
/** daemon shutdown signal: aborts park/retry sleeps so a drain never sits
|
|
541
|
-
* out a depleted-pool countdown. */
|
|
542
|
-
drainSignal?: AbortSignal;
|
|
543
|
-
/** Steering seam. Called with a steer function while a query attempt is
|
|
544
|
-
* live and with null when it ends; steer(text) returns true when the text
|
|
545
|
-
* was written into the RUNNING attempt's stdin (the CLI folds it into the
|
|
546
|
-
* turn at the next tool boundary, or runs it as its own turn in the same
|
|
547
|
-
* child when the fold window is gone - either way it reaches the session,
|
|
548
|
-
* verified against claude 2.1.220), false once the attempt's result
|
|
549
|
-
* arrived or the attempt died - the caller then queues the message as a
|
|
550
|
-
* normal next turn instead. Accepted texts also fold into any RETRY
|
|
551
|
-
* attempt's prompt, mirroring the existing resend-the-full-prompt retry
|
|
552
|
-
* tradeoff (duplication in the resumed transcript beats loss).
|
|
553
|
-
* input.requesterIds is shared by reference on purpose: the caller may
|
|
554
|
-
* push a steer author's id so a retry attempt's UserPromptSubmit context
|
|
555
|
-
* names them (the hook does not fire for folded mid-turn messages). */
|
|
556
|
-
onSteer?: (steer: ((text: string, onAccept?: () => void) => boolean) | null) => void;
|
|
557
|
-
}): Promise<TurnOutcome> {
|
|
558
|
-
const outcome: TurnOutcome = { sessionId: input.sessionId, failed: false, rateLimited: false, finish: false, attention: false, announcedDrop: false, resultReceived: false, deferUntil: null, steerLost: false };
|
|
559
|
-
let segment: ReturnType<typeof pushableStream> | null = null;
|
|
560
|
-
// acc mirrors the segment's pushed text (bounded by SEGMENT_TEXT_MAX plus a
|
|
561
|
-
// small overshoot): fence parity must be computed over the ACCUMULATED text,
|
|
562
|
-
// never per chunk - SDK deltas do not respect markdown token boundaries, so
|
|
563
|
-
// a ``` split across two deltas would be invisible to per-chunk counting
|
|
564
|
-
// (pullfrog review catch, PR #42).
|
|
565
|
-
let segmentMeta: { text: boolean; acc: string; reopenFence: boolean } | null = null;
|
|
566
|
-
// the open segment's rotation clock (see armSegmentTimer).
|
|
567
|
-
let segmentTimer: ReturnType<typeof setTimeout> | null = null;
|
|
568
|
-
let segmentOpenedAt = 0;
|
|
569
|
-
// a timer rotation closed the segment inside a code fence: the next
|
|
570
|
-
// segment must reopen it, exactly like pushText's size splits do (codex
|
|
571
|
-
// review catch). Consumed by openSegment; the salvage handler's own
|
|
572
|
-
// delivered-prefix recomputation overrides it there.
|
|
573
|
-
let pendingReopenFence = false;
|
|
574
|
-
let lastPost: Promise<unknown> = Promise.resolve();
|
|
575
|
-
// Reply TEXT that died with a rejected segment and was neither salvaged into
|
|
576
|
-
// a follow-on message nor re-delivered by a later text-bearing segment: the
|
|
577
|
-
// user has not seen the answer. A lost card-only segment never sets this
|
|
578
|
-
// (decoration, not the answer). Tradeoff (flagged and accepted): a later
|
|
579
|
-
// delivered text segment clears the flag even though it is a continuation,
|
|
580
|
-
// because a rejection whose text chunks were all consumed pre-append-failure
|
|
581
|
-
// was almost certainly delivered (the adapter appends per chunk) except for
|
|
582
|
-
// an unobservable renderer-held tail; sticky loss would fail every long turn
|
|
583
|
-
// with a spurious diagnostic.
|
|
584
|
-
let textLost = false;
|
|
585
|
-
let textLostDetail: string | null = null;
|
|
586
|
-
let postedText = false;
|
|
587
|
-
// Salvage: a rejected post's undelivered chunks re-post as a fresh message
|
|
588
|
-
// (Slack finalizes an idle stream after an UNDOCUMENTED window - verified
|
|
589
|
-
// absent from the chat.startStream/appendStream docs 2026-07-21 - so
|
|
590
|
-
// recovery is reactive on any append failure, never a keepalive tuned to a
|
|
591
|
-
// guessed constant). The budget bounds FUTILITY, not recovery: a death
|
|
592
|
-
// after the message delivered something is progress and refills it (a long
|
|
593
|
-
// turn can outlive any number of idle finalizations, each losing only the
|
|
594
|
-
// gap tail), while a surface that delivers nothing (revoked channel, hard
|
|
595
|
-
// cap on the very first append) burns a strike per attempt and stops.
|
|
596
|
-
const MAX_SEGMENT_SALVAGES = 5;
|
|
597
|
-
let salvagesLeft = MAX_SEGMENT_SALVAGES;
|
|
598
|
-
const openSegment = () => {
|
|
599
|
-
const seg = pushableStream();
|
|
600
|
-
segment = seg;
|
|
601
|
-
segmentOpenedAt = Date.now();
|
|
602
|
-
armSegmentTimer();
|
|
603
|
-
// reopenFence: the delivered prefix of a REJECTED predecessor left a code
|
|
604
|
-
// fence open, so the first TEXT entering this salvage segment must be
|
|
605
|
-
// preceded by a reopen or it renders outside the code block. Pending
|
|
606
|
-
// rather than pushed eagerly (pullfrog catches, PR #42): a card-only
|
|
607
|
-
// salvage would otherwise either skip the reopen (later text joining the
|
|
608
|
-
// segment renders unfenced) or dangle an empty open fence at message end
|
|
609
|
-
// when no text ever follows. Materialized by BOTH text entry points.
|
|
610
|
-
const meta = { text: false, acc: "", reopenFence: pendingReopenFence };
|
|
611
|
-
pendingReopenFence = false;
|
|
612
|
-
segmentMeta = meta;
|
|
613
|
-
lastPost = input.post(seg.iterable).then(
|
|
614
|
-
() => {
|
|
615
|
-
salvagesLeft = MAX_SEGMENT_SALVAGES;
|
|
616
|
-
// segments settle in order (push awaits lastPost before opening the
|
|
617
|
-
// next), so delivered text supersedes an earlier loss.
|
|
618
|
-
if (meta.text) textLost = false;
|
|
619
|
-
},
|
|
620
|
-
(e: unknown) => {
|
|
621
|
-
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
622
|
-
log("serve.post_error", { err: detail });
|
|
623
|
-
// the consumer is gone (e.g. Slack finalized an idle stream:
|
|
624
|
-
// message_not_in_streaming_state). Salvage runs in TEXT space, not
|
|
625
|
-
// chunk space, because the adapter's renderer buffers across chunks
|
|
626
|
-
// (it holds back the trailing unterminated line, unconfirmed table
|
|
627
|
-
// headers, and unclosed inline markers until the post-iteration
|
|
628
|
-
// finish() flush - pullfrog catch on PR #45: a reply whose last line
|
|
629
|
-
// has no trailing newline dies entirely in that forced flush, with
|
|
630
|
-
// every chunk already consumed). Proven-delivered text = the mirror
|
|
631
|
-
// renderer's committable prefix over the CONFIRMED chunks (each pull
|
|
632
|
-
// proves the previous append landed; the adapter runs the renderer
|
|
633
|
-
// with wrapTablesForAppend: false, so committable is a raw prefix and
|
|
634
|
-
// the mirror is chunking-invariant). Renderer drift would break the
|
|
635
|
-
// prefix check and degrade to a full re-post: duplication, never
|
|
636
|
-
// loss. Same tradeoff for a stop()-failure after a complete flush:
|
|
637
|
-
// the held tail re-posts once rather than risking silent loss.
|
|
638
|
-
const { chunks, confirmed } = seg.ledger();
|
|
639
|
-
if (segment === seg) segment = null;
|
|
640
|
-
const confirmedRaw = chunks
|
|
641
|
-
.slice(0, confirmed)
|
|
642
|
-
.flatMap((c) => (c instanceof Object ? [] : [c]))
|
|
643
|
-
.join("");
|
|
644
|
-
const fullRaw = chunks.flatMap((c) => (c instanceof Object ? [] : [c])).join("");
|
|
645
|
-
const mirror = new StreamingMarkdownRenderer({ wrapTablesForAppend: false });
|
|
646
|
-
mirror.push(confirmedRaw);
|
|
647
|
-
const committed = mirror.getCommittableText();
|
|
648
|
-
const deliveredLen = confirmedRaw.startsWith(committed) ? committed.length : 0;
|
|
649
|
-
const textRemainder = fullRaw.slice(deliveredLen);
|
|
650
|
-
// A confirmed card's append landed (the adapter sends a card inline
|
|
651
|
-
// in the loop body before the next pull), so unconfirmed cards are
|
|
652
|
-
// the set it never appended.
|
|
653
|
-
const deliveredCard = chunks.slice(0, confirmed).some((c) => c instanceof Object);
|
|
654
|
-
// Delivery progress means this death does not count toward the
|
|
655
|
-
// futility budget: only a message that delivered nothing burns one.
|
|
656
|
-
// The key is ACTUAL delivery (committable text or a landed card),
|
|
657
|
-
// never chunk consumption: a reply whose final line has no trailing
|
|
658
|
-
// newline is consumed whole (confirmed advances) while its only
|
|
659
|
-
// append is the post-iteration forced flush, so a consumption key
|
|
660
|
-
// would refill the budget on every persistently failing flush and
|
|
661
|
-
// salvage the same held-back text forever (vercel review catch,
|
|
662
|
-
// PR #45). deliveredLen stays 0 there, the strike is spent, and the
|
|
663
|
-
// zero-delivery case terminates.
|
|
664
|
-
if (deliveredLen > 0 || deliveredCard) salvagesLeft = MAX_SEGMENT_SALVAGES;
|
|
665
|
-
// The salvage sequence preserves STREAM ORDER (cursor review catch,
|
|
666
|
-
// PR #45: re-posting all remainder text and then all cards showed
|
|
667
|
-
// task cards after prose that originally followed them): walk the
|
|
668
|
-
// ledger in order, keeping each text chunk's undelivered suffix and
|
|
669
|
-
// each unconfirmed card at its original position. Text splits at
|
|
670
|
-
// line boundaries (rendering is unchanged: chunks concatenate) so a
|
|
671
|
-
// salvage message that dies too still confirms per line, keeping
|
|
672
|
-
// progress attribution fine-grained.
|
|
673
|
-
const lost: SegmentChunk[] = [];
|
|
674
|
-
let offset = 0;
|
|
675
|
-
for (const [i, c] of chunks.entries()) {
|
|
676
|
-
if (c instanceof Object) {
|
|
677
|
-
if (i >= confirmed) lost.push(c);
|
|
678
|
-
continue;
|
|
679
|
-
}
|
|
680
|
-
const end = offset + c.length;
|
|
681
|
-
let rest = end > deliveredLen ? c.slice(Math.max(0, deliveredLen - offset)) : "";
|
|
682
|
-
offset = end;
|
|
683
|
-
while (rest !== "") {
|
|
684
|
-
const nl = rest.indexOf("\n");
|
|
685
|
-
if (nl === -1) {
|
|
686
|
-
lost.push(rest);
|
|
687
|
-
break;
|
|
688
|
-
}
|
|
689
|
-
lost.push(rest.slice(0, nl + 1));
|
|
690
|
-
rest = rest.slice(nl + 1);
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
// A fence OPENED in the delivered prefix leaves later code fenceless
|
|
694
|
-
// in the fresh salvage message (pullfrog catches, PR #42): arm the
|
|
695
|
-
// segment's pending reopen, materialized right before the FIRST text
|
|
696
|
-
// that enters it - whether a salvaged remainder piece here or a later
|
|
697
|
-
// streamed push joining the segment (a card-only salvage must not
|
|
698
|
-
// skip the reopen, and a text-less segment must not dangle one).
|
|
699
|
-
// Fold this dying segment's OWN still-armed reopen into the parity:
|
|
700
|
-
// an armed-but-never-materialized reopenFence means the segment
|
|
701
|
-
// logically BEGAN inside an open fence (a card-only salvage that
|
|
702
|
-
// inherited one and died before any text materialized it), so that
|
|
703
|
-
// open state must propagate to the next salvage or its later text
|
|
704
|
-
// renders unfenced (vercel + cubic chained-salvage catch, PR #42).
|
|
705
|
-
// Once materialized, reopenFence is false and the reopen chunk is in
|
|
706
|
-
// fullRaw, so the XOR is a no-op; a normal segment's flag is false.
|
|
707
|
-
const deliveredFenceOpen =
|
|
708
|
-
((fullRaw.slice(0, deliveredLen).split("```").length - 1) % 2 === 1) !== meta.reopenFence;
|
|
709
|
-
if (lost.length > 0 && salvagesLeft > 0) {
|
|
710
|
-
salvagesLeft -= 1;
|
|
711
|
-
log("serve.post_salvage", { chunks: lost.length, left: salvagesLeft });
|
|
712
|
-
// this handler runs synchronously as the post settles, so opening
|
|
713
|
-
// the salvage segment here keeps the salvaged content ordered ahead
|
|
714
|
-
// of any push still awaiting lastPost; the salvage segment's own
|
|
715
|
-
// settle then decides whether its text counts as delivered.
|
|
716
|
-
// A salvage always recomputes its fence state from the delivered
|
|
717
|
-
// prefix (the override below), so it must never CONSUME a pending
|
|
718
|
-
// rotation reopen that belongs to a different segment's
|
|
719
|
-
// continuation - a rejected notice settling after notify()
|
|
720
|
-
// restored the flag would otherwise eat it (cubic review catch,
|
|
721
|
-
// round 4). Held across the lost-chunk re-push too: the salvaged
|
|
722
|
-
// text is the dying segment's, not the continuation's.
|
|
723
|
-
const heldReopen = pendingReopenFence;
|
|
724
|
-
pendingReopenFence = false;
|
|
725
|
-
const next = openSegment();
|
|
726
|
-
next.meta.reopenFence = deliveredFenceOpen;
|
|
727
|
-
for (const c of lost) next.pushInto(c);
|
|
728
|
-
pendingReopenFence = heldReopen;
|
|
729
|
-
} else if (textRemainder !== "") {
|
|
730
|
-
textLost = true;
|
|
731
|
-
textLostDetail = detail;
|
|
732
|
-
}
|
|
733
|
-
},
|
|
734
|
-
);
|
|
735
|
-
const pushInto = (chunk: SegmentChunk) => {
|
|
736
|
-
// meta.text only, NEVER postedText: salvaged text was already counted
|
|
737
|
-
// at its original push, and postedText is attempt-scoped - a salvage
|
|
738
|
-
// landing after a retry reset would otherwise re-arm it and suppress
|
|
739
|
-
// the retry's `!postedText && result` fallback, silently dropping a
|
|
740
|
-
// result-only answer (adversarial-review catch on PR #45). acc still
|
|
741
|
-
// accumulates: it mirrors the segment's FULL text on every entry path,
|
|
742
|
-
// so pushText's room accounting and fence parity see salvaged text too
|
|
743
|
-
// (a salvage segment that continued via pushText could otherwise grow
|
|
744
|
-
// past the msg_too_long cap).
|
|
745
|
-
if (!(chunk instanceof Object)) {
|
|
746
|
-
if (meta.reopenFence) {
|
|
747
|
-
meta.reopenFence = false;
|
|
748
|
-
meta.acc += "```\n";
|
|
749
|
-
seg.push("```\n");
|
|
750
|
-
}
|
|
751
|
-
meta.text = true;
|
|
752
|
-
meta.acc += chunk;
|
|
753
|
-
}
|
|
754
|
-
seg.push(chunk);
|
|
755
|
-
if (segment === seg) armSegmentTimer();
|
|
756
|
-
};
|
|
757
|
-
return { seg, meta, pushInto };
|
|
758
|
-
};
|
|
759
|
-
const push = async (chunk: SegmentChunk) => {
|
|
760
|
-
if (segment === null) {
|
|
761
|
-
await lastPost; // strict message order: previous segment fully posted first
|
|
762
|
-
// a rejection handler may have opened a salvage segment during the wait;
|
|
763
|
-
// joining it instead of opening another keeps its post from being
|
|
764
|
-
// orphaned un-ended.
|
|
765
|
-
}
|
|
766
|
-
const target = segment ?? openSegment().seg;
|
|
767
|
-
if (!(chunk instanceof Object)) {
|
|
768
|
-
// a pending rotation reopen can outlive its openSegment hand-off when
|
|
769
|
-
// text JOINS a segment a rejection handler opened during the wait
|
|
770
|
-
// (the salvage recomputes its own fence state without consuming the
|
|
771
|
-
// flag): adopt it here so the continuation still reopens its fence,
|
|
772
|
-
// UNLESS the joined segment already sits inside an open fence - then
|
|
773
|
-
// the state is satisfied and a second marker would close it (cubic
|
|
774
|
-
// review catch, round 4).
|
|
775
|
-
if (pendingReopenFence) {
|
|
776
|
-
pendingReopenFence = false;
|
|
777
|
-
if (!fenceOpen()) segmentMeta!.reopenFence = true;
|
|
778
|
-
}
|
|
779
|
-
// materialize a salvage segment's pending fence reopen (see
|
|
780
|
-
// openSegment) before the first text from this entry point too.
|
|
781
|
-
if (segmentMeta!.reopenFence) {
|
|
782
|
-
segmentMeta!.reopenFence = false;
|
|
783
|
-
segmentMeta!.acc += "```\n";
|
|
784
|
-
target.push("```\n");
|
|
785
|
-
}
|
|
786
|
-
postedText = true;
|
|
787
|
-
segmentMeta!.text = true;
|
|
788
|
-
segmentMeta!.acc += chunk;
|
|
789
|
-
}
|
|
790
|
-
target.push(chunk);
|
|
791
|
-
if (segment === target) armSegmentTimer();
|
|
792
|
-
};
|
|
793
|
-
// parity by occurrence count over the segment's accumulated text: an odd
|
|
794
|
-
// number of ``` markers means the segment currently sits inside a fence.
|
|
795
|
-
const fenceOpen = () => segmentMeta !== null && (segmentMeta.acc.split("```").length - 1) % 2 === 1;
|
|
796
|
-
const breakSegment = () => {
|
|
797
|
-
if (segmentTimer !== null) {
|
|
798
|
-
clearTimeout(segmentTimer);
|
|
799
|
-
segmentTimer = null;
|
|
800
|
-
}
|
|
801
|
-
// a segment dying with an armed-but-unmaterialized reopen (no text ever
|
|
802
|
-
// arrived - a card-only segment that consumed the flag at open) folds it
|
|
803
|
-
// back: the reply still logically sits inside an open fence, and the
|
|
804
|
-
// next text-bearing segment must reopen it (cubic review catch, round
|
|
805
|
-
// 4: the errored result's closing card ate the rotation's reopen).
|
|
806
|
-
if (segmentMeta?.reopenFence) pendingReopenFence = true;
|
|
807
|
-
segment?.end();
|
|
808
|
-
segment = null;
|
|
809
|
-
};
|
|
810
|
-
const armSegmentTimer = () => {
|
|
811
|
-
if (segmentTimer !== null) clearTimeout(segmentTimer);
|
|
812
|
-
const ageLeft = segmentOpenedAt + SEGMENT_ROTATION.maxAgeMs - Date.now();
|
|
813
|
-
segmentTimer = setTimeout(() => {
|
|
814
|
-
segmentTimer = null;
|
|
815
|
-
// a rotation mid-fence closes the fence and arms the reopen, so both
|
|
816
|
-
// message halves render as code - pushText's split contract. An
|
|
817
|
-
// armed-but-unmaterialized reopen needs no close marker; breakSegment
|
|
818
|
-
// folds it forward.
|
|
819
|
-
if (segment !== null && fenceOpen()) {
|
|
820
|
-
segmentMeta!.acc += "\n```";
|
|
821
|
-
segment.push("\n```");
|
|
822
|
-
pendingReopenFence = true;
|
|
823
|
-
}
|
|
824
|
-
breakSegment();
|
|
825
|
-
}, Math.max(0, Math.min(SEGMENT_ROTATION.idleMs, ageLeft)));
|
|
826
|
-
};
|
|
827
|
-
/** Reply text routed through here splits across Slack messages before the
|
|
828
|
-
* msg_too_long cap (see SEGMENT_TEXT_MAX): a break prefers the last newline
|
|
829
|
-
* inside the remaining room, and a break forced inside a code fence closes
|
|
830
|
-
* it and reopens it in the next message so both halves render as code. */
|
|
831
|
-
const pushText = async (text: string) => {
|
|
832
|
-
for (let rest = text; rest !== "";) {
|
|
833
|
-
const room = SEGMENT_TEXT_MAX - (segment === null ? 0 : segmentMeta!.acc.length);
|
|
834
|
-
// a fence-close suffix can nudge a segment a few chars past the cap;
|
|
835
|
-
// a full segment just breaks and the loop re-measures a fresh one.
|
|
836
|
-
if (room <= 0) {
|
|
837
|
-
breakSegment();
|
|
838
|
-
continue;
|
|
839
|
-
}
|
|
840
|
-
if (rest.length <= room) {
|
|
841
|
-
await push(rest);
|
|
842
|
-
return;
|
|
843
|
-
}
|
|
844
|
-
// prefer a newline cut only when it lands in the back half of the room:
|
|
845
|
-
// an early newline followed by one giant unbroken run would otherwise
|
|
846
|
-
// make no progress and (with the fence-reopen prefix) loop forever.
|
|
847
|
-
const nl = rest.lastIndexOf("\n", room - 1);
|
|
848
|
-
let cut = nl >= Math.floor(room / 2) ? nl + 1 : room;
|
|
849
|
-
// never slice through a backtick run: a cut inside ``` would strand a
|
|
850
|
-
// partial delimiter on each side and break both halves' rendering
|
|
851
|
-
// (cubic review catch, PR #42). Walk the cut left past the run; a run
|
|
852
|
-
// reaching position 0 keeps the original cut (progress beats rendering
|
|
853
|
-
// for pathological all-backtick input).
|
|
854
|
-
if (rest[cut - 1] === "`" && rest[cut] === "`") {
|
|
855
|
-
let backedUp = cut;
|
|
856
|
-
while (backedUp > 0 && rest[backedUp - 1] === "`") backedUp -= 1;
|
|
857
|
-
if (backedUp > 0) cut = backedUp;
|
|
858
|
-
}
|
|
859
|
-
const head = rest.slice(0, cut);
|
|
860
|
-
await push(head);
|
|
861
|
-
const reopen = fenceOpen();
|
|
862
|
-
if (reopen) await push("\n```");
|
|
863
|
-
breakSegment();
|
|
864
|
-
rest = (reopen ? "```\n" : "") + rest.slice(head.length);
|
|
865
|
-
}
|
|
866
|
-
};
|
|
867
|
-
// settle the whole post chain: a rejection handler may replace lastPost with
|
|
868
|
-
// a salvage segment's post, which still needs ending and settling.
|
|
869
|
-
const settlePosts = async () => {
|
|
870
|
-
while (true) {
|
|
871
|
-
breakSegment();
|
|
872
|
-
const settled = lastPost;
|
|
873
|
-
await settled;
|
|
874
|
-
if (lastPost === settled) return;
|
|
875
|
-
}
|
|
876
|
-
};
|
|
877
|
-
// a recovery status line reads as its own Slack message, not part of a
|
|
878
|
-
// streamed segment.
|
|
879
|
-
const notify = async (text: string) => {
|
|
880
|
-
breakSegment();
|
|
881
|
-
// a notice must never inherit a rotation's pending fence reopen: the
|
|
882
|
-
// reopen belongs to the interrupted reply's continuation, not to the
|
|
883
|
-
// status line - consuming it here would render the notice as a code
|
|
884
|
-
// block AND strand the continuation unfenced (cubic review catch,
|
|
885
|
-
// round 3). Held aside and restored for the real continuation.
|
|
886
|
-
const heldReopen = pendingReopenFence;
|
|
887
|
-
pendingReopenFence = false;
|
|
888
|
-
await push(text);
|
|
889
|
-
breakSegment();
|
|
890
|
-
pendingReopenFence = heldReopen;
|
|
891
|
-
};
|
|
892
|
-
/** notify + confirm the post actually landed in Slack. A drop notice that
|
|
893
|
-
* never reached the user must NOT count as announced. DURING A DRAIN an
|
|
894
|
-
* unannounced drop keeps the resume marker so startup replays instead;
|
|
895
|
-
* outside a drain the marker is still cleared ON PURPOSE (closing-review
|
|
896
|
-
* catch corrected this doc, not the behavior): a non-drain drop happens
|
|
897
|
-
* after the turn's spawn decisions ran, and retaining its marker would
|
|
898
|
-
* make the next daemon restart RE-EXECUTE a possibly-metered turn whose
|
|
899
|
-
* outcome the user may already have seen - duplicate execution is worse
|
|
900
|
-
* than a lost message behind an already-broken Slack surface. The
|
|
901
|
-
* unannounced non-drain loss is logged loudly (serve.drop_unannounced) by
|
|
902
|
-
* the caller so it is at least operator-visible. The notice is text, so
|
|
903
|
-
* its own delivery resets textLost. */
|
|
904
|
-
const notifyDelivered = async (text: string) => {
|
|
905
|
-
await notify(text);
|
|
906
|
-
await settlePosts();
|
|
907
|
-
return !textLost;
|
|
908
|
-
};
|
|
909
|
-
// false when the daemon started draining mid-sleep.
|
|
910
|
-
const sleep = async (ms: number) => {
|
|
911
|
-
try {
|
|
912
|
-
await delay(Math.max(ms, 0), { signal: input.drainSignal });
|
|
913
|
-
return true;
|
|
914
|
-
} catch {
|
|
915
|
-
return false;
|
|
916
|
-
}
|
|
917
|
-
};
|
|
918
|
-
const inWord = (epochMs: number | null) => (epochMs == null ? "an unknown time" : `~${fmtResetShort(epochMs, Date.now()) || "1m"}`);
|
|
919
|
-
|
|
920
|
-
// a non-limit failure line held back until the depleted-pool probe rules:
|
|
921
|
-
// posted verbatim on a plain failure, discarded when the turn defers (the
|
|
922
|
-
// deferral notice explains the pause; the raw line would invite a manual
|
|
923
|
-
// re-send of work the daemon resumes itself - cubic catch, PR #44).
|
|
924
|
-
let pendingFailureLine: string | null = null;
|
|
925
|
-
// texts steered into this message's turn, kept across retries: a retry
|
|
926
|
-
// resumes the session and re-sends the attempt prompt (the established
|
|
927
|
-
// duplication-beats-loss tradeoff), so steered text folds into every
|
|
928
|
-
// retry attempt's first message too - a steer written just before a
|
|
929
|
-
// mid-turn limit killed the child must not vanish from the turn it joined.
|
|
930
|
-
const steeredTexts: string[] = [];
|
|
931
|
-
// what the next spawn submits ahead of the steered texts: the original
|
|
932
|
-
// message, until a transient retry swaps in the continuation wrapper (see
|
|
933
|
-
// the retry branch).
|
|
934
|
-
let prompt = input.prompt;
|
|
935
|
-
const runQueryOnce = async () => {
|
|
936
|
-
postedText = false;
|
|
937
|
-
pendingFailureLine = null;
|
|
938
|
-
outcome.failed = false;
|
|
939
|
-
outcome.rateLimited = false;
|
|
940
|
-
outcome.resultReceived = false;
|
|
941
|
-
// outcome.finish and outcome.attention stay sticky across retries: the
|
|
942
|
-
// tool calls already happened in this session, and a limit right after
|
|
943
|
-
// one must not unfinish the thread or drop the pending ask.
|
|
944
|
-
// the identity this spawn meters: a limit observation is attributed to it,
|
|
945
|
-
// never to whatever account a concurrent thread swaps live mid-turn. Read
|
|
946
|
-
// inside the try: a malformed claude.json must fail the TURN, not the
|
|
947
|
-
// relay's never-throws contract.
|
|
948
|
-
let spawnOrg: string | null = null;
|
|
949
|
-
// Streaming input: the prompt rides an open stdin stream instead of a
|
|
950
|
-
// one-shot string, which is what lets a mid-turn Slack reply steer the
|
|
951
|
-
// running turn (the CLI folds a queued stream-json user message into the
|
|
952
|
-
// current turn at the next tool boundary; one that misses the last fold
|
|
953
|
-
// window runs as its own turn in the same child before exit, so nothing
|
|
954
|
-
// is ever dropped - both verified against claude 2.1.220 + SDK 0.3.214).
|
|
955
|
-
// steer() refuses the moment the attempt's result arrives: stdin ends
|
|
956
|
-
// then, and the SDK silently drops writes to an ended stdin.
|
|
957
|
-
const stream = pushableMessages();
|
|
958
|
-
let steerable = true;
|
|
959
|
-
const steer = (text: string, onAccept?: () => void): boolean => {
|
|
960
|
-
if (!steerable) return false;
|
|
961
|
-
// the caller's SYNCHRONOUS durable commit runs inside the acceptance,
|
|
962
|
-
// in the same JS tick as the liveness check and BEFORE the child sees
|
|
963
|
-
// the text (adversarial-review catch, round 3): steerable=true here
|
|
964
|
-
// proves the turn has not ended, so the commit's view of the turn
|
|
965
|
-
// state cannot be stale, a crash between commit and push replays a
|
|
966
|
-
// steer the child never saw (duplication over loss), and a refusal
|
|
967
|
-
// commits NOTHING - a stale acceptor invocation racing the turn's end
|
|
968
|
-
// can no longer resurrect a finished turn's marker or clobber
|
|
969
|
-
// post-turn state. A THROWING commit escapes to the caller before
|
|
970
|
-
// anything is pushed or recorded here: the stream, steeredTexts, and
|
|
971
|
-
// the sticky flags are untouched, so the caller can treat the throw
|
|
972
|
-
// as a refusal and a later steer still works.
|
|
973
|
-
onAccept?.();
|
|
974
|
-
steeredTexts.push(text);
|
|
975
|
-
stream.push(steerUserMessage(text));
|
|
976
|
-
// a steer answers a LIVE ask too (codex review catch on PR #50): when
|
|
977
|
-
// need_attention already fired in this attempt, the ask exists only as
|
|
978
|
-
// this sticky flag until settleTurn persists it - left set, the
|
|
979
|
-
// answered ask would still get a question mark and a nudge. A later
|
|
980
|
-
// need_attention call re-arms it for a genuinely new ask.
|
|
981
|
-
outcome.attention = false;
|
|
982
|
-
return true;
|
|
983
|
-
};
|
|
984
|
-
try {
|
|
985
|
-
spawnOrg = readOAuthAccount()?.organizationUuid ?? null;
|
|
986
|
-
const pooled = pooledOptions();
|
|
987
|
-
stream.push(steerUserMessage([prompt, ...steeredTexts].join("\n\n")));
|
|
988
|
-
const q = query({
|
|
989
|
-
prompt: stream.iterable,
|
|
990
|
-
options: {
|
|
991
|
-
...pooled,
|
|
992
|
-
// claude >= 2.1.142 emits the structured Task tools by default and
|
|
993
|
-
// TodoWrite (the source of the Todos checklist card) never fires;
|
|
994
|
-
// this documented opt-out restores it (agent-sdk todo-tracking docs,
|
|
995
|
-
// verified 2026-07-18 against SDK 0.3.214 + claude 2.1.214). Reuses
|
|
996
|
-
// pooled.env so the scrubbed env copy is built once per turn (cubic
|
|
997
|
-
// review catch on PR #5).
|
|
998
|
-
env: { ...pooled.env, CLAUDE_CODE_ENABLE_TASKS: "0" },
|
|
999
|
-
cwd: input.cwd,
|
|
1000
|
-
permissionMode: input.link.permissionMode,
|
|
1001
|
-
// the SDK refuses bypassPermissions without this explicit opt-in.
|
|
1002
|
-
...(input.link.permissionMode === "bypassPermissions" ? { allowDangerouslySkipPermissions: true } : {}),
|
|
1003
|
-
includePartialMessages: true,
|
|
1004
|
-
systemPrompt: SLACK_SYSTEM_PROMPT,
|
|
1005
|
-
// no one can answer an interactive question dialog through Slack;
|
|
1006
|
-
// without the tool the model asks in prose and the user's thread
|
|
1007
|
-
// reply becomes the next turn.
|
|
1008
|
-
disallowedTools: ["AskUserQuestion"],
|
|
1009
|
-
spawnClaudeCodeProcess: (spawnOptions) => {
|
|
1010
|
-
const child = detachedClaudeSpawn(spawnOptions);
|
|
1011
|
-
if (child.pid !== undefined) {
|
|
1012
|
-
try {
|
|
1013
|
-
input.onSpawn?.(child.pid);
|
|
1014
|
-
} catch (e) {
|
|
1015
|
-
// a failed marker persist must not leave an untracked group
|
|
1016
|
-
// running (cubic review catch): kill it, then fail the spawn
|
|
1017
|
-
// loudly through the SDK.
|
|
1018
|
-
killGroup(child.pid);
|
|
1019
|
-
throw e;
|
|
1020
|
-
}
|
|
1021
|
-
}
|
|
1022
|
-
return child;
|
|
1023
|
-
},
|
|
1024
|
-
// the user saying "we're done" closes the thread, and the model
|
|
1025
|
-
// asking the user for a decision marks it waiting: both flagged via
|
|
1026
|
-
// in-process tools, both acted on by the daemon post-turn. Like
|
|
1027
|
-
// finish, attention stays sticky across retries: the tool call
|
|
1028
|
-
// already happened in this session.
|
|
1029
|
-
mcpServers: {
|
|
1030
|
-
tokenmaxxing: serveToolServer({
|
|
1031
|
-
onFinish: () => { outcome.finish = true; },
|
|
1032
|
-
onAttention: () => { outcome.attention = true; },
|
|
1033
|
-
}),
|
|
1034
|
-
},
|
|
1035
|
-
allowedTools: [FINISH_THREAD_TOOL, NEED_ATTENTION_TOOL],
|
|
1036
|
-
// serve skills (ask-the-user, serve-session); discovered skills are
|
|
1037
|
-
// enabled by default, so no `skills` option is needed.
|
|
1038
|
-
plugins: [{ type: "local", path: SERVE_PLUGIN_DIR }],
|
|
1039
|
-
hooks: {
|
|
1040
|
-
UserPromptSubmit: [{
|
|
1041
|
-
hooks: [async () => ({
|
|
1042
|
-
hookSpecificOutput: {
|
|
1043
|
-
hookEventName: "UserPromptSubmit",
|
|
1044
|
-
additionalContext: serveTurnContext({ requesterIds: input.requesterIds }),
|
|
1045
|
-
},
|
|
1046
|
-
})],
|
|
1047
|
-
}],
|
|
1048
|
-
Stop: [{ hooks: [stopHookCheck] }],
|
|
1049
|
-
},
|
|
1050
|
-
...(input.link.model ? { model: input.link.model } : {}),
|
|
1051
|
-
// a retry resumes the session the failed attempt opened, so no
|
|
1052
|
-
// context is lost across recoveries.
|
|
1053
|
-
...(outcome.sessionId ? { resume: outcome.sessionId } : {}),
|
|
1054
|
-
},
|
|
1055
|
-
});
|
|
1056
|
-
// registered only while this attempt is live (cleared in the finally):
|
|
1057
|
-
// the caller's fallback for a refused steer is the normal queued turn.
|
|
1058
|
-
input.onSteer?.(steer);
|
|
1059
|
-
const mapState = newStreamMapState();
|
|
1060
|
-
let result: string | null = null;
|
|
1061
|
-
// reply text streamed since the last result boundary: each turn in the
|
|
1062
|
-
// child (the primary one, plus any post-fold-window steer drained as
|
|
1063
|
-
// its own turn) delivers its answer independently - a tool-only turn's
|
|
1064
|
-
// answer lives ONLY in its result message, and flushing it at that
|
|
1065
|
-
// result is what keeps a trailing turn's notice or result from
|
|
1066
|
-
// suppressing or clobbering it (adversarial-review catch, round 2).
|
|
1067
|
-
let streamedSinceResult = false;
|
|
1068
|
-
for await (const message of q) {
|
|
1069
|
-
if (message.type === "system" && message.subtype === "init") {
|
|
1070
|
-
// persist BEFORE the turn ends so a first-turn kill stays
|
|
1071
|
-
// resumable; compared against the last known id, so retry attempts
|
|
1072
|
-
// resuming the same session re-fire only on an actual change.
|
|
1073
|
-
if (message.session_id !== outcome.sessionId) input.onSessionId?.(message.session_id);
|
|
1074
|
-
outcome.sessionId = message.session_id;
|
|
1075
|
-
}
|
|
1076
|
-
if (message.type === "result") {
|
|
1077
|
-
outcome.sessionId = message.session_id;
|
|
1078
|
-
// any result ends steerability and closes the child's stdin: a
|
|
1079
|
-
// steer arriving now queues as its own next turn instead. A steer
|
|
1080
|
-
// accepted BEFORE this that missed its fold window still runs -
|
|
1081
|
-
// the CLI drains queued commands as their own turns in this same
|
|
1082
|
-
// child before exiting, emitting a further result each time, so
|
|
1083
|
-
// this loop just keeps consuming until the child exits (end() is
|
|
1084
|
-
// idempotent).
|
|
1085
|
-
steerable = false;
|
|
1086
|
-
stream.end();
|
|
1087
|
-
// is_error can ride a "success" subtype (a mid-turn usage limit
|
|
1088
|
-
// arrives exactly that way: result "Claude AI usage limit
|
|
1089
|
-
// reached|<epoch>"), so errored is a field check, not a subtype
|
|
1090
|
-
// check - and only an errored result is ever limit-classified.
|
|
1091
|
-
if (message.is_error || message.subtype !== "success") {
|
|
1092
|
-
const text = erroredResultText(message);
|
|
1093
|
-
const limited = isRateLimitText({ text });
|
|
1094
|
-
// persist the observation either way: the next spawn decision
|
|
1095
|
-
// must see the limit even when this turn does not retry.
|
|
1096
|
-
if (limited) await recordObservedLimit({ text, now: Date.now(), org: spawnOrg });
|
|
1097
|
-
if (outcome.resultReceived) {
|
|
1098
|
-
// SUCCESS IS STICKY within an attempt (adversarial-review
|
|
1099
|
-
// catch): this errored result belongs to a post-fold-window
|
|
1100
|
-
// steer's own drained turn, arriving AFTER the primary turn
|
|
1101
|
-
// already succeeded and delivered its answer. Marking the turn
|
|
1102
|
-
// failed here would send the whole prompt back through the
|
|
1103
|
-
// retry/defer machinery and re-run completed work - the exact
|
|
1104
|
-
// duplicate-execution the outcome contract forbids. The steer
|
|
1105
|
-
// is announced lost instead (drop-beats-false-promise), and
|
|
1106
|
-
// steerLost settles the steered messages' reactions as failed.
|
|
1107
|
-
outcome.steerLost = true;
|
|
1108
|
-
log("serve.steered_turn_failed", { limited });
|
|
1109
|
-
await notify(
|
|
1110
|
-
limited
|
|
1111
|
-
? "a steered follow-up message hit a usage limit before it could run - please re-send it."
|
|
1112
|
-
: "a steered follow-up message failed - please re-send it.",
|
|
1113
|
-
);
|
|
1114
|
-
} else {
|
|
1115
|
-
outcome.failed = true;
|
|
1116
|
-
// a limit classification is sticky across a child's errored
|
|
1117
|
-
// results (adversarial-review catch, round 2): the drained
|
|
1118
|
-
// steer turn's generic death must not declassify the primary
|
|
1119
|
-
// turn's recoverable limit back to a plain failure.
|
|
1120
|
-
outcome.rateLimited = outcome.rateLimited || limited;
|
|
1121
|
-
// hold the REAL errored text for the terminal diagnostic (codex
|
|
1122
|
-
// review catches: a streamed-then-errored turn used to end with
|
|
1123
|
-
// a truncated answer and only the x reaction, and a no-text
|
|
1124
|
-
// errored turn used to discard the reason for a generic line).
|
|
1125
|
-
// The depleted-pool probe still outranks the line.
|
|
1126
|
-
if (!limited) {
|
|
1127
|
-
const reason = text.slice(0, 200) || "no error detail";
|
|
1128
|
-
pendingFailureLine = postedText
|
|
1129
|
-
? `tokenmaxxing: the turn errored before finishing (${reason}) - the reply above may be incomplete.`
|
|
1130
|
-
: `tokenmaxxing: the turn errored without a result (${reason}) - trying again may help.`;
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
} else {
|
|
1134
|
-
result = message.result;
|
|
1135
|
-
outcome.resultReceived = true;
|
|
1136
|
-
// a turn that streamed no reply text (tool-only turns) still
|
|
1137
|
-
// reports: its answer is flushed HERE, per result, not after the
|
|
1138
|
-
// loop - a later turn in the same child must not suppress it. A
|
|
1139
|
-
// paragraph break ahead of it when text already posted, or two
|
|
1140
|
-
// result-only answers would concatenate mid-line (codex review
|
|
1141
|
-
// catch on PR #50).
|
|
1142
|
-
if (!streamedSinceResult && result) await pushText(`${postedText ? "\n\n" : ""}${result}`);
|
|
1143
|
-
}
|
|
1144
|
-
streamedSinceResult = false;
|
|
1145
|
-
}
|
|
1146
|
-
for (const part of agentEventChunks({ state: mapState, message })) {
|
|
1147
|
-
if (part instanceof Object) await push(part);
|
|
1148
|
-
else {
|
|
1149
|
-
if (part.trim() !== "") streamedSinceResult = true;
|
|
1150
|
-
await pushText(part);
|
|
1151
|
-
}
|
|
1152
|
-
}
|
|
1153
|
-
}
|
|
1154
|
-
} catch (e) {
|
|
1155
|
-
outcome.failed = true;
|
|
1156
|
-
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
1157
|
-
outcome.rateLimited = isRateLimitText({ text: detail });
|
|
1158
|
-
if (outcome.rateLimited) await recordObservedLimit({ text: detail, now: Date.now(), org: spawnOrg });
|
|
1159
|
-
log("serve.turn_error", { err: detail });
|
|
1160
|
-
// same hold-back: the detail already reached the log above, and a
|
|
1161
|
-
// deferral's own notice explains the pause better than a raw child
|
|
1162
|
-
// error that reads as "please re-send".
|
|
1163
|
-
if (!outcome.rateLimited) pendingFailureLine = `tokenmaxxing: turn failed: ${detail}`;
|
|
1164
|
-
} finally {
|
|
1165
|
-
// a died attempt must stop accepting steers (they would be silent
|
|
1166
|
-
// drops on an ended stdin) and must release the caller's steer hook
|
|
1167
|
-
// before the retry loop decides anything.
|
|
1168
|
-
steerable = false;
|
|
1169
|
-
stream.end();
|
|
1170
|
-
input.onSteer?.(null);
|
|
1171
|
-
}
|
|
1172
|
-
};
|
|
1173
|
-
|
|
1174
|
-
let recoveries = 0;
|
|
1175
|
-
let transientRetries = 0;
|
|
1176
|
-
const parkDeadline = Date.now() + PARK_MAX_MS;
|
|
1177
|
-
while (true) {
|
|
1178
|
-
// the switch decision runs at the spawn boundary, same as the CLI hooks.
|
|
1179
|
-
let decision: SwapDecision;
|
|
1180
|
-
try {
|
|
1181
|
-
decision = await ensureBestAccount();
|
|
1182
|
-
} catch (e) {
|
|
1183
|
-
outcome.failed = true;
|
|
1184
|
-
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
1185
|
-
log("serve.turn_error", { err: detail });
|
|
1186
|
-
await push(`tokenmaxxing: turn failed: ${detail}`);
|
|
1187
|
-
break;
|
|
1188
|
-
}
|
|
1189
|
-
const plan = parkPlan({ decision, recoveries, deadline: parkDeadline });
|
|
1190
|
-
if (plan.kind === "drop") {
|
|
1191
|
-
// recovery time unknown: an auto-resume promise would be unkeepable,
|
|
1192
|
-
// so the honest drop survives for exactly this case.
|
|
1193
|
-
outcome.failed = true;
|
|
1194
|
-
outcome.rateLimited = true;
|
|
1195
|
-
log("serve.pool_depleted_drop", {});
|
|
1196
|
-
outcome.announcedDrop = await notifyDelivered("every pooled account is at its usage limit (recovers at an unknown time) - this message was dropped; re-send it once the pool recovers.");
|
|
1197
|
-
break;
|
|
1198
|
-
}
|
|
1199
|
-
if (plan.kind === "defer") {
|
|
1200
|
-
// release the queue slot and let the daemon resume the turn from its
|
|
1201
|
-
// durable marker once the pool recovers (2026-07-20 incident: dropped
|
|
1202
|
-
// messages sat dead for hours after recovery until re-sent by hand).
|
|
1203
|
-
outcome.failed = true;
|
|
1204
|
-
outcome.rateLimited = true;
|
|
1205
|
-
outcome.deferUntil = plan.resumeAt;
|
|
1206
|
-
log("serve.pool_depleted_defer", { resumeAt: plan.resumeAt });
|
|
1207
|
-
await notify(`every pooled account is at its usage limit - holding this message; it will resume automatically in ${inWord(plan.resumeAt)}.`);
|
|
1208
|
-
break;
|
|
1209
|
-
}
|
|
1210
|
-
if (plan.kind === "park") {
|
|
1211
|
-
recoveries += 1;
|
|
1212
|
-
log("serve.pool_depleted_park", { wakeAt: plan.wakeAt, recoveries });
|
|
1213
|
-
await notify(`every pooled account is at its usage limit - holding this message and retrying in ${inWord(plan.wakeAt)}.`);
|
|
1214
|
-
if (!(await sleep(plan.wakeAt - Date.now()))) {
|
|
1215
|
-
// a drain aborted the park: the turn never spawned, so the marker
|
|
1216
|
-
// survives (presumedKilled) and the next daemon start replays it.
|
|
1217
|
-
outcome.failed = true;
|
|
1218
|
-
await notify("tokenmaxxing is restarting - this message resumes after the restart.");
|
|
1219
|
-
break;
|
|
1220
|
-
}
|
|
1221
|
-
continue;
|
|
1222
|
-
}
|
|
1223
|
-
await runQueryOnce();
|
|
1224
|
-
if (!outcome.failed) break;
|
|
1225
|
-
if (!outcome.rateLimited) {
|
|
1226
|
-
// an unclassifiable child failure (e.g. "Claude Code process exited
|
|
1227
|
-
// with code 1" - the 2026-07-20 Fable-cap death carried no limit
|
|
1228
|
-
// phrase) against an exhausted pool IS the limit: the pool state is
|
|
1229
|
-
// the evidence the error text did not carry. A completed result stays
|
|
1230
|
-
// terminal, and a usable pool keeps the plain failure.
|
|
1231
|
-
if (!outcome.resultReceived) {
|
|
1232
|
-
// outranks the transient retry below even when the recovery time is
|
|
1233
|
-
// unknown (cubic review catch): a depleted pool explains the failure,
|
|
1234
|
-
// and respawning against it would just burn doomed attempts.
|
|
1235
|
-
let probeDepleted = false;
|
|
1236
|
-
try {
|
|
1237
|
-
const verdict = await ensureBestAccount();
|
|
1238
|
-
probeDepleted = verdict.reason === "all-depleted" || verdict.reason === "depleted-wait";
|
|
1239
|
-
const wake = probeDepleted ? verdict.waitUntil ?? null : null;
|
|
1240
|
-
if (wake != null) {
|
|
1241
|
-
outcome.rateLimited = true;
|
|
1242
|
-
outcome.deferUntil = wake + PARK_GRACE_MS;
|
|
1243
|
-
pendingFailureLine = null;
|
|
1244
|
-
log("serve.turn_failed_depleted_defer", { resumeAt: outcome.deferUntil });
|
|
1245
|
-
await notify(`the account pool is exhausted - pausing this turn; it will resume automatically in ${inWord(outcome.deferUntil)}.`);
|
|
1246
|
-
}
|
|
1247
|
-
} catch (e) {
|
|
1248
|
-
// keep the original failure; the probe must never mask it.
|
|
1249
|
-
log("serve.defer_probe_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
|
|
1250
|
-
}
|
|
1251
|
-
// a transient non-limit failure (child crash, API blip, errored
|
|
1252
|
-
// result) against a usable pool retries silently into the same
|
|
1253
|
-
// session, mirroring the rate-limit path's invisible short retry
|
|
1254
|
-
// (2026-07-27 report: these turns died terminally, telling the user
|
|
1255
|
-
// "trying again may help" instead of trying again). resultReceived
|
|
1256
|
-
// stays terminal above - a completed answer must never re-run - and
|
|
1257
|
-
// the probe's deferral outranks a retry: an exhausted pool explains
|
|
1258
|
-
// the failure better than "transient".
|
|
1259
|
-
if (!probeDepleted && outcome.deferUntil === null && transientRetries < MAX_TRANSIENT_RETRIES) {
|
|
1260
|
-
transientRetries += 1;
|
|
1261
|
-
breakSegment();
|
|
1262
|
-
log("serve.transient_retry", { attempt: transientRetries });
|
|
1263
|
-
// once a session exists, the failed attempt may have executed
|
|
1264
|
-
// side-effectful tools before dying, so the retry must CONTINUE,
|
|
1265
|
-
// never re-instruct: replaying the original prompt verbatim into
|
|
1266
|
-
// the resumed session reads as "do it again" (codex review catch).
|
|
1267
|
-
// The wrapper mirrors resumeDecision's deferral resume. A pre-init
|
|
1268
|
-
// death ran nothing, so the original replays verbatim there.
|
|
1269
|
-
if (outcome.sessionId !== null) {
|
|
1270
|
-
prompt = `Your previous turn was interrupted by an error mid-run; this session's transcript already holds any work it completed, including tool calls whose side effects already happened. Pick up exactly where you left off and finish the task without re-running completed side-effectful steps. If the work was already complete, just summarize the final state. The original request was:\n\n${input.prompt}`;
|
|
1271
|
-
}
|
|
1272
|
-
if (!(await sleep(RETRY_DELAY_MS))) {
|
|
1273
|
-
// a drain aborted the retry sleep: same as a killed child, the
|
|
1274
|
-
// marker survives (presumedKilled) and the next daemon start
|
|
1275
|
-
// resumes the session where it stopped.
|
|
1276
|
-
await notify("tokenmaxxing is restarting - this turn resumes after the restart.");
|
|
1277
|
-
break;
|
|
1278
|
-
}
|
|
1279
|
-
continue;
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
if (pendingFailureLine !== null) {
|
|
1283
|
-
// same fence-reopen bypass as notify: a diagnostic line opening a
|
|
1284
|
-
// fresh segment must not render inside a reopened code fence.
|
|
1285
|
-
pendingReopenFence = false;
|
|
1286
|
-
await push(transientRetries > 0 ? `${pendingFailureLine} (after ${transientRetries + 1} attempts)` : pendingFailureLine);
|
|
1287
|
-
}
|
|
1288
|
-
break;
|
|
1289
|
-
}
|
|
1290
|
-
if (recoveries >= MAX_RECOVERIES) {
|
|
1291
|
-
// out of in-handler retry budget: defer to the pool's own recovery
|
|
1292
|
-
// clock when it is known, drop honestly when it is not (a stale cache
|
|
1293
|
-
// claiming a usable pool while every retry limits out lands here too,
|
|
1294
|
-
// and deferring on no evidence would just spin the resume cap).
|
|
1295
|
-
let wake: number | null = null;
|
|
1296
|
-
try {
|
|
1297
|
-
const verdict = await ensureBestAccount();
|
|
1298
|
-
const depleted = verdict.reason === "all-depleted" || verdict.reason === "depleted-wait";
|
|
1299
|
-
wake = depleted ? verdict.waitUntil ?? null : null;
|
|
1300
|
-
} catch (e) {
|
|
1301
|
-
log("serve.defer_probe_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
|
|
1302
|
-
}
|
|
1303
|
-
if (wake != null) {
|
|
1304
|
-
outcome.deferUntil = wake + PARK_GRACE_MS;
|
|
1305
|
-
log("serve.rate_limited_defer", { recoveries, resumeAt: outcome.deferUntil });
|
|
1306
|
-
await notify(`still at a usage limit after retries - pausing this turn; it will resume automatically in ${inWord(outcome.deferUntil)}.`);
|
|
1307
|
-
break;
|
|
1308
|
-
}
|
|
1309
|
-
log("serve.rate_limited_drop", { recoveries });
|
|
1310
|
-
outcome.announcedDrop = await notifyDelivered("still at a usage limit after retries - this message was dropped; reply when you want to try again.");
|
|
1311
|
-
break;
|
|
1312
|
-
}
|
|
1313
|
-
// a limit the cached pool state did not predict: give the pool one beat
|
|
1314
|
-
// to observe it, then re-decide and retry the same prompt into the same
|
|
1315
|
-
// session (slaude's silent short retry - a successful swap makes it
|
|
1316
|
-
// invisible in the thread).
|
|
1317
|
-
recoveries += 1;
|
|
1318
|
-
breakSegment();
|
|
1319
|
-
log("serve.rate_limited_retry", { recoveries });
|
|
1320
|
-
// wait out an active post-swap cooldown too: a swap-then-instant-limit
|
|
1321
|
-
// would otherwise burn every retry inside the 45s window where the
|
|
1322
|
-
// decision refuses to re-evaluate, respawning the same limited account
|
|
1323
|
-
// (review catch, PR #18). The persisted observation then makes the
|
|
1324
|
-
// post-cooldown decision see the depleted account immediately.
|
|
1325
|
-
// loadLastSwapAt throws on a corrupt swap clock; every failure in this
|
|
1326
|
-
// loop must settle the turn in-thread (announced, never a bare throw),
|
|
1327
|
-
// same as the ensureBestAccount guard above (review catch, PR #31).
|
|
1328
|
-
let cooldownUntil: number;
|
|
1329
|
-
try {
|
|
1330
|
-
cooldownUntil = (loadLastSwapAt() ?? 0) + POST_SWAP_COOLDOWN_MS + 1_000;
|
|
1331
|
-
} catch (e) {
|
|
1332
|
-
outcome.failed = true;
|
|
1333
|
-
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
1334
|
-
log("serve.turn_error", { err: detail });
|
|
1335
|
-
await push(`tokenmaxxing: turn failed: ${detail}`);
|
|
1336
|
-
break;
|
|
1337
|
-
}
|
|
1338
|
-
if (!(await sleep(Math.max(RETRY_DELAY_MS, cooldownUntil - Date.now())))) {
|
|
1339
|
-
// a drain aborted the retry sleep: same as a killed child, the marker
|
|
1340
|
-
// survives (presumedKilled) and the next daemon start resumes the
|
|
1341
|
-
// session where it stopped.
|
|
1342
|
-
outcome.failed = true;
|
|
1343
|
-
await notify("tokenmaxxing is restarting - this turn resumes after the restart.");
|
|
1344
|
-
break;
|
|
1345
|
-
}
|
|
1346
|
-
}
|
|
1347
|
-
await settlePosts();
|
|
1348
|
-
// Reply text died with a rejected segment, salvage could not re-deliver it
|
|
1349
|
-
// (budget exhausted or the salvage posts died too), and nothing later
|
|
1350
|
-
// re-delivered it: the answer silently vanished while the outcome would
|
|
1351
|
-
// report success. Fail the turn and make one best-effort fresh-message
|
|
1352
|
-
// diagnostic.
|
|
1353
|
-
if (textLost) {
|
|
1354
|
-
outcome.failed = true;
|
|
1355
|
-
const detail = textLostDetail ?? "unknown error";
|
|
1356
|
-
await input
|
|
1357
|
-
.post((async function* () {
|
|
1358
|
-
yield `tokenmaxxing: the reply could not be posted to Slack: ${detail}`;
|
|
1359
|
-
})())
|
|
1360
|
-
.catch((e: unknown) => log("serve.post_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) }));
|
|
1361
|
-
}
|
|
1362
|
-
return outcome;
|
|
1363
|
-
}
|