tokenmaxxing 1.6.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/DESIGN.md +2 -30
  2. package/LICENSE +21 -0
  3. package/README.md +1 -2
  4. package/agent-plugin/bin/tokenmaxxing-mcp +7 -0
  5. package/agent-plugin/mcp.json +10 -0
  6. package/agent-plugin/plugin.json +14 -0
  7. package/agent-plugin/skills/codex-pool/SKILL.md +23 -0
  8. package/agent-plugin/skills/codex-pool/references/codex.md +5 -0
  9. package/agent-plugin/skills/credentials-hygiene/SKILL.md +26 -0
  10. package/agent-plugin/skills/credentials-hygiene/references/credentials.md +6 -0
  11. package/agent-plugin/skills/doctor-diagnostics/SKILL.md +26 -0
  12. package/agent-plugin/skills/doctor-diagnostics/references/troubleshooting.md +5 -0
  13. package/agent-plugin/skills/pool-status/SKILL.md +27 -0
  14. package/agent-plugin/skills/pool-status/references/commands.md +8 -0
  15. package/agent-plugin/skills/safe-contribution/SKILL.md +27 -0
  16. package/agent-plugin/skills/safe-contribution/references/ship.md +5 -0
  17. package/agent-plugin/skills/sdk-pairing/SKILL.md +33 -0
  18. package/agent-plugin/skills/sdk-pairing/references/sdk.md +6 -0
  19. package/agent-plugin/skills/switching-policy/SKILL.md +29 -0
  20. package/agent-plugin/skills/switching-policy/references/policy.md +7 -0
  21. package/package.json +3 -5
  22. package/src/entries/mcp.ts +288 -0
  23. package/src/lib/decide.ts +2 -4
  24. package/src/lib/lock.ts +3 -7
  25. package/src/lib/log.ts +8 -11
  26. package/src/lib/paths.ts +0 -9
  27. package/src/lib/usage.ts +6 -5
  28. package/src/main.ts +1 -6
  29. package/src/cli/serve.ts +0 -1790
  30. package/src/lib/slackbridge.ts +0 -1363
  31. package/src/lib/slackstate.ts +0 -352
  32. package/src/lib/slackstream.ts +0 -300
  33. package/src/serve-plugin/.claude-plugin/plugin.json +0 -4
  34. package/src/serve-plugin/skills/ask-the-user/SKILL.md +0 -41
  35. package/src/serve-plugin/skills/serve-session/SKILL.md +0 -50
@@ -1,1363 +0,0 @@
1
- // The Slack->claude relay. One claude turn per Slack message via the Agent SDK
2
- // (re-query with resume, never a persistent streaming query: the SDK subprocess
3
- // reads credentials at spawn, so per-turn spawns are what let the pool decision
4
- // pick the freshest account at every boundary and let the daemon restart
5
- // without losing threads). Verified against @anthropic-ai/claude-agent-sdk
6
- // 0.3.214 and code.claude.com/docs 2026-07-18; both change monthly.
7
-
8
- import { spawn } from "node:child_process";
9
- import { join } from "node:path";
10
- import { z } from "zod";
11
- import { delay } from "es-toolkit";
12
- import { createSdkMcpServer, query, tool, type SDKUserMessage, type SpawnOptions } from "@anthropic-ai/claude-agent-sdk";
13
- import { StreamingMarkdownRenderer, type StreamChunk } from "chat";
14
- import { ensureBestAccount, pooledOptions, stopHookCheck, type SwapDecision } from "../sdk.ts";
15
- import { POST_SWAP_COOLDOWN_MS } from "./decide.ts";
16
- import { readOAuthAccount } from "./claudejson.ts";
17
- import { http, safeErrorDetail } from "./http.ts";
18
- import { loadLastSwapAt } from "./state.ts";
19
- import { fmtResetShort, recordObservedLimit } from "./usage.ts";
20
- import { deleteSlackThread, type SlackLink } from "./slackstate.ts";
21
- import { agentEventChunks, newStreamMapState } from "./slackstream.ts";
22
- import { log } from "./log.ts";
23
-
24
- /** With systemPrompt omitted the SDK runs a MINIMAL system prompt (the
25
- * claude_code preset is opt-in since SDK 0.1.0, re-verified for 0.3.214
26
- * 2026-07-18), so this small standalone prompt replaces nothing. It exists
27
- * because a relayed model once answered with a literal "<br>": Slack renders
28
- * markdown, never HTML. */
29
- const SLACK_SYSTEM_PROMPT =
30
- "Your replies are relayed into a Slack thread and render as Slack-flavored markdown. Write plain markdown only - never HTML tags such as <br> (use real line breaks).";
31
-
32
- export const TurnOutcomeSchema = z.object({
33
- sessionId: z.string().nullable(),
34
- failed: z.boolean(),
35
- /** the FAILED turn hit a usage/rate limit (or the pool was depleted before
36
- * it could spawn). Only an errored result is ever limit-classified. */
37
- rateLimited: z.boolean(),
38
- /** the model called finish_thread this turn: garbage-collect after the turn. */
39
- finish: z.boolean(),
40
- /** the model called need_attention this turn (it asked the user for a
41
- * decision): the daemon marks the thread waiting - question-mark status
42
- * reaction, a one-time nudge if the user stays quiet, and an asked user's
43
- * reaction relays as their answer. */
44
- attention: z.boolean(),
45
- /** relayThread posted a TERMINAL drop notice for this message ("dropped;
46
- * re-send it"): a drain must NOT presume a killed child and retain the
47
- * resume marker, or startup replays work the user was told to resend. */
48
- announcedDrop: z.boolean(),
49
- /** the LAST attempt's claude child ran to a SUCCESSFUL result: the work is
50
- * done even if Slack delivery later failed (textLost sets failed for the
51
- * operator's benefit). A drain must not read that delivery failure as a
52
- * killed child and re-run completed work (adversarial-review catch). */
53
- resultReceived: z.boolean(),
54
- /** epoch ms when the pool is expected usable again: set when the turn ended
55
- * at a usage limit whose recovery time is known. The caller keeps the
56
- * thread's activeTurn marker with resumeAt and the daemon resumes the turn
57
- * itself, instead of the old "re-send it once the pool recovers" drop. */
58
- deferUntil: z.number().nullable(),
59
- /** a steered follow-up's own drained turn FAILED after the primary turn
60
- * already succeeded (success stays sticky, the loss is announced
61
- * in-thread): the caller settles the steered messages' reactions as
62
- * failed, never as done - a lost instruction must not read green.
63
- * Attribution is per-turn, not per-message (steer() carries text only),
64
- * so when several messages were steered a successfully folded one can
65
- * read failed too - accepted: a false re-send ask beats a false green. */
66
- steerLost: z.boolean(),
67
- });
68
- export type TurnOutcome = z.infer<typeof TurnOutcomeSchema>;
69
-
70
- // ---- depleted-pool recovery policy (pure, unit-tested) ---------------------
71
-
72
- /** TOTAL parking budget for one Slack message (a single deadline across all
73
- * its parks, not per park): the parked handler holds the thread's queue slot,
74
- * so recovery further out gets an honest drop notice instead of a hostage
75
- * handler, and the daemon's queue-entry TTL is sized to outlast a full
76
- * park + turn so follow-ups fold instead of silently expiring. */
77
- export const PARK_MAX_MS = 840_000;
78
- /** spawn slightly after the reset passes, never right on the boundary. */
79
- const PARK_GRACE_MS = 5_000;
80
- /** post-limit short retry: one beat for the pool to observe the limit and
81
- * swap (slaude's parkShortRetry - a successful swap makes it invisible). */
82
- const RETRY_DELAY_MS = 10_000;
83
- /** parks + retries per Slack message; keeps a stale usage cache from looping
84
- * a thread forever. */
85
- export const MAX_RECOVERIES = 3;
86
- /** extra spawns for a NON-limit failure whose child never completed (crash,
87
- * API blip, errored result): each resumes the same session, so a retry
88
- * continues the turn instead of re-running it. Separate from MAX_RECOVERIES
89
- * on purpose - that budget bounds limit-driven parking, this one bounds
90
- * quota spent chasing a possibly-permanent error. */
91
- export const MAX_TRANSIENT_RETRIES = 2;
92
- /** Slack expires a native stream SERVER-SIDE on undocumented timers
93
- * (Slack-maintainer-confirmed in slackapi/python-slack-sdk#1859: idle
94
- * around 30s, total lifetime around 300s measured), and an expired stream
95
- * freezes in the Slack client as a grey "Something went wrong" pill - the
96
- * exact 2026-07-27 report. The salvage path recovers the CONTENT but
97
- * cannot un-freeze the pill, so the fix is to never let Slack expire a
98
- * stream we own: rotate the open segment - a clean end() that the adapter
99
- * finishes with a proper stream stop - before either timer can fire, and
100
- * let the next chunk open a fresh message, the same flow pushText's size
101
- * splits already use. Thresholds sit well inside Slack's observed margins;
102
- * read at every arm (a mutable object, the file's test seam pattern). */
103
- export const SEGMENT_ROTATION = { idleMs: 20_000, maxAgeMs: 240_000 };
104
-
105
- const ParkPlanSchema = z.union([
106
- z.object({ kind: z.literal("proceed") }),
107
- z.object({ kind: z.literal("park"), wakeAt: z.number() }),
108
- z.object({ kind: z.literal("defer"), resumeAt: z.number() }),
109
- z.object({ kind: z.literal("drop") }),
110
- ]);
111
- export type ParkPlan = z.infer<typeof ParkPlanSchema>;
112
-
113
- /** What to do with a spawn-boundary switch decision: proceed on a usable pool;
114
- * park in-handler until the soonest recovery when it lands inside the
115
- * message's one shared deadline; DEFER when recovery is known but further
116
- * out (or the in-handler recovery budget is spent): the handler releases the
117
- * queue slot and the daemon resumes the turn from its durable marker once
118
- * the pool recovers (2026-07-20 incident: dropped messages sat dead for
119
- * hours after the pool recovered until the user re-sent them by hand -
120
- * superseding the older drop-instead-of-promise rule, whose premise was that
121
- * a will-resume promise could not be kept; the marker + scheduler + startup
122
- * scan make it durable). Only an UNKNOWN recovery time still drops honestly.
123
- * The deadline is fixed when the message's relay starts, so chained parks
124
- * can never hold the queue slot longer than PARK_MAX_MS in total. */
125
- export function parkPlan(input: { decision: SwapDecision; recoveries: number; deadline: number }): ParkPlan {
126
- const depleted = input.decision.reason === "all-depleted" || input.decision.reason === "depleted-wait";
127
- if (!depleted) return { kind: "proceed" };
128
- const wake = input.decision.waitUntil ?? null;
129
- if (wake == null) return { kind: "drop" };
130
- // the grace counts against the deadline too: the promised total hold is
131
- // exact, not deadline-plus-grace (review catch, PR #18).
132
- if (wake + PARK_GRACE_MS > input.deadline || input.recoveries >= MAX_RECOVERIES) {
133
- return { kind: "defer", resumeAt: wake + PARK_GRACE_MS };
134
- }
135
- return { kind: "park", wakeAt: wake + PARK_GRACE_MS };
136
- }
137
-
138
- /** Phrases claude's ERRORED results carry at a usage/rate limit (ported from
139
- * slaude's battle-tested set). Checked only against errored results: a
140
- * successful answer that merely discusses usage limits (routine in this
141
- * repo's own threads) must never be discarded and re-run. */
142
- const RATE_LIMIT_PHRASES = [
143
- "usage limit reached",
144
- "rate limit reached",
145
- "rate limit exceeded",
146
- "rate limit hit",
147
- "hit your usage limit",
148
- "hit your weekly limit",
149
- "limit will reset",
150
- "5-hour limit",
151
- "out of extra usage",
152
- ];
153
-
154
- export function isRateLimitText(input: { text: string }): boolean {
155
- const lower = input.text.toLowerCase();
156
- return RATE_LIMIT_PHRASES.some((phrase) => lower.includes(phrase));
157
- }
158
-
159
- /** The CLI puts an errored result's reason in `result` even on non-success
160
- * subtypes (the exact shape PingResultSchema in usage.ts handles), while the
161
- * SDK's error type declares only `errors` - so limit text is gathered
162
- * loosely from both fields. */
163
- const ResultTextSchema = z.looseObject({
164
- result: z.string().optional(),
165
- errors: z.array(z.string()).optional(),
166
- });
167
-
168
- function erroredResultText(message: unknown): string {
169
- const parsed = ResultTextSchema.safeParse(message);
170
- if (!parsed.success) return "";
171
- return [parsed.data.result, ...(parsed.data.errors ?? [])]
172
- .filter((t): t is string => t != null && t !== "")
173
- .join("\n");
174
- }
175
-
176
- // ---- workspace identity ----------------------------------------------------
177
-
178
- const AuthTestSchema = z.looseObject({
179
- ok: z.boolean(),
180
- team_id: z.string().optional(),
181
- error: z.string().optional(),
182
- });
183
-
184
- /** auth.test: the home workspace (team) id the bot token belongs to - the
185
- * reference `isOutsideAuthor` compares message origins against. Errors carry
186
- * the Slack error code only, never the token. */
187
- export async function fetchWorkspaceTeamId(input: { botToken: string }): Promise<string> {
188
- const res = await http
189
- .post("https://slack.com/api/auth.test", {
190
- headers: { authorization: `Bearer ${input.botToken}` },
191
- })
192
- .catch((e: unknown) => {
193
- // a thrown ky error (timeout, network) carries its Request with the
194
- // Authorization header - rethrow message-only so no caller can ever
195
- // log the token.
196
- throw new Error(`Slack auth.test failed: ${e instanceof Error ? e.message : String(e)}`);
197
- });
198
- const text = await res.text();
199
- if (!res.ok) throw new Error(`Slack auth.test failed: HTTP ${res.status} (${safeErrorDetail({ text })})`);
200
- const body: unknown = (() => {
201
- try {
202
- return JSON.parse(text);
203
- } catch {
204
- return null;
205
- }
206
- })();
207
- const parsed = AuthTestSchema.safeParse(body);
208
- if (!parsed.success || !parsed.data.ok || !parsed.data.team_id) {
209
- throw new Error(
210
- `Slack auth.test failed: ${parsed.success ? (parsed.data.error ?? "no team_id in response") : "unrecognized response"}`,
211
- );
212
- }
213
- return parsed.data.team_id;
214
- }
215
-
216
- /** The serve plugin shipped inside the package (src/serve-plugin/): skills
217
- * that teach a relayed session how to behave in a Slack thread, loaded per
218
- * turn via the SDK's local-plugin option and namespaced `tokenmaxxing:...`. */
219
- const SERVE_PLUGIN_DIR = join(import.meta.dir, "..", "serve-plugin");
220
-
221
- /**
222
- * The per-turn context a UserPromptSubmit hook injects. The skills are static
223
- * files, so the one dynamic fact they cannot carry - WHO asked - rides in
224
- * here as the requester's raw mention token (`<@U...>` passes verbatim
225
- * through the streamed markdown_text path, and the post-and-edit fallback's
226
- * finalize leaves an already-formed mention intact). Wording is load-bearing:
227
- * the ask-the-user skill points at the "Slack relay context" note.
228
- */
229
- export function serveTurnContext(input: { requesterIds: string[] }): string {
230
- const tokens = input.requesterIds.map((id) => `<@${id}>`);
231
- let requester = "The requesting user is unknown this turn, so no mention token is available.";
232
- if (tokens.length === 1) requester = `The requesting user's Slack mention token is ${tokens[0]}; include it literally in reply text to notify them.`;
233
- if (tokens.length > 1) requester = `This turn folds messages from several users; their Slack mention tokens are ${tokens.join(" ")}. Include the relevant user's token literally in reply text to notify them.`;
234
- return [
235
- "Slack relay context: this session is relayed into a Slack thread by tokenmaxxing serve, and your reply posts back into the thread.",
236
- requester,
237
- "When you need the user's decision, approval, or input, follow the tokenmaxxing:ask-the-user skill (tag them, ask, end the turn).",
238
- "The tokenmaxxing:serve-session skill explains how this session runs.",
239
- ].join(" ");
240
- }
241
-
242
- const SegmentChunkSchema = z.union([z.string(), z.custom<StreamChunk>()]);
243
- type SegmentChunk = z.infer<typeof SegmentChunkSchema>;
244
-
245
- /** The exact user-message shape the SDK's own string-prompt path writes to the
246
- * child's stdin (verified in @anthropic-ai/claude-agent-sdk 0.3.214: the SDK
247
- * serializes yielded messages verbatim, adding nothing). No uuid on purpose:
248
- * the CLI dedupes stream-json user messages by uuid and silently swallows a
249
- * reused one, so a uuid derived from a Slack message ts would eat retries.
250
- * No priority either: the default "next" folds the message into the running
251
- * turn at the next tool boundary, while "now" is an undocumented hard
252
- * interrupt that would abort the turn mid-tool (both verified in the claude
253
- * 2.1.220 binary). */
254
- function steerUserMessage(text: string): SDKUserMessage {
255
- return { type: "user", session_id: "", message: { role: "user", content: [{ type: "text", text }] }, parent_tool_use_id: null };
256
- }
257
-
258
- /** A hand-pushed async iterable of SDK user messages: the streaming-input
259
- * prompt for one query attempt. The initial prompt is pushed before query()
260
- * and steered messages join mid-turn; end() closes the child's stdin (the
261
- * SDK ends the stream when the iterable finishes). */
262
- function pushableMessages(): {
263
- iterable: AsyncIterable<SDKUserMessage>;
264
- push: (m: SDKUserMessage) => void;
265
- end: () => void;
266
- } {
267
- const queued: SDKUserMessage[] = [];
268
- let cursor = 0;
269
- let done = false;
270
- let notify: (() => void) | null = null;
271
- return {
272
- push(m) {
273
- queued.push(m);
274
- notify?.();
275
- },
276
- end() {
277
- done = true;
278
- notify?.();
279
- },
280
- iterable: {
281
- async *[Symbol.asyncIterator]() {
282
- while (true) {
283
- while (cursor < queued.length) {
284
- const next = queued[cursor]!;
285
- cursor += 1;
286
- yield next;
287
- }
288
- if (done) return;
289
- await new Promise<void>((resolve) => {
290
- notify = resolve;
291
- });
292
- notify = null;
293
- }
294
- },
295
- },
296
- };
297
- }
298
-
299
- /** A hand-pushed async iterable: relayThread feeds one of these per Slack
300
- * message segment while thread.post concurrently drains it. */
301
- function pushableStream(): {
302
- iterable: AsyncIterable<SegmentChunk>;
303
- push: (chunk: SegmentChunk) => void;
304
- end: () => void;
305
- ledger: () => { chunks: SegmentChunk[]; confirmed: number };
306
- } {
307
- const chunks: SegmentChunk[] = [];
308
- let cursor = 0;
309
- let confirmed = 0;
310
- let done = false;
311
- let notify: (() => void) | null = null;
312
- return {
313
- push(chunk) {
314
- chunks.push(chunk);
315
- notify?.();
316
- },
317
- end() {
318
- done = true;
319
- notify?.();
320
- },
321
- /** Everything ever pushed plus how much of it the consumer PROVED
322
- * delivered. The Slack adapter pulls one chunk, awaits its Slack append,
323
- * then pulls the next (verified in @chat-adapter/slack 4.34.0 stream()),
324
- * so each pull confirms the append for the previous chunk landed; a
325
- * consumer that dies mid-append unwinds the for-await with an implicit
326
- * return() at the yield, leaving that chunk and everything after it
327
- * unconfirmed. */
328
- ledger() {
329
- return { chunks: [...chunks], confirmed };
330
- },
331
- iterable: {
332
- async *[Symbol.asyncIterator]() {
333
- while (true) {
334
- while (cursor < chunks.length) {
335
- const next = chunks[cursor]!;
336
- cursor += 1;
337
- yield next;
338
- confirmed = cursor;
339
- }
340
- if (done) return;
341
- await new Promise<void>((resolve) => {
342
- notify = resolve;
343
- });
344
- notify = null;
345
- }
346
- },
347
- },
348
- };
349
- }
350
-
351
- /** Slack rejects an over-long message with msg_too_long, and NOTHING in chat
352
- * 4.34.0 or the Slack adapter bounds, truncates, or splits reply text
353
- * (verified in-source 2026-07-20): a natively streamed message accumulates
354
- * server-side toward the 12,000-char markdown_text envelope (docs.slack.dev
355
- * documents that limit on chat.postMessage/update and all three streaming
356
- * methods), the post-and-edit fallback re-sends the FULL accumulated text as
357
- * markdown_text on every edit, and once anything rendered natively a failed
358
- * append REJECTS the whole thread.post - the reply dies (live incident
359
- * 2026-07-20, three failed turns). relayThread therefore splits reply text
360
- * across Slack messages BEFORE the cap; the 2,000-char margin absorbs the
361
- * renderer's markdown normalization and mention-linkification expansion.
362
- * Tradeoff (accepted): the adapter-internal plain-text fallback edits via
363
- * chat.update `text` (hard 4,000-char cap), but it only engages when the
364
- * workspace refused native streaming outright - splitting every normal reply
365
- * 3x tighter to cover that never-hit path is worse than the residual risk. */
366
- export const SEGMENT_TEXT_MAX = 10_000;
367
-
368
- /** Full permission names of the in-process tools (mcp__<server>__<tool>):
369
- * they must be in allowedTools, because no one can answer a permission
370
- * prompt through Slack. */
371
- const FINISH_THREAD_TOOL = "mcp__tokenmaxxing__finish_thread";
372
- const NEED_ATTENTION_TOOL = "mcp__tokenmaxxing__need_attention";
373
-
374
- /** The per-turn in-process MCP server exposing finish_thread and
375
- * need_attention. The handlers run in the daemon process, but they must NOT
376
- * act inline: the claude subprocess is still mid-turn and segments are still
377
- * streaming to Slack, so each only records the request and the daemon acts
378
- * after the turn ends (serve.ts). alwaysLoad keeps the tools visible in the
379
- * prompt instead of deferred behind tool search: they have to be in view at
380
- * the exact moment the user says the work is done or the model hits a fork. */
381
- function serveToolServer(input: { onFinish: () => void; onAttention: () => void }) {
382
- return createSdkMcpServer({
383
- name: "tokenmaxxing",
384
- alwaysLoad: true,
385
- tools: [
386
- tool(
387
- "finish_thread",
388
- "Close out this Slack thread when the user clearly states the work is finished (shipped, done, clean this up) and wants the thread closed. After this turn ends the daemon drops the thread's session record, unsubscribes, and posts a confirmation; the repo checkout and everything in it are untouched. Do not call this for a merely answered question - only for an explicit wrap-up.",
389
- {},
390
- async () => {
391
- input.onFinish();
392
- return { content: [{ type: "text", text: "close-out scheduled - it runs right after this turn ends and posts its own confirmation; just acknowledge the wrap-up now" }] };
393
- },
394
- ),
395
- tool(
396
- "need_attention",
397
- "Flag this Slack thread as waiting on the requesting user. Call it in the same turn in which you ask them for a decision, approval, or missing information (the ask-the-user skill), then ask in your reply text with their mention token and end the turn. After the turn ends the daemon marks the thread attention-needed (a question-mark reaction on the triggering message) and tags the user once more if they stay quiet; a reaction from them, such as a thumbs up, relays back to you as their answer. Do not call this for rhetorical questions or ordinary replies.",
398
- {},
399
- async () => {
400
- input.onAttention();
401
- return { content: [{ type: "text", text: "attention flagged - after this turn ends the daemon marks the thread as waiting on the user and nudges them if they stay quiet; ask your question in the reply text with their mention token, then end the turn" }] };
402
- },
403
- ),
404
- ],
405
- });
406
- }
407
-
408
- const CleanupOutcomeSchema = z.object({
409
- /** the thread's state is gone; a fresh @mention starts a new session. */
410
- removed: z.boolean(),
411
- message: z.string(),
412
- });
413
- export type CleanupOutcome = z.infer<typeof CleanupOutcomeSchema>;
414
-
415
- /**
416
- * Close out a finished thread. Threads run IN the linked repo checkout (no
417
- * per-thread worktree or branch since #14), so there is nothing on disk to
418
- * collect: dropping the slack-threads record is the whole cleanup, and the
419
- * shared checkout is never touched. The worktree-era residue gate and branch
420
- * archiving died with the worktrees themselves.
421
- */
422
- export function cleanupThread(input: { threadId: string }): CleanupOutcome {
423
- deleteSlackThread(input.threadId);
424
- return { removed: true, message: "thread finished - session closed; a fresh @mention here starts a new one" };
425
- }
426
-
427
- /** Live detached process-group leader pids: one process-exit hook SIGTERMs
428
- * them all, so a forced daemon exit (second signal, drain timeout) cannot
429
- * leak claude's tool subprocesses. */
430
- const liveGroups = new Set<number>();
431
- let groupExitHookArmed = false;
432
-
433
- /** Exported for serve's orphan reaping: a daemon killed uncatchably (SIGKILL,
434
- * crash) never runs the exit hook below, so the next generation must be able
435
- * to terminate a surviving detached group before resuming its turn. */
436
- export function killGroup(pid: number, signal: "SIGTERM" | "SIGKILL" = "SIGTERM"): void {
437
- try {
438
- process.kill(-pid, signal);
439
- } catch (e) {
440
- // ESRCH = the group is already gone, which is the state we wanted;
441
- // anything else (EPERM, a bad pid) must surface, not silently leak.
442
- if (!(e instanceof Error && "code" in e && e.code === "ESRCH")) throw e;
443
- }
444
- }
445
-
446
- /**
447
- * Spawns the claude child in its OWN process group (post-0.19.1 review catch):
448
- * a terminal Ctrl-C delivers SIGINT to the whole foreground group, so a
449
- * non-detached child died at the same instant the daemon's drain started and
450
- * the drain could never preserve the in-flight turn. Detached, only the daemon
451
- * receives the terminal signal. Two consequences the review on PR #16 caught:
452
- * the SDK's SpawnedProcess contract consumes only stdin/stdout, so stderr must
453
- * be ignored outright (a piped-but-never-read stderr fills and blocks a chatty
454
- * child; exit errors lose the stderr tail, an accepted cost of turn survival),
455
- * and the SDK's abort path kills the lone PID, so the forwarded abort signal
456
- * and a process-exit hook SIGTERM the whole detached group instead - claude's
457
- * tool subprocesses must not outlive the daemon or the turn.
458
- */
459
- export function detachedClaudeSpawn(options: SpawnOptions) {
460
- const stdio: ["pipe", "pipe", "ignore"] = ["pipe", "pipe", "ignore"];
461
- const child = spawn(options.command, options.args, {
462
- cwd: options.cwd,
463
- env: options.env,
464
- stdio,
465
- detached: true,
466
- });
467
- if (!groupExitHookArmed) {
468
- groupExitHookArmed = true;
469
- process.once("exit", () => {
470
- for (const pid of liveGroups) killGroup(pid);
471
- });
472
- }
473
- if (child.pid !== undefined) {
474
- const pid = child.pid;
475
- liveGroups.add(pid);
476
- const onAbort = () => killGroup(pid);
477
- // an already-aborted signal never fires "abort" again (cubic review
478
- // catch): a cancellation racing the spawn must still kill the group.
479
- if (options.signal?.aborted) onAbort();
480
- else options.signal?.addEventListener("abort", onAbort, { once: true });
481
- child.once("exit", () => {
482
- liveGroups.delete(pid);
483
- options.signal?.removeEventListener("abort", onAbort);
484
- });
485
- }
486
- return child;
487
- }
488
-
489
- /**
490
- * One claude turn relayed into a Slack thread as ONE streamed message: reply
491
- * text streams natively and thinking/tool calls stream as task_update cards
492
- * (see slackstream.ts) that Slack groups into a single collapsible plan block
493
- * (user ask 2026-07-20: "squash them into one dropdown"; the serve edge wraps
494
- * each posted segment in a StreamingPlan with groupTasks "plan", superseding
495
- * the 2026-07-18 separate-messages-around-tool-runs shape). Segments still
496
- * exist as the posting machinery: recovery notices post as their own
497
- * messages, a rejected post opens a fresh one (so an over-long turn that
498
- * trips Slack's message cap degrades to a follow-on message instead of
499
- * vanishing), and segments post strictly in order: the next opens only after
500
- * the previous post resolves. Never throws: a failure posts a short
501
- * diagnostic line and sets outcome.failed (the daemon must keep serving other
502
- * threads). Error text is message-only - a raw error body could echo request
503
- * material.
504
- *
505
- * Depleted-pool recovery (ported from slaude at its shutdown, reshaped around
506
- * the pool): the spawn-boundary switch decision is CONSUMED, not discarded -
507
- * a depleted pool parks BEFORE a doomed spawn burns a failed turn, with an
508
- * honest in-thread notice either way; a mid-turn limit the cached pool state
509
- * did not predict is persisted (recordObservedLimit) and retried silently into
510
- * the same session. Total in-handler parking is bounded by one shared
511
- * PARK_MAX_MS deadline plus MAX_RECOVERIES; a limit whose recovery lands
512
- * beyond that budget DEFERS instead of dropping (outcome.deferUntil): the
513
- * caller keeps the thread's durable marker with resumeAt and the daemon
514
- * resumes the turn itself once the pool recovers (2026-07-20 incident).
515
- * Only an unknown recovery time still drops, and every drop the relay itself
516
- * performs is announced in-thread (a queue-entry TTL expiry upstream is the
517
- * one drop it cannot see).
518
- */
519
- export async function relayThread(input: {
520
- cwd: string;
521
- sessionId: string | null;
522
- prompt: string;
523
- /** bare Slack user id (U...) of the triggering message's author. */
524
- requesterIds: string[];
525
- link: SlackLink;
526
- post: (m: AsyncIterable<SegmentChunk>) => Promise<unknown>;
527
- /** fires the moment an init message assigns a session id the caller has not
528
- * persisted yet, so a first-turn kill stays resumable (2026-07-18
529
- * incident: a restart killed a first turn and the thread record kept
530
- * sessionId null, stranding the session). Retries resume the same session,
531
- * so re-fires only on an actual id change. */
532
- onSessionId?: (sessionId: string) => void;
533
- /** fires with the DETACHED claude child's pid (= its process-group id) the
534
- * moment it spawns - once per spawn, so a retry's fresh child replaces the
535
- * previous pid - so the caller can persist it into the activeTurn marker:
536
- * a daemon death that skips the exit hook (SIGKILL, crash) leaves that
537
- * group alive, and the next generation must find and reap it before
538
- * resuming the turn. */
539
- onSpawn?: (pid: number) => void;
540
- /** daemon shutdown signal: aborts park/retry sleeps so a drain never sits
541
- * out a depleted-pool countdown. */
542
- drainSignal?: AbortSignal;
543
- /** Steering seam. Called with a steer function while a query attempt is
544
- * live and with null when it ends; steer(text) returns true when the text
545
- * was written into the RUNNING attempt's stdin (the CLI folds it into the
546
- * turn at the next tool boundary, or runs it as its own turn in the same
547
- * child when the fold window is gone - either way it reaches the session,
548
- * verified against claude 2.1.220), false once the attempt's result
549
- * arrived or the attempt died - the caller then queues the message as a
550
- * normal next turn instead. Accepted texts also fold into any RETRY
551
- * attempt's prompt, mirroring the existing resend-the-full-prompt retry
552
- * tradeoff (duplication in the resumed transcript beats loss).
553
- * input.requesterIds is shared by reference on purpose: the caller may
554
- * push a steer author's id so a retry attempt's UserPromptSubmit context
555
- * names them (the hook does not fire for folded mid-turn messages). */
556
- onSteer?: (steer: ((text: string, onAccept?: () => void) => boolean) | null) => void;
557
- }): Promise<TurnOutcome> {
558
- const outcome: TurnOutcome = { sessionId: input.sessionId, failed: false, rateLimited: false, finish: false, attention: false, announcedDrop: false, resultReceived: false, deferUntil: null, steerLost: false };
559
- let segment: ReturnType<typeof pushableStream> | null = null;
560
- // acc mirrors the segment's pushed text (bounded by SEGMENT_TEXT_MAX plus a
561
- // small overshoot): fence parity must be computed over the ACCUMULATED text,
562
- // never per chunk - SDK deltas do not respect markdown token boundaries, so
563
- // a ``` split across two deltas would be invisible to per-chunk counting
564
- // (pullfrog review catch, PR #42).
565
- let segmentMeta: { text: boolean; acc: string; reopenFence: boolean } | null = null;
566
- // the open segment's rotation clock (see armSegmentTimer).
567
- let segmentTimer: ReturnType<typeof setTimeout> | null = null;
568
- let segmentOpenedAt = 0;
569
- // a timer rotation closed the segment inside a code fence: the next
570
- // segment must reopen it, exactly like pushText's size splits do (codex
571
- // review catch). Consumed by openSegment; the salvage handler's own
572
- // delivered-prefix recomputation overrides it there.
573
- let pendingReopenFence = false;
574
- let lastPost: Promise<unknown> = Promise.resolve();
575
- // Reply TEXT that died with a rejected segment and was neither salvaged into
576
- // a follow-on message nor re-delivered by a later text-bearing segment: the
577
- // user has not seen the answer. A lost card-only segment never sets this
578
- // (decoration, not the answer). Tradeoff (flagged and accepted): a later
579
- // delivered text segment clears the flag even though it is a continuation,
580
- // because a rejection whose text chunks were all consumed pre-append-failure
581
- // was almost certainly delivered (the adapter appends per chunk) except for
582
- // an unobservable renderer-held tail; sticky loss would fail every long turn
583
- // with a spurious diagnostic.
584
- let textLost = false;
585
- let textLostDetail: string | null = null;
586
- let postedText = false;
587
- // Salvage: a rejected post's undelivered chunks re-post as a fresh message
588
- // (Slack finalizes an idle stream after an UNDOCUMENTED window - verified
589
- // absent from the chat.startStream/appendStream docs 2026-07-21 - so
590
- // recovery is reactive on any append failure, never a keepalive tuned to a
591
- // guessed constant). The budget bounds FUTILITY, not recovery: a death
592
- // after the message delivered something is progress and refills it (a long
593
- // turn can outlive any number of idle finalizations, each losing only the
594
- // gap tail), while a surface that delivers nothing (revoked channel, hard
595
- // cap on the very first append) burns a strike per attempt and stops.
596
- const MAX_SEGMENT_SALVAGES = 5;
597
- let salvagesLeft = MAX_SEGMENT_SALVAGES;
598
- const openSegment = () => {
599
- const seg = pushableStream();
600
- segment = seg;
601
- segmentOpenedAt = Date.now();
602
- armSegmentTimer();
603
- // reopenFence: the delivered prefix of a REJECTED predecessor left a code
604
- // fence open, so the first TEXT entering this salvage segment must be
605
- // preceded by a reopen or it renders outside the code block. Pending
606
- // rather than pushed eagerly (pullfrog catches, PR #42): a card-only
607
- // salvage would otherwise either skip the reopen (later text joining the
608
- // segment renders unfenced) or dangle an empty open fence at message end
609
- // when no text ever follows. Materialized by BOTH text entry points.
610
- const meta = { text: false, acc: "", reopenFence: pendingReopenFence };
611
- pendingReopenFence = false;
612
- segmentMeta = meta;
613
- lastPost = input.post(seg.iterable).then(
614
- () => {
615
- salvagesLeft = MAX_SEGMENT_SALVAGES;
616
- // segments settle in order (push awaits lastPost before opening the
617
- // next), so delivered text supersedes an earlier loss.
618
- if (meta.text) textLost = false;
619
- },
620
- (e: unknown) => {
621
- const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
622
- log("serve.post_error", { err: detail });
623
- // the consumer is gone (e.g. Slack finalized an idle stream:
624
- // message_not_in_streaming_state). Salvage runs in TEXT space, not
625
- // chunk space, because the adapter's renderer buffers across chunks
626
- // (it holds back the trailing unterminated line, unconfirmed table
627
- // headers, and unclosed inline markers until the post-iteration
628
- // finish() flush - pullfrog catch on PR #45: a reply whose last line
629
- // has no trailing newline dies entirely in that forced flush, with
630
- // every chunk already consumed). Proven-delivered text = the mirror
631
- // renderer's committable prefix over the CONFIRMED chunks (each pull
632
- // proves the previous append landed; the adapter runs the renderer
633
- // with wrapTablesForAppend: false, so committable is a raw prefix and
634
- // the mirror is chunking-invariant). Renderer drift would break the
635
- // prefix check and degrade to a full re-post: duplication, never
636
- // loss. Same tradeoff for a stop()-failure after a complete flush:
637
- // the held tail re-posts once rather than risking silent loss.
638
- const { chunks, confirmed } = seg.ledger();
639
- if (segment === seg) segment = null;
640
- const confirmedRaw = chunks
641
- .slice(0, confirmed)
642
- .flatMap((c) => (c instanceof Object ? [] : [c]))
643
- .join("");
644
- const fullRaw = chunks.flatMap((c) => (c instanceof Object ? [] : [c])).join("");
645
- const mirror = new StreamingMarkdownRenderer({ wrapTablesForAppend: false });
646
- mirror.push(confirmedRaw);
647
- const committed = mirror.getCommittableText();
648
- const deliveredLen = confirmedRaw.startsWith(committed) ? committed.length : 0;
649
- const textRemainder = fullRaw.slice(deliveredLen);
650
- // A confirmed card's append landed (the adapter sends a card inline
651
- // in the loop body before the next pull), so unconfirmed cards are
652
- // the set it never appended.
653
- const deliveredCard = chunks.slice(0, confirmed).some((c) => c instanceof Object);
654
- // Delivery progress means this death does not count toward the
655
- // futility budget: only a message that delivered nothing burns one.
656
- // The key is ACTUAL delivery (committable text or a landed card),
657
- // never chunk consumption: a reply whose final line has no trailing
658
- // newline is consumed whole (confirmed advances) while its only
659
- // append is the post-iteration forced flush, so a consumption key
660
- // would refill the budget on every persistently failing flush and
661
- // salvage the same held-back text forever (vercel review catch,
662
- // PR #45). deliveredLen stays 0 there, the strike is spent, and the
663
- // zero-delivery case terminates.
664
- if (deliveredLen > 0 || deliveredCard) salvagesLeft = MAX_SEGMENT_SALVAGES;
665
- // The salvage sequence preserves STREAM ORDER (cursor review catch,
666
- // PR #45: re-posting all remainder text and then all cards showed
667
- // task cards after prose that originally followed them): walk the
668
- // ledger in order, keeping each text chunk's undelivered suffix and
669
- // each unconfirmed card at its original position. Text splits at
670
- // line boundaries (rendering is unchanged: chunks concatenate) so a
671
- // salvage message that dies too still confirms per line, keeping
672
- // progress attribution fine-grained.
673
- const lost: SegmentChunk[] = [];
674
- let offset = 0;
675
- for (const [i, c] of chunks.entries()) {
676
- if (c instanceof Object) {
677
- if (i >= confirmed) lost.push(c);
678
- continue;
679
- }
680
- const end = offset + c.length;
681
- let rest = end > deliveredLen ? c.slice(Math.max(0, deliveredLen - offset)) : "";
682
- offset = end;
683
- while (rest !== "") {
684
- const nl = rest.indexOf("\n");
685
- if (nl === -1) {
686
- lost.push(rest);
687
- break;
688
- }
689
- lost.push(rest.slice(0, nl + 1));
690
- rest = rest.slice(nl + 1);
691
- }
692
- }
693
- // A fence OPENED in the delivered prefix leaves later code fenceless
694
- // in the fresh salvage message (pullfrog catches, PR #42): arm the
695
- // segment's pending reopen, materialized right before the FIRST text
696
- // that enters it - whether a salvaged remainder piece here or a later
697
- // streamed push joining the segment (a card-only salvage must not
698
- // skip the reopen, and a text-less segment must not dangle one).
699
- // Fold this dying segment's OWN still-armed reopen into the parity:
700
- // an armed-but-never-materialized reopenFence means the segment
701
- // logically BEGAN inside an open fence (a card-only salvage that
702
- // inherited one and died before any text materialized it), so that
703
- // open state must propagate to the next salvage or its later text
704
- // renders unfenced (vercel + cubic chained-salvage catch, PR #42).
705
- // Once materialized, reopenFence is false and the reopen chunk is in
706
- // fullRaw, so the XOR is a no-op; a normal segment's flag is false.
707
- const deliveredFenceOpen =
708
- ((fullRaw.slice(0, deliveredLen).split("```").length - 1) % 2 === 1) !== meta.reopenFence;
709
- if (lost.length > 0 && salvagesLeft > 0) {
710
- salvagesLeft -= 1;
711
- log("serve.post_salvage", { chunks: lost.length, left: salvagesLeft });
712
- // this handler runs synchronously as the post settles, so opening
713
- // the salvage segment here keeps the salvaged content ordered ahead
714
- // of any push still awaiting lastPost; the salvage segment's own
715
- // settle then decides whether its text counts as delivered.
716
- // A salvage always recomputes its fence state from the delivered
717
- // prefix (the override below), so it must never CONSUME a pending
718
- // rotation reopen that belongs to a different segment's
719
- // continuation - a rejected notice settling after notify()
720
- // restored the flag would otherwise eat it (cubic review catch,
721
- // round 4). Held across the lost-chunk re-push too: the salvaged
722
- // text is the dying segment's, not the continuation's.
723
- const heldReopen = pendingReopenFence;
724
- pendingReopenFence = false;
725
- const next = openSegment();
726
- next.meta.reopenFence = deliveredFenceOpen;
727
- for (const c of lost) next.pushInto(c);
728
- pendingReopenFence = heldReopen;
729
- } else if (textRemainder !== "") {
730
- textLost = true;
731
- textLostDetail = detail;
732
- }
733
- },
734
- );
735
- const pushInto = (chunk: SegmentChunk) => {
736
- // meta.text only, NEVER postedText: salvaged text was already counted
737
- // at its original push, and postedText is attempt-scoped - a salvage
738
- // landing after a retry reset would otherwise re-arm it and suppress
739
- // the retry's `!postedText && result` fallback, silently dropping a
740
- // result-only answer (adversarial-review catch on PR #45). acc still
741
- // accumulates: it mirrors the segment's FULL text on every entry path,
742
- // so pushText's room accounting and fence parity see salvaged text too
743
- // (a salvage segment that continued via pushText could otherwise grow
744
- // past the msg_too_long cap).
745
- if (!(chunk instanceof Object)) {
746
- if (meta.reopenFence) {
747
- meta.reopenFence = false;
748
- meta.acc += "```\n";
749
- seg.push("```\n");
750
- }
751
- meta.text = true;
752
- meta.acc += chunk;
753
- }
754
- seg.push(chunk);
755
- if (segment === seg) armSegmentTimer();
756
- };
757
- return { seg, meta, pushInto };
758
- };
759
- const push = async (chunk: SegmentChunk) => {
760
- if (segment === null) {
761
- await lastPost; // strict message order: previous segment fully posted first
762
- // a rejection handler may have opened a salvage segment during the wait;
763
- // joining it instead of opening another keeps its post from being
764
- // orphaned un-ended.
765
- }
766
- const target = segment ?? openSegment().seg;
767
- if (!(chunk instanceof Object)) {
768
- // a pending rotation reopen can outlive its openSegment hand-off when
769
- // text JOINS a segment a rejection handler opened during the wait
770
- // (the salvage recomputes its own fence state without consuming the
771
- // flag): adopt it here so the continuation still reopens its fence,
772
- // UNLESS the joined segment already sits inside an open fence - then
773
- // the state is satisfied and a second marker would close it (cubic
774
- // review catch, round 4).
775
- if (pendingReopenFence) {
776
- pendingReopenFence = false;
777
- if (!fenceOpen()) segmentMeta!.reopenFence = true;
778
- }
779
- // materialize a salvage segment's pending fence reopen (see
780
- // openSegment) before the first text from this entry point too.
781
- if (segmentMeta!.reopenFence) {
782
- segmentMeta!.reopenFence = false;
783
- segmentMeta!.acc += "```\n";
784
- target.push("```\n");
785
- }
786
- postedText = true;
787
- segmentMeta!.text = true;
788
- segmentMeta!.acc += chunk;
789
- }
790
- target.push(chunk);
791
- if (segment === target) armSegmentTimer();
792
- };
793
- // parity by occurrence count over the segment's accumulated text: an odd
794
- // number of ``` markers means the segment currently sits inside a fence.
795
- const fenceOpen = () => segmentMeta !== null && (segmentMeta.acc.split("```").length - 1) % 2 === 1;
796
- const breakSegment = () => {
797
- if (segmentTimer !== null) {
798
- clearTimeout(segmentTimer);
799
- segmentTimer = null;
800
- }
801
- // a segment dying with an armed-but-unmaterialized reopen (no text ever
802
- // arrived - a card-only segment that consumed the flag at open) folds it
803
- // back: the reply still logically sits inside an open fence, and the
804
- // next text-bearing segment must reopen it (cubic review catch, round
805
- // 4: the errored result's closing card ate the rotation's reopen).
806
- if (segmentMeta?.reopenFence) pendingReopenFence = true;
807
- segment?.end();
808
- segment = null;
809
- };
810
- const armSegmentTimer = () => {
811
- if (segmentTimer !== null) clearTimeout(segmentTimer);
812
- const ageLeft = segmentOpenedAt + SEGMENT_ROTATION.maxAgeMs - Date.now();
813
- segmentTimer = setTimeout(() => {
814
- segmentTimer = null;
815
- // a rotation mid-fence closes the fence and arms the reopen, so both
816
- // message halves render as code - pushText's split contract. An
817
- // armed-but-unmaterialized reopen needs no close marker; breakSegment
818
- // folds it forward.
819
- if (segment !== null && fenceOpen()) {
820
- segmentMeta!.acc += "\n```";
821
- segment.push("\n```");
822
- pendingReopenFence = true;
823
- }
824
- breakSegment();
825
- }, Math.max(0, Math.min(SEGMENT_ROTATION.idleMs, ageLeft)));
826
- };
827
- /** Reply text routed through here splits across Slack messages before the
828
- * msg_too_long cap (see SEGMENT_TEXT_MAX): a break prefers the last newline
829
- * inside the remaining room, and a break forced inside a code fence closes
830
- * it and reopens it in the next message so both halves render as code. */
831
- const pushText = async (text: string) => {
832
- for (let rest = text; rest !== "";) {
833
- const room = SEGMENT_TEXT_MAX - (segment === null ? 0 : segmentMeta!.acc.length);
834
- // a fence-close suffix can nudge a segment a few chars past the cap;
835
- // a full segment just breaks and the loop re-measures a fresh one.
836
- if (room <= 0) {
837
- breakSegment();
838
- continue;
839
- }
840
- if (rest.length <= room) {
841
- await push(rest);
842
- return;
843
- }
844
- // prefer a newline cut only when it lands in the back half of the room:
845
- // an early newline followed by one giant unbroken run would otherwise
846
- // make no progress and (with the fence-reopen prefix) loop forever.
847
- const nl = rest.lastIndexOf("\n", room - 1);
848
- let cut = nl >= Math.floor(room / 2) ? nl + 1 : room;
849
- // never slice through a backtick run: a cut inside ``` would strand a
850
- // partial delimiter on each side and break both halves' rendering
851
- // (cubic review catch, PR #42). Walk the cut left past the run; a run
852
- // reaching position 0 keeps the original cut (progress beats rendering
853
- // for pathological all-backtick input).
854
- if (rest[cut - 1] === "`" && rest[cut] === "`") {
855
- let backedUp = cut;
856
- while (backedUp > 0 && rest[backedUp - 1] === "`") backedUp -= 1;
857
- if (backedUp > 0) cut = backedUp;
858
- }
859
- const head = rest.slice(0, cut);
860
- await push(head);
861
- const reopen = fenceOpen();
862
- if (reopen) await push("\n```");
863
- breakSegment();
864
- rest = (reopen ? "```\n" : "") + rest.slice(head.length);
865
- }
866
- };
867
- // settle the whole post chain: a rejection handler may replace lastPost with
868
- // a salvage segment's post, which still needs ending and settling.
869
- const settlePosts = async () => {
870
- while (true) {
871
- breakSegment();
872
- const settled = lastPost;
873
- await settled;
874
- if (lastPost === settled) return;
875
- }
876
- };
877
- // a recovery status line reads as its own Slack message, not part of a
878
- // streamed segment.
879
- const notify = async (text: string) => {
880
- breakSegment();
881
- // a notice must never inherit a rotation's pending fence reopen: the
882
- // reopen belongs to the interrupted reply's continuation, not to the
883
- // status line - consuming it here would render the notice as a code
884
- // block AND strand the continuation unfenced (cubic review catch,
885
- // round 3). Held aside and restored for the real continuation.
886
- const heldReopen = pendingReopenFence;
887
- pendingReopenFence = false;
888
- await push(text);
889
- breakSegment();
890
- pendingReopenFence = heldReopen;
891
- };
892
- /** notify + confirm the post actually landed in Slack. A drop notice that
893
- * never reached the user must NOT count as announced. DURING A DRAIN an
894
- * unannounced drop keeps the resume marker so startup replays instead;
895
- * outside a drain the marker is still cleared ON PURPOSE (closing-review
896
- * catch corrected this doc, not the behavior): a non-drain drop happens
897
- * after the turn's spawn decisions ran, and retaining its marker would
898
- * make the next daemon restart RE-EXECUTE a possibly-metered turn whose
899
- * outcome the user may already have seen - duplicate execution is worse
900
- * than a lost message behind an already-broken Slack surface. The
901
- * unannounced non-drain loss is logged loudly (serve.drop_unannounced) by
902
- * the caller so it is at least operator-visible. The notice is text, so
903
- * its own delivery resets textLost. */
904
- const notifyDelivered = async (text: string) => {
905
- await notify(text);
906
- await settlePosts();
907
- return !textLost;
908
- };
909
- // false when the daemon started draining mid-sleep.
910
- const sleep = async (ms: number) => {
911
- try {
912
- await delay(Math.max(ms, 0), { signal: input.drainSignal });
913
- return true;
914
- } catch {
915
- return false;
916
- }
917
- };
918
- const inWord = (epochMs: number | null) => (epochMs == null ? "an unknown time" : `~${fmtResetShort(epochMs, Date.now()) || "1m"}`);
919
-
920
- // a non-limit failure line held back until the depleted-pool probe rules:
921
- // posted verbatim on a plain failure, discarded when the turn defers (the
922
- // deferral notice explains the pause; the raw line would invite a manual
923
- // re-send of work the daemon resumes itself - cubic catch, PR #44).
924
- let pendingFailureLine: string | null = null;
925
- // texts steered into this message's turn, kept across retries: a retry
926
- // resumes the session and re-sends the attempt prompt (the established
927
- // duplication-beats-loss tradeoff), so steered text folds into every
928
- // retry attempt's first message too - a steer written just before a
929
- // mid-turn limit killed the child must not vanish from the turn it joined.
930
- const steeredTexts: string[] = [];
931
- // what the next spawn submits ahead of the steered texts: the original
932
- // message, until a transient retry swaps in the continuation wrapper (see
933
- // the retry branch).
934
- let prompt = input.prompt;
935
- const runQueryOnce = async () => {
936
- postedText = false;
937
- pendingFailureLine = null;
938
- outcome.failed = false;
939
- outcome.rateLimited = false;
940
- outcome.resultReceived = false;
941
- // outcome.finish and outcome.attention stay sticky across retries: the
942
- // tool calls already happened in this session, and a limit right after
943
- // one must not unfinish the thread or drop the pending ask.
944
- // the identity this spawn meters: a limit observation is attributed to it,
945
- // never to whatever account a concurrent thread swaps live mid-turn. Read
946
- // inside the try: a malformed claude.json must fail the TURN, not the
947
- // relay's never-throws contract.
948
- let spawnOrg: string | null = null;
949
- // Streaming input: the prompt rides an open stdin stream instead of a
950
- // one-shot string, which is what lets a mid-turn Slack reply steer the
951
- // running turn (the CLI folds a queued stream-json user message into the
952
- // current turn at the next tool boundary; one that misses the last fold
953
- // window runs as its own turn in the same child before exit, so nothing
954
- // is ever dropped - both verified against claude 2.1.220 + SDK 0.3.214).
955
- // steer() refuses the moment the attempt's result arrives: stdin ends
956
- // then, and the SDK silently drops writes to an ended stdin.
957
- const stream = pushableMessages();
958
- let steerable = true;
959
- const steer = (text: string, onAccept?: () => void): boolean => {
960
- if (!steerable) return false;
961
- // the caller's SYNCHRONOUS durable commit runs inside the acceptance,
962
- // in the same JS tick as the liveness check and BEFORE the child sees
963
- // the text (adversarial-review catch, round 3): steerable=true here
964
- // proves the turn has not ended, so the commit's view of the turn
965
- // state cannot be stale, a crash between commit and push replays a
966
- // steer the child never saw (duplication over loss), and a refusal
967
- // commits NOTHING - a stale acceptor invocation racing the turn's end
968
- // can no longer resurrect a finished turn's marker or clobber
969
- // post-turn state. A THROWING commit escapes to the caller before
970
- // anything is pushed or recorded here: the stream, steeredTexts, and
971
- // the sticky flags are untouched, so the caller can treat the throw
972
- // as a refusal and a later steer still works.
973
- onAccept?.();
974
- steeredTexts.push(text);
975
- stream.push(steerUserMessage(text));
976
- // a steer answers a LIVE ask too (codex review catch on PR #50): when
977
- // need_attention already fired in this attempt, the ask exists only as
978
- // this sticky flag until settleTurn persists it - left set, the
979
- // answered ask would still get a question mark and a nudge. A later
980
- // need_attention call re-arms it for a genuinely new ask.
981
- outcome.attention = false;
982
- return true;
983
- };
984
- try {
985
- spawnOrg = readOAuthAccount()?.organizationUuid ?? null;
986
- const pooled = pooledOptions();
987
- stream.push(steerUserMessage([prompt, ...steeredTexts].join("\n\n")));
988
- const q = query({
989
- prompt: stream.iterable,
990
- options: {
991
- ...pooled,
992
- // claude >= 2.1.142 emits the structured Task tools by default and
993
- // TodoWrite (the source of the Todos checklist card) never fires;
994
- // this documented opt-out restores it (agent-sdk todo-tracking docs,
995
- // verified 2026-07-18 against SDK 0.3.214 + claude 2.1.214). Reuses
996
- // pooled.env so the scrubbed env copy is built once per turn (cubic
997
- // review catch on PR #5).
998
- env: { ...pooled.env, CLAUDE_CODE_ENABLE_TASKS: "0" },
999
- cwd: input.cwd,
1000
- permissionMode: input.link.permissionMode,
1001
- // the SDK refuses bypassPermissions without this explicit opt-in.
1002
- ...(input.link.permissionMode === "bypassPermissions" ? { allowDangerouslySkipPermissions: true } : {}),
1003
- includePartialMessages: true,
1004
- systemPrompt: SLACK_SYSTEM_PROMPT,
1005
- // no one can answer an interactive question dialog through Slack;
1006
- // without the tool the model asks in prose and the user's thread
1007
- // reply becomes the next turn.
1008
- disallowedTools: ["AskUserQuestion"],
1009
- spawnClaudeCodeProcess: (spawnOptions) => {
1010
- const child = detachedClaudeSpawn(spawnOptions);
1011
- if (child.pid !== undefined) {
1012
- try {
1013
- input.onSpawn?.(child.pid);
1014
- } catch (e) {
1015
- // a failed marker persist must not leave an untracked group
1016
- // running (cubic review catch): kill it, then fail the spawn
1017
- // loudly through the SDK.
1018
- killGroup(child.pid);
1019
- throw e;
1020
- }
1021
- }
1022
- return child;
1023
- },
1024
- // the user saying "we're done" closes the thread, and the model
1025
- // asking the user for a decision marks it waiting: both flagged via
1026
- // in-process tools, both acted on by the daemon post-turn. Like
1027
- // finish, attention stays sticky across retries: the tool call
1028
- // already happened in this session.
1029
- mcpServers: {
1030
- tokenmaxxing: serveToolServer({
1031
- onFinish: () => { outcome.finish = true; },
1032
- onAttention: () => { outcome.attention = true; },
1033
- }),
1034
- },
1035
- allowedTools: [FINISH_THREAD_TOOL, NEED_ATTENTION_TOOL],
1036
- // serve skills (ask-the-user, serve-session); discovered skills are
1037
- // enabled by default, so no `skills` option is needed.
1038
- plugins: [{ type: "local", path: SERVE_PLUGIN_DIR }],
1039
- hooks: {
1040
- UserPromptSubmit: [{
1041
- hooks: [async () => ({
1042
- hookSpecificOutput: {
1043
- hookEventName: "UserPromptSubmit",
1044
- additionalContext: serveTurnContext({ requesterIds: input.requesterIds }),
1045
- },
1046
- })],
1047
- }],
1048
- Stop: [{ hooks: [stopHookCheck] }],
1049
- },
1050
- ...(input.link.model ? { model: input.link.model } : {}),
1051
- // a retry resumes the session the failed attempt opened, so no
1052
- // context is lost across recoveries.
1053
- ...(outcome.sessionId ? { resume: outcome.sessionId } : {}),
1054
- },
1055
- });
1056
- // registered only while this attempt is live (cleared in the finally):
1057
- // the caller's fallback for a refused steer is the normal queued turn.
1058
- input.onSteer?.(steer);
1059
- const mapState = newStreamMapState();
1060
- let result: string | null = null;
1061
- // reply text streamed since the last result boundary: each turn in the
1062
- // child (the primary one, plus any post-fold-window steer drained as
1063
- // its own turn) delivers its answer independently - a tool-only turn's
1064
- // answer lives ONLY in its result message, and flushing it at that
1065
- // result is what keeps a trailing turn's notice or result from
1066
- // suppressing or clobbering it (adversarial-review catch, round 2).
1067
- let streamedSinceResult = false;
1068
- for await (const message of q) {
1069
- if (message.type === "system" && message.subtype === "init") {
1070
- // persist BEFORE the turn ends so a first-turn kill stays
1071
- // resumable; compared against the last known id, so retry attempts
1072
- // resuming the same session re-fire only on an actual change.
1073
- if (message.session_id !== outcome.sessionId) input.onSessionId?.(message.session_id);
1074
- outcome.sessionId = message.session_id;
1075
- }
1076
- if (message.type === "result") {
1077
- outcome.sessionId = message.session_id;
1078
- // any result ends steerability and closes the child's stdin: a
1079
- // steer arriving now queues as its own next turn instead. A steer
1080
- // accepted BEFORE this that missed its fold window still runs -
1081
- // the CLI drains queued commands as their own turns in this same
1082
- // child before exiting, emitting a further result each time, so
1083
- // this loop just keeps consuming until the child exits (end() is
1084
- // idempotent).
1085
- steerable = false;
1086
- stream.end();
1087
- // is_error can ride a "success" subtype (a mid-turn usage limit
1088
- // arrives exactly that way: result "Claude AI usage limit
1089
- // reached|<epoch>"), so errored is a field check, not a subtype
1090
- // check - and only an errored result is ever limit-classified.
1091
- if (message.is_error || message.subtype !== "success") {
1092
- const text = erroredResultText(message);
1093
- const limited = isRateLimitText({ text });
1094
- // persist the observation either way: the next spawn decision
1095
- // must see the limit even when this turn does not retry.
1096
- if (limited) await recordObservedLimit({ text, now: Date.now(), org: spawnOrg });
1097
- if (outcome.resultReceived) {
1098
- // SUCCESS IS STICKY within an attempt (adversarial-review
1099
- // catch): this errored result belongs to a post-fold-window
1100
- // steer's own drained turn, arriving AFTER the primary turn
1101
- // already succeeded and delivered its answer. Marking the turn
1102
- // failed here would send the whole prompt back through the
1103
- // retry/defer machinery and re-run completed work - the exact
1104
- // duplicate-execution the outcome contract forbids. The steer
1105
- // is announced lost instead (drop-beats-false-promise), and
1106
- // steerLost settles the steered messages' reactions as failed.
1107
- outcome.steerLost = true;
1108
- log("serve.steered_turn_failed", { limited });
1109
- await notify(
1110
- limited
1111
- ? "a steered follow-up message hit a usage limit before it could run - please re-send it."
1112
- : "a steered follow-up message failed - please re-send it.",
1113
- );
1114
- } else {
1115
- outcome.failed = true;
1116
- // a limit classification is sticky across a child's errored
1117
- // results (adversarial-review catch, round 2): the drained
1118
- // steer turn's generic death must not declassify the primary
1119
- // turn's recoverable limit back to a plain failure.
1120
- outcome.rateLimited = outcome.rateLimited || limited;
1121
- // hold the REAL errored text for the terminal diagnostic (codex
1122
- // review catches: a streamed-then-errored turn used to end with
1123
- // a truncated answer and only the x reaction, and a no-text
1124
- // errored turn used to discard the reason for a generic line).
1125
- // The depleted-pool probe still outranks the line.
1126
- if (!limited) {
1127
- const reason = text.slice(0, 200) || "no error detail";
1128
- pendingFailureLine = postedText
1129
- ? `tokenmaxxing: the turn errored before finishing (${reason}) - the reply above may be incomplete.`
1130
- : `tokenmaxxing: the turn errored without a result (${reason}) - trying again may help.`;
1131
- }
1132
- }
1133
- } else {
1134
- result = message.result;
1135
- outcome.resultReceived = true;
1136
- // a turn that streamed no reply text (tool-only turns) still
1137
- // reports: its answer is flushed HERE, per result, not after the
1138
- // loop - a later turn in the same child must not suppress it. A
1139
- // paragraph break ahead of it when text already posted, or two
1140
- // result-only answers would concatenate mid-line (codex review
1141
- // catch on PR #50).
1142
- if (!streamedSinceResult && result) await pushText(`${postedText ? "\n\n" : ""}${result}`);
1143
- }
1144
- streamedSinceResult = false;
1145
- }
1146
- for (const part of agentEventChunks({ state: mapState, message })) {
1147
- if (part instanceof Object) await push(part);
1148
- else {
1149
- if (part.trim() !== "") streamedSinceResult = true;
1150
- await pushText(part);
1151
- }
1152
- }
1153
- }
1154
- } catch (e) {
1155
- outcome.failed = true;
1156
- const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
1157
- outcome.rateLimited = isRateLimitText({ text: detail });
1158
- if (outcome.rateLimited) await recordObservedLimit({ text: detail, now: Date.now(), org: spawnOrg });
1159
- log("serve.turn_error", { err: detail });
1160
- // same hold-back: the detail already reached the log above, and a
1161
- // deferral's own notice explains the pause better than a raw child
1162
- // error that reads as "please re-send".
1163
- if (!outcome.rateLimited) pendingFailureLine = `tokenmaxxing: turn failed: ${detail}`;
1164
- } finally {
1165
- // a died attempt must stop accepting steers (they would be silent
1166
- // drops on an ended stdin) and must release the caller's steer hook
1167
- // before the retry loop decides anything.
1168
- steerable = false;
1169
- stream.end();
1170
- input.onSteer?.(null);
1171
- }
1172
- };
1173
-
1174
- let recoveries = 0;
1175
- let transientRetries = 0;
1176
- const parkDeadline = Date.now() + PARK_MAX_MS;
1177
- while (true) {
1178
- // the switch decision runs at the spawn boundary, same as the CLI hooks.
1179
- let decision: SwapDecision;
1180
- try {
1181
- decision = await ensureBestAccount();
1182
- } catch (e) {
1183
- outcome.failed = true;
1184
- const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
1185
- log("serve.turn_error", { err: detail });
1186
- await push(`tokenmaxxing: turn failed: ${detail}`);
1187
- break;
1188
- }
1189
- const plan = parkPlan({ decision, recoveries, deadline: parkDeadline });
1190
- if (plan.kind === "drop") {
1191
- // recovery time unknown: an auto-resume promise would be unkeepable,
1192
- // so the honest drop survives for exactly this case.
1193
- outcome.failed = true;
1194
- outcome.rateLimited = true;
1195
- log("serve.pool_depleted_drop", {});
1196
- outcome.announcedDrop = await notifyDelivered("every pooled account is at its usage limit (recovers at an unknown time) - this message was dropped; re-send it once the pool recovers.");
1197
- break;
1198
- }
1199
- if (plan.kind === "defer") {
1200
- // release the queue slot and let the daemon resume the turn from its
1201
- // durable marker once the pool recovers (2026-07-20 incident: dropped
1202
- // messages sat dead for hours after recovery until re-sent by hand).
1203
- outcome.failed = true;
1204
- outcome.rateLimited = true;
1205
- outcome.deferUntil = plan.resumeAt;
1206
- log("serve.pool_depleted_defer", { resumeAt: plan.resumeAt });
1207
- await notify(`every pooled account is at its usage limit - holding this message; it will resume automatically in ${inWord(plan.resumeAt)}.`);
1208
- break;
1209
- }
1210
- if (plan.kind === "park") {
1211
- recoveries += 1;
1212
- log("serve.pool_depleted_park", { wakeAt: plan.wakeAt, recoveries });
1213
- await notify(`every pooled account is at its usage limit - holding this message and retrying in ${inWord(plan.wakeAt)}.`);
1214
- if (!(await sleep(plan.wakeAt - Date.now()))) {
1215
- // a drain aborted the park: the turn never spawned, so the marker
1216
- // survives (presumedKilled) and the next daemon start replays it.
1217
- outcome.failed = true;
1218
- await notify("tokenmaxxing is restarting - this message resumes after the restart.");
1219
- break;
1220
- }
1221
- continue;
1222
- }
1223
- await runQueryOnce();
1224
- if (!outcome.failed) break;
1225
- if (!outcome.rateLimited) {
1226
- // an unclassifiable child failure (e.g. "Claude Code process exited
1227
- // with code 1" - the 2026-07-20 Fable-cap death carried no limit
1228
- // phrase) against an exhausted pool IS the limit: the pool state is
1229
- // the evidence the error text did not carry. A completed result stays
1230
- // terminal, and a usable pool keeps the plain failure.
1231
- if (!outcome.resultReceived) {
1232
- // outranks the transient retry below even when the recovery time is
1233
- // unknown (cubic review catch): a depleted pool explains the failure,
1234
- // and respawning against it would just burn doomed attempts.
1235
- let probeDepleted = false;
1236
- try {
1237
- const verdict = await ensureBestAccount();
1238
- probeDepleted = verdict.reason === "all-depleted" || verdict.reason === "depleted-wait";
1239
- const wake = probeDepleted ? verdict.waitUntil ?? null : null;
1240
- if (wake != null) {
1241
- outcome.rateLimited = true;
1242
- outcome.deferUntil = wake + PARK_GRACE_MS;
1243
- pendingFailureLine = null;
1244
- log("serve.turn_failed_depleted_defer", { resumeAt: outcome.deferUntil });
1245
- await notify(`the account pool is exhausted - pausing this turn; it will resume automatically in ${inWord(outcome.deferUntil)}.`);
1246
- }
1247
- } catch (e) {
1248
- // keep the original failure; the probe must never mask it.
1249
- log("serve.defer_probe_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
1250
- }
1251
- // a transient non-limit failure (child crash, API blip, errored
1252
- // result) against a usable pool retries silently into the same
1253
- // session, mirroring the rate-limit path's invisible short retry
1254
- // (2026-07-27 report: these turns died terminally, telling the user
1255
- // "trying again may help" instead of trying again). resultReceived
1256
- // stays terminal above - a completed answer must never re-run - and
1257
- // the probe's deferral outranks a retry: an exhausted pool explains
1258
- // the failure better than "transient".
1259
- if (!probeDepleted && outcome.deferUntil === null && transientRetries < MAX_TRANSIENT_RETRIES) {
1260
- transientRetries += 1;
1261
- breakSegment();
1262
- log("serve.transient_retry", { attempt: transientRetries });
1263
- // once a session exists, the failed attempt may have executed
1264
- // side-effectful tools before dying, so the retry must CONTINUE,
1265
- // never re-instruct: replaying the original prompt verbatim into
1266
- // the resumed session reads as "do it again" (codex review catch).
1267
- // The wrapper mirrors resumeDecision's deferral resume. A pre-init
1268
- // death ran nothing, so the original replays verbatim there.
1269
- if (outcome.sessionId !== null) {
1270
- prompt = `Your previous turn was interrupted by an error mid-run; this session's transcript already holds any work it completed, including tool calls whose side effects already happened. Pick up exactly where you left off and finish the task without re-running completed side-effectful steps. If the work was already complete, just summarize the final state. The original request was:\n\n${input.prompt}`;
1271
- }
1272
- if (!(await sleep(RETRY_DELAY_MS))) {
1273
- // a drain aborted the retry sleep: same as a killed child, the
1274
- // marker survives (presumedKilled) and the next daemon start
1275
- // resumes the session where it stopped.
1276
- await notify("tokenmaxxing is restarting - this turn resumes after the restart.");
1277
- break;
1278
- }
1279
- continue;
1280
- }
1281
- }
1282
- if (pendingFailureLine !== null) {
1283
- // same fence-reopen bypass as notify: a diagnostic line opening a
1284
- // fresh segment must not render inside a reopened code fence.
1285
- pendingReopenFence = false;
1286
- await push(transientRetries > 0 ? `${pendingFailureLine} (after ${transientRetries + 1} attempts)` : pendingFailureLine);
1287
- }
1288
- break;
1289
- }
1290
- if (recoveries >= MAX_RECOVERIES) {
1291
- // out of in-handler retry budget: defer to the pool's own recovery
1292
- // clock when it is known, drop honestly when it is not (a stale cache
1293
- // claiming a usable pool while every retry limits out lands here too,
1294
- // and deferring on no evidence would just spin the resume cap).
1295
- let wake: number | null = null;
1296
- try {
1297
- const verdict = await ensureBestAccount();
1298
- const depleted = verdict.reason === "all-depleted" || verdict.reason === "depleted-wait";
1299
- wake = depleted ? verdict.waitUntil ?? null : null;
1300
- } catch (e) {
1301
- log("serve.defer_probe_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
1302
- }
1303
- if (wake != null) {
1304
- outcome.deferUntil = wake + PARK_GRACE_MS;
1305
- log("serve.rate_limited_defer", { recoveries, resumeAt: outcome.deferUntil });
1306
- await notify(`still at a usage limit after retries - pausing this turn; it will resume automatically in ${inWord(outcome.deferUntil)}.`);
1307
- break;
1308
- }
1309
- log("serve.rate_limited_drop", { recoveries });
1310
- outcome.announcedDrop = await notifyDelivered("still at a usage limit after retries - this message was dropped; reply when you want to try again.");
1311
- break;
1312
- }
1313
- // a limit the cached pool state did not predict: give the pool one beat
1314
- // to observe it, then re-decide and retry the same prompt into the same
1315
- // session (slaude's silent short retry - a successful swap makes it
1316
- // invisible in the thread).
1317
- recoveries += 1;
1318
- breakSegment();
1319
- log("serve.rate_limited_retry", { recoveries });
1320
- // wait out an active post-swap cooldown too: a swap-then-instant-limit
1321
- // would otherwise burn every retry inside the 45s window where the
1322
- // decision refuses to re-evaluate, respawning the same limited account
1323
- // (review catch, PR #18). The persisted observation then makes the
1324
- // post-cooldown decision see the depleted account immediately.
1325
- // loadLastSwapAt throws on a corrupt swap clock; every failure in this
1326
- // loop must settle the turn in-thread (announced, never a bare throw),
1327
- // same as the ensureBestAccount guard above (review catch, PR #31).
1328
- let cooldownUntil: number;
1329
- try {
1330
- cooldownUntil = (loadLastSwapAt() ?? 0) + POST_SWAP_COOLDOWN_MS + 1_000;
1331
- } catch (e) {
1332
- outcome.failed = true;
1333
- const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
1334
- log("serve.turn_error", { err: detail });
1335
- await push(`tokenmaxxing: turn failed: ${detail}`);
1336
- break;
1337
- }
1338
- if (!(await sleep(Math.max(RETRY_DELAY_MS, cooldownUntil - Date.now())))) {
1339
- // a drain aborted the retry sleep: same as a killed child, the marker
1340
- // survives (presumedKilled) and the next daemon start resumes the
1341
- // session where it stopped.
1342
- outcome.failed = true;
1343
- await notify("tokenmaxxing is restarting - this turn resumes after the restart.");
1344
- break;
1345
- }
1346
- }
1347
- await settlePosts();
1348
- // Reply text died with a rejected segment, salvage could not re-deliver it
1349
- // (budget exhausted or the salvage posts died too), and nothing later
1350
- // re-delivered it: the answer silently vanished while the outcome would
1351
- // report success. Fail the turn and make one best-effort fresh-message
1352
- // diagnostic.
1353
- if (textLost) {
1354
- outcome.failed = true;
1355
- const detail = textLostDetail ?? "unknown error";
1356
- await input
1357
- .post((async function* () {
1358
- yield `tokenmaxxing: the reply could not be posted to Slack: ${detail}`;
1359
- })())
1360
- .catch((e: unknown) => log("serve.post_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) }));
1361
- }
1362
- return outcome;
1363
- }