tokenmaxxing 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/serve.ts +140 -13
- package/src/lib/install.ts +3 -2
- package/src/lib/slackbridge.ts +146 -11
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tokenmaxxing",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.4.0",
|
|
4
4
|
"description": "Automatic Claude Code account switching: pool multiple accounts and hot-swap when quota fills, resuming your session on the fresh account.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/cli/serve.ts
CHANGED
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
// serve links list links
|
|
16
16
|
// serve run the daemon
|
|
17
17
|
|
|
18
|
-
import { existsSync, realpathSync } from "node:fs";
|
|
18
|
+
import { existsSync, readFileSync, readdirSync, realpathSync } from "node:fs";
|
|
19
|
+
import { join } from "node:path";
|
|
19
20
|
import { delay, omit, uniq } from "es-toolkit";
|
|
20
21
|
import { z } from "zod";
|
|
21
22
|
import { Chat, ConsoleLogger, StreamingPlan, ThreadImpl, type StreamChunk } from "chat";
|
|
@@ -36,6 +37,7 @@ import {
|
|
|
36
37
|
stripLeadingMention,
|
|
37
38
|
upsertLink,
|
|
38
39
|
SlackLinkSchema,
|
|
40
|
+
SlackThreadSchema,
|
|
39
41
|
type ActiveTurn,
|
|
40
42
|
type SlackConfig,
|
|
41
43
|
type SlackLink,
|
|
@@ -369,6 +371,8 @@ export function buildServeRuntime(seam: {
|
|
|
369
371
|
let draining = false;
|
|
370
372
|
// channels already diagnosed as unlinked this run (see handleTurn).
|
|
371
373
|
const unlinkedLogged = new Set<string>();
|
|
374
|
+
// corrupt thread records already logged as skipped this run (see nudgeSweep).
|
|
375
|
+
const sweepSkipLogged = new Set<string>();
|
|
372
376
|
|
|
373
377
|
/** Best-effort status reaction: reaction state is decoration, so every
|
|
374
378
|
* failure (missing reactions:write until the app is reinstalled,
|
|
@@ -730,10 +734,22 @@ export function buildServeRuntime(seam: {
|
|
|
730
734
|
return true;
|
|
731
735
|
};
|
|
732
736
|
|
|
737
|
+
/** The ownership funnel for every task the daemon spawns: registration in
|
|
738
|
+
* activeTurns so a shutdown drains it, and the daemon's terminal error
|
|
739
|
+
* boundary. Bun kills the WHOLE process on any unhandled rejection
|
|
740
|
+
* (default-mode exit verified 2026-07-27), so a `void tracked(...)`
|
|
741
|
+
* fire-and-forget whose task threw would otherwise take every concurrent
|
|
742
|
+
* session's turn down with it - one thread's bad state file must never
|
|
743
|
+
* end another thread's half-streamed answer. Site-specific handling (the
|
|
744
|
+
* in-thread crash notice in onMessage) stays at the site that has the
|
|
745
|
+
* context; whatever escapes lands here, logged, and the daemon keeps
|
|
746
|
+
* serving. */
|
|
733
747
|
const tracked = async (turn: Promise<void>) => {
|
|
734
748
|
activeTurns.add(turn);
|
|
735
749
|
try {
|
|
736
750
|
await turn;
|
|
751
|
+
} catch (e) {
|
|
752
|
+
log("serve.task_crashed", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
|
|
737
753
|
} finally {
|
|
738
754
|
activeTurns.delete(turn);
|
|
739
755
|
}
|
|
@@ -775,7 +791,47 @@ export function buildServeRuntime(seam: {
|
|
|
775
791
|
const onMessage = async (input: { thread: ServeThread; message: ServeMessage; skipped: ServeMessage[]; isMention: boolean }) => {
|
|
776
792
|
const relayed = [...input.skipped, input.message].filter(relayable).map((m) => ({ text: m.text, authorId: m.author.userId, id: m.id }));
|
|
777
793
|
if (relayed.length === 0) return; // outsider mentions never open a session
|
|
778
|
-
await tracked(serialized(input.thread.id, () =>
|
|
794
|
+
await tracked(serialized(input.thread.id, async () => {
|
|
795
|
+
try {
|
|
796
|
+
await handleTurn({ thread: input.thread, relayed, isMention: input.isMention });
|
|
797
|
+
} catch (e) {
|
|
798
|
+
// an escaped handleTurn throw (a state-file parse, a Slack API
|
|
799
|
+
// rejection outside relayThread's never-throws boundary) previously
|
|
800
|
+
// died in the chat SDK's catch-and-log: the user's message vanished
|
|
801
|
+
// with no reply and no log line of ours (2026-07-27 report). Tell
|
|
802
|
+
// the thread and keep the daemon serving.
|
|
803
|
+
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
804
|
+
log("serve.turn_crashed", { thread: input.thread.id, err: detail });
|
|
805
|
+
// a surviving activeTurn marker means the turn is PRESERVED (a drain
|
|
806
|
+
// kill kept it for the next generation's auto-resume): a "re-send it"
|
|
807
|
+
// notice would invite a duplicate run and a failed x would misread a
|
|
808
|
+
// guaranteed retry (cubic review catch, round 3) - log only, the
|
|
809
|
+
// resume machinery owns the messaging. The read is best-effort: an
|
|
810
|
+
// unreadable record (possibly the crash itself) takes the visible
|
|
811
|
+
// crash path.
|
|
812
|
+
let preserved = false;
|
|
813
|
+
try {
|
|
814
|
+
preserved = loadSlackThread(input.thread.id)?.activeTurn !== undefined;
|
|
815
|
+
} catch { /* unreadable record: treat as not preserved */ }
|
|
816
|
+
if (preserved) return;
|
|
817
|
+
try {
|
|
818
|
+
await input.thread.post(
|
|
819
|
+
(async function* () {
|
|
820
|
+
yield `tokenmaxxing: this message's handling crashed: ${detail}. If no reply landed above, re-send it.`;
|
|
821
|
+
})(),
|
|
822
|
+
);
|
|
823
|
+
} catch (postErr) {
|
|
824
|
+
log("serve.turn_crash_notice_failed", { thread: input.thread.id, err: (postErr instanceof Error ? postErr.message : String(postErr)).slice(0, 300) });
|
|
825
|
+
}
|
|
826
|
+
// a crash after runTurn added the hourglass would otherwise read as
|
|
827
|
+
// "processing" forever (codex review catch); setStatus never throws.
|
|
828
|
+
const messageId = relayed.at(-1)?.id;
|
|
829
|
+
if (messageId) {
|
|
830
|
+
await setStatus({ threadId: input.thread.id, messageId, emoji: STATUS_EMOJI.failed, op: "add" });
|
|
831
|
+
await setStatus({ threadId: input.thread.id, messageId, emoji: STATUS_EMOJI.processing, op: "remove" });
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
}));
|
|
779
835
|
};
|
|
780
836
|
|
|
781
837
|
/** A user reaction in a tracked thread. While the thread waits on an asked
|
|
@@ -829,16 +885,47 @@ export function buildServeRuntime(seam: {
|
|
|
829
885
|
// folds into the next turn.
|
|
830
886
|
if (!draining && asked && asked.requesterIds.includes(input.userId) && afterAsk && onAskMessage) {
|
|
831
887
|
log("serve.reaction_answer", { thread: input.threadId, emoji: input.emoji });
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
888
|
+
try {
|
|
889
|
+
const { thread } = await streamable(input.threadId);
|
|
890
|
+
await handleTurn({
|
|
891
|
+
thread,
|
|
892
|
+
relayed: [{
|
|
893
|
+
text: `<@${input.userId}> answered your pending question with the Slack reaction :${input.emoji}:. Interpret the reaction as their reply and continue.`,
|
|
894
|
+
authorId: input.userId,
|
|
895
|
+
id: input.messageId,
|
|
896
|
+
}],
|
|
897
|
+
isMention: false,
|
|
898
|
+
});
|
|
899
|
+
} catch (e) {
|
|
900
|
+
// a crashed answer turn must not read as an accepted answer (codex
|
|
901
|
+
// review catch): tell the thread and settle the reacted-to
|
|
902
|
+
// message's status so it never reads as processing forever.
|
|
903
|
+
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
904
|
+
log("serve.reaction_crashed", { thread: input.threadId, err: detail });
|
|
905
|
+
try {
|
|
906
|
+
await seam.postToThread({ threadId: input.threadId, text: `tokenmaxxing: handling your reaction answer crashed: ${detail}. Reply in the thread to answer instead.` });
|
|
907
|
+
} catch (postErr) {
|
|
908
|
+
log("serve.reaction_crash_notice_failed", { thread: input.threadId, err: (postErr instanceof Error ? postErr.message : String(postErr)).slice(0, 300) });
|
|
909
|
+
}
|
|
910
|
+
await setStatus({ threadId: input.threadId, messageId: input.messageId, emoji: STATUS_EMOJI.failed, op: "add" });
|
|
911
|
+
await setStatus({ threadId: input.threadId, messageId: input.messageId, emoji: STATUS_EMOJI.processing, op: "remove" });
|
|
912
|
+
// handleTurn consumed the attention state (and its question mark)
|
|
913
|
+
// before the turn ran; a crashed answer must not eat the ask (cubic
|
|
914
|
+
// review catch, round 2). Restore both so the nudge sweep and the
|
|
915
|
+
// reaction-answer gates keep working; the restore itself is
|
|
916
|
+
// best-effort (the crash may BE an unreadable record).
|
|
917
|
+
try {
|
|
918
|
+
const cur = loadSlackThread(input.threadId);
|
|
919
|
+
if (cur && !cur.attention) {
|
|
920
|
+
saveSlackThread({ ...cur, attention: asked });
|
|
921
|
+
if (asked.messageId) {
|
|
922
|
+
await setStatus({ threadId: input.threadId, messageId: asked.messageId, emoji: STATUS_EMOJI.attention, op: "add" });
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
} catch (restoreErr) {
|
|
926
|
+
log("serve.attention_restore_failed", { thread: input.threadId, err: (restoreErr instanceof Error ? restoreErr.message : String(restoreErr)).slice(0, 300) });
|
|
927
|
+
}
|
|
928
|
+
}
|
|
842
929
|
return;
|
|
843
930
|
}
|
|
844
931
|
const home = await seam.isHomeUser({ userId: input.userId });
|
|
@@ -868,7 +955,32 @@ export function buildServeRuntime(seam: {
|
|
|
868
955
|
const nudgeSweep = async (input?: { now?: number }) => {
|
|
869
956
|
if (draining) return;
|
|
870
957
|
const now = input?.now ?? Date.now();
|
|
871
|
-
|
|
958
|
+
// per-record parsing, not listSlackThreads: state files that fail to
|
|
959
|
+
// parse THROW by contract, but here one corrupt record aborting the
|
|
960
|
+
// whole sweep would silence every OTHER thread's overdue nudge on every
|
|
961
|
+
// tick (codex review catch) - and before the daemon's rejection backstop
|
|
962
|
+
// existed, this bare-interval throw was a whole-daemon crash killing
|
|
963
|
+
// every in-flight turn (2026-07-27 report shape). The skip is logged
|
|
964
|
+
// once per file per daemon run; a 60s tick would repeat it forever.
|
|
965
|
+
let files: string[] = [];
|
|
966
|
+
try {
|
|
967
|
+
files = existsSync(paths.slackThreadsDir) ? readdirSync(paths.slackThreadsDir) : [];
|
|
968
|
+
} catch (e) {
|
|
969
|
+
log("serve.nudge_sweep_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
for (const f of files) {
|
|
973
|
+
if (!f.endsWith(".json")) continue;
|
|
974
|
+
let record: SlackThread;
|
|
975
|
+
try {
|
|
976
|
+
record = SlackThreadSchema.parse(JSON.parse(readFileSync(join(paths.slackThreadsDir, f), "utf8")));
|
|
977
|
+
} catch (e) {
|
|
978
|
+
if (!sweepSkipLogged.has(f)) {
|
|
979
|
+
sweepSkipLogged.add(f);
|
|
980
|
+
log("serve.nudge_record_skipped", { file: f, err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
|
|
981
|
+
}
|
|
982
|
+
continue;
|
|
983
|
+
}
|
|
872
984
|
const asked = record.attention;
|
|
873
985
|
if (!asked || asked.nudgedAt !== undefined || now - Date.parse(asked.askedAt) < ATTENTION_NUDGE_MS) continue;
|
|
874
986
|
// unlinked channels are contractually silent in Slack (review catch:
|
|
@@ -1338,6 +1450,21 @@ async function runDaemon(): Promise<number> {
|
|
|
1338
1450
|
// 2s, zero API calls).
|
|
1339
1451
|
process.on("SIGHUP", () => void shutdown("SIGHUP"));
|
|
1340
1452
|
|
|
1453
|
+
// LAST-RESORT backstop, not the error strategy: `tracked` is the daemon's
|
|
1454
|
+
// own boundary, so this should stay idle - it exists for rejections minted
|
|
1455
|
+
// outside our funnels (the chat SDK's socket client, adapter internals),
|
|
1456
|
+
// where Bun's default is to kill the whole process (verified 2026-07-27)
|
|
1457
|
+
// and with it every concurrent session's in-flight turn, leaving
|
|
1458
|
+
// half-streamed Slack messages and no daemon to resume the markers.
|
|
1459
|
+
// Sync throws keep the default crash: an uncaughtException means state is
|
|
1460
|
+
// undefined and the durable markers make a restart the honest recovery.
|
|
1461
|
+
// message-only, like every other logged error here: a rejection minted by
|
|
1462
|
+
// an auth-carrying HTTP client must not persist its request into the log
|
|
1463
|
+
// (cursor review catch).
|
|
1464
|
+
process.on("unhandledRejection", (reason) => {
|
|
1465
|
+
log("serve.unhandled_rejection", { err: (reason instanceof Error ? reason.message : String(reason)).slice(0, 300) });
|
|
1466
|
+
});
|
|
1467
|
+
|
|
1341
1468
|
// initialize() starts the PERSISTENT Socket Mode client (auto-reconnecting)
|
|
1342
1469
|
// wired straight into event routing; the daemon only has to stay alive.
|
|
1343
1470
|
// Never call startSocketModeListener here: that is the serverless leased
|
package/src/lib/install.ts
CHANGED
|
@@ -406,8 +406,9 @@ export function findClaudeShadowers(rcText: string): ShellShadower[] {
|
|
|
406
406
|
/** Remove ONLY the marker-tagged PATH line this tool added; a hand-added
|
|
407
407
|
* PATH entry without the marker is the user's own. A stale `# tokenmaxxing
|
|
408
408
|
* PATH` line pointing at an emptied binDir is exactly how the supervisor
|
|
409
|
-
* recursion incident started (
|
|
410
|
-
* one behind (closing-review catch).
|
|
409
|
+
* recursion incident started (.memory/supervisor-recursion-guards.md), so
|
|
410
|
+
* uninstall must not leave one behind (closing-review catch).
|
|
411
|
+
* Returns true when a line was removed. */
|
|
411
412
|
export function removePathFromRc(rc: string): boolean {
|
|
412
413
|
if (!existsSync(rc)) return false;
|
|
413
414
|
// same symlink + mode treatment as ensurePathInRc: write through a
|
package/src/lib/slackbridge.ts
CHANGED
|
@@ -75,6 +75,24 @@ const RETRY_DELAY_MS = 10_000;
|
|
|
75
75
|
/** parks + retries per Slack message; keeps a stale usage cache from looping
|
|
76
76
|
* a thread forever. */
|
|
77
77
|
export const MAX_RECOVERIES = 3;
|
|
78
|
+
/** extra spawns for a NON-limit failure whose child never completed (crash,
|
|
79
|
+
* API blip, errored result): each resumes the same session, so a retry
|
|
80
|
+
* continues the turn instead of re-running it. Separate from MAX_RECOVERIES
|
|
81
|
+
* on purpose - that budget bounds limit-driven parking, this one bounds
|
|
82
|
+
* quota spent chasing a possibly-permanent error. */
|
|
83
|
+
export const MAX_TRANSIENT_RETRIES = 2;
|
|
84
|
+
/** Slack expires a native stream SERVER-SIDE on undocumented timers
|
|
85
|
+
* (Slack-maintainer-confirmed in slackapi/python-slack-sdk#1859: idle
|
|
86
|
+
* around 30s, total lifetime around 300s measured), and an expired stream
|
|
87
|
+
* freezes in the Slack client as a grey "Something went wrong" pill - the
|
|
88
|
+
* exact 2026-07-27 report. The salvage path recovers the CONTENT but
|
|
89
|
+
* cannot un-freeze the pill, so the fix is to never let Slack expire a
|
|
90
|
+
* stream we own: rotate the open segment - a clean end() that the adapter
|
|
91
|
+
* finishes with a proper stream stop - before either timer can fire, and
|
|
92
|
+
* let the next chunk open a fresh message, the same flow pushText's size
|
|
93
|
+
* splits already use. Thresholds sit well inside Slack's observed margins;
|
|
94
|
+
* read at every arm (a mutable object, the file's test seam pattern). */
|
|
95
|
+
export const SEGMENT_ROTATION = { idleMs: 20_000, maxAgeMs: 240_000 };
|
|
78
96
|
|
|
79
97
|
const ParkPlanSchema = z.union([
|
|
80
98
|
z.object({ kind: z.literal("proceed") }),
|
|
@@ -469,6 +487,14 @@ export async function relayThread(input: {
|
|
|
469
487
|
// a ``` split across two deltas would be invisible to per-chunk counting
|
|
470
488
|
// (pullfrog review catch, PR #42).
|
|
471
489
|
let segmentMeta: { text: boolean; acc: string; reopenFence: boolean } | null = null;
|
|
490
|
+
// the open segment's rotation clock (see armSegmentTimer).
|
|
491
|
+
let segmentTimer: ReturnType<typeof setTimeout> | null = null;
|
|
492
|
+
let segmentOpenedAt = 0;
|
|
493
|
+
// a timer rotation closed the segment inside a code fence: the next
|
|
494
|
+
// segment must reopen it, exactly like pushText's size splits do (codex
|
|
495
|
+
// review catch). Consumed by openSegment; the salvage handler's own
|
|
496
|
+
// delivered-prefix recomputation overrides it there.
|
|
497
|
+
let pendingReopenFence = false;
|
|
472
498
|
let lastPost: Promise<unknown> = Promise.resolve();
|
|
473
499
|
// Reply TEXT that died with a rejected segment and was neither salvaged into
|
|
474
500
|
// a follow-on message nor re-delivered by a later text-bearing segment: the
|
|
@@ -496,6 +522,8 @@ export async function relayThread(input: {
|
|
|
496
522
|
const openSegment = () => {
|
|
497
523
|
const seg = pushableStream();
|
|
498
524
|
segment = seg;
|
|
525
|
+
segmentOpenedAt = Date.now();
|
|
526
|
+
armSegmentTimer();
|
|
499
527
|
// reopenFence: the delivered prefix of a REJECTED predecessor left a code
|
|
500
528
|
// fence open, so the first TEXT entering this salvage segment must be
|
|
501
529
|
// preceded by a reopen or it renders outside the code block. Pending
|
|
@@ -503,7 +531,8 @@ export async function relayThread(input: {
|
|
|
503
531
|
// salvage would otherwise either skip the reopen (later text joining the
|
|
504
532
|
// segment renders unfenced) or dangle an empty open fence at message end
|
|
505
533
|
// when no text ever follows. Materialized by BOTH text entry points.
|
|
506
|
-
const meta = { text: false, acc: "", reopenFence:
|
|
534
|
+
const meta = { text: false, acc: "", reopenFence: pendingReopenFence };
|
|
535
|
+
pendingReopenFence = false;
|
|
507
536
|
segmentMeta = meta;
|
|
508
537
|
lastPost = input.post(seg.iterable).then(
|
|
509
538
|
() => {
|
|
@@ -608,9 +637,19 @@ export async function relayThread(input: {
|
|
|
608
637
|
// the salvage segment here keeps the salvaged content ordered ahead
|
|
609
638
|
// of any push still awaiting lastPost; the salvage segment's own
|
|
610
639
|
// settle then decides whether its text counts as delivered.
|
|
640
|
+
// A salvage always recomputes its fence state from the delivered
|
|
641
|
+
// prefix (the override below), so it must never CONSUME a pending
|
|
642
|
+
// rotation reopen that belongs to a different segment's
|
|
643
|
+
// continuation - a rejected notice settling after notify()
|
|
644
|
+
// restored the flag would otherwise eat it (cubic review catch,
|
|
645
|
+
// round 4). Held across the lost-chunk re-push too: the salvaged
|
|
646
|
+
// text is the dying segment's, not the continuation's.
|
|
647
|
+
const heldReopen = pendingReopenFence;
|
|
648
|
+
pendingReopenFence = false;
|
|
611
649
|
const next = openSegment();
|
|
612
650
|
next.meta.reopenFence = deliveredFenceOpen;
|
|
613
651
|
for (const c of lost) next.pushInto(c);
|
|
652
|
+
pendingReopenFence = heldReopen;
|
|
614
653
|
} else if (textRemainder !== "") {
|
|
615
654
|
textLost = true;
|
|
616
655
|
textLostDetail = detail;
|
|
@@ -637,6 +676,7 @@ export async function relayThread(input: {
|
|
|
637
676
|
meta.acc += chunk;
|
|
638
677
|
}
|
|
639
678
|
seg.push(chunk);
|
|
679
|
+
if (segment === seg) armSegmentTimer();
|
|
640
680
|
};
|
|
641
681
|
return { seg, meta, pushInto };
|
|
642
682
|
};
|
|
@@ -649,6 +689,17 @@ export async function relayThread(input: {
|
|
|
649
689
|
}
|
|
650
690
|
const target = segment ?? openSegment().seg;
|
|
651
691
|
if (!(chunk instanceof Object)) {
|
|
692
|
+
// a pending rotation reopen can outlive its openSegment hand-off when
|
|
693
|
+
// text JOINS a segment a rejection handler opened during the wait
|
|
694
|
+
// (the salvage recomputes its own fence state without consuming the
|
|
695
|
+
// flag): adopt it here so the continuation still reopens its fence,
|
|
696
|
+
// UNLESS the joined segment already sits inside an open fence - then
|
|
697
|
+
// the state is satisfied and a second marker would close it (cubic
|
|
698
|
+
// review catch, round 4).
|
|
699
|
+
if (pendingReopenFence) {
|
|
700
|
+
pendingReopenFence = false;
|
|
701
|
+
if (!fenceOpen()) segmentMeta!.reopenFence = true;
|
|
702
|
+
}
|
|
652
703
|
// materialize a salvage segment's pending fence reopen (see
|
|
653
704
|
// openSegment) before the first text from this entry point too.
|
|
654
705
|
if (segmentMeta!.reopenFence) {
|
|
@@ -661,14 +712,42 @@ export async function relayThread(input: {
|
|
|
661
712
|
segmentMeta!.acc += chunk;
|
|
662
713
|
}
|
|
663
714
|
target.push(chunk);
|
|
715
|
+
if (segment === target) armSegmentTimer();
|
|
664
716
|
};
|
|
665
717
|
// parity by occurrence count over the segment's accumulated text: an odd
|
|
666
718
|
// number of ``` markers means the segment currently sits inside a fence.
|
|
667
719
|
const fenceOpen = () => segmentMeta !== null && (segmentMeta.acc.split("```").length - 1) % 2 === 1;
|
|
668
720
|
const breakSegment = () => {
|
|
721
|
+
if (segmentTimer !== null) {
|
|
722
|
+
clearTimeout(segmentTimer);
|
|
723
|
+
segmentTimer = null;
|
|
724
|
+
}
|
|
725
|
+
// a segment dying with an armed-but-unmaterialized reopen (no text ever
|
|
726
|
+
// arrived - a card-only segment that consumed the flag at open) folds it
|
|
727
|
+
// back: the reply still logically sits inside an open fence, and the
|
|
728
|
+
// next text-bearing segment must reopen it (cubic review catch, round
|
|
729
|
+
// 4: the errored result's closing card ate the rotation's reopen).
|
|
730
|
+
if (segmentMeta?.reopenFence) pendingReopenFence = true;
|
|
669
731
|
segment?.end();
|
|
670
732
|
segment = null;
|
|
671
733
|
};
|
|
734
|
+
const armSegmentTimer = () => {
|
|
735
|
+
if (segmentTimer !== null) clearTimeout(segmentTimer);
|
|
736
|
+
const ageLeft = segmentOpenedAt + SEGMENT_ROTATION.maxAgeMs - Date.now();
|
|
737
|
+
segmentTimer = setTimeout(() => {
|
|
738
|
+
segmentTimer = null;
|
|
739
|
+
// a rotation mid-fence closes the fence and arms the reopen, so both
|
|
740
|
+
// message halves render as code - pushText's split contract. An
|
|
741
|
+
// armed-but-unmaterialized reopen needs no close marker; breakSegment
|
|
742
|
+
// folds it forward.
|
|
743
|
+
if (segment !== null && fenceOpen()) {
|
|
744
|
+
segmentMeta!.acc += "\n```";
|
|
745
|
+
segment.push("\n```");
|
|
746
|
+
pendingReopenFence = true;
|
|
747
|
+
}
|
|
748
|
+
breakSegment();
|
|
749
|
+
}, Math.max(0, Math.min(SEGMENT_ROTATION.idleMs, ageLeft)));
|
|
750
|
+
};
|
|
672
751
|
/** Reply text routed through here splits across Slack messages before the
|
|
673
752
|
* msg_too_long cap (see SEGMENT_TEXT_MAX): a break prefers the last newline
|
|
674
753
|
* inside the remaining room, and a break forced inside a code fence closes
|
|
@@ -723,8 +802,16 @@ export async function relayThread(input: {
|
|
|
723
802
|
// streamed segment.
|
|
724
803
|
const notify = async (text: string) => {
|
|
725
804
|
breakSegment();
|
|
805
|
+
// a notice must never inherit a rotation's pending fence reopen: the
|
|
806
|
+
// reopen belongs to the interrupted reply's continuation, not to the
|
|
807
|
+
// status line - consuming it here would render the notice as a code
|
|
808
|
+
// block AND strand the continuation unfenced (cubic review catch,
|
|
809
|
+
// round 3). Held aside and restored for the real continuation.
|
|
810
|
+
const heldReopen = pendingReopenFence;
|
|
811
|
+
pendingReopenFence = false;
|
|
726
812
|
await push(text);
|
|
727
813
|
breakSegment();
|
|
814
|
+
pendingReopenFence = heldReopen;
|
|
728
815
|
};
|
|
729
816
|
/** notify + confirm the post actually landed in Slack. A drop notice that
|
|
730
817
|
* never reached the user must NOT count as announced. DURING A DRAIN an
|
|
@@ -759,6 +846,9 @@ export async function relayThread(input: {
|
|
|
759
846
|
// deferral notice explains the pause; the raw line would invite a manual
|
|
760
847
|
// re-send of work the daemon resumes itself - cubic catch, PR #44).
|
|
761
848
|
let pendingFailureLine: string | null = null;
|
|
849
|
+
// what the next spawn submits: the original message, until a transient
|
|
850
|
+
// retry swaps in the continuation wrapper (see the retry branch).
|
|
851
|
+
let prompt = input.prompt;
|
|
762
852
|
const runQueryOnce = async () => {
|
|
763
853
|
postedText = false;
|
|
764
854
|
pendingFailureLine = null;
|
|
@@ -777,7 +867,7 @@ export async function relayThread(input: {
|
|
|
777
867
|
spawnOrg = readOAuthAccount()?.organizationUuid ?? null;
|
|
778
868
|
const pooled = pooledOptions();
|
|
779
869
|
const q = query({
|
|
780
|
-
prompt
|
|
870
|
+
prompt,
|
|
781
871
|
options: {
|
|
782
872
|
...pooled,
|
|
783
873
|
// claude >= 2.1.142 emits the structured Task tools by default and
|
|
@@ -868,6 +958,17 @@ export async function relayThread(input: {
|
|
|
868
958
|
// the stale pre-limit snapshot (poll TTL) and respawns the same
|
|
869
959
|
// depleted account - a serve process has no statusLine tee.
|
|
870
960
|
if (outcome.rateLimited) await recordObservedLimit({ text, now: Date.now(), org: spawnOrg });
|
|
961
|
+
// hold the REAL errored text for the terminal diagnostic (codex
|
|
962
|
+
// review catches: a streamed-then-errored turn used to end with
|
|
963
|
+
// a truncated answer and only the x reaction, and a no-text
|
|
964
|
+
// errored turn used to discard the reason for a generic line).
|
|
965
|
+
// The depleted-pool probe still outranks the line.
|
|
966
|
+
else {
|
|
967
|
+
const reason = text.slice(0, 200) || "no error detail";
|
|
968
|
+
pendingFailureLine = postedText
|
|
969
|
+
? `tokenmaxxing: the turn errored before finishing (${reason}) - the reply above may be incomplete.`
|
|
970
|
+
: `tokenmaxxing: the turn errored without a result (${reason}) - trying again may help.`;
|
|
971
|
+
}
|
|
871
972
|
} else {
|
|
872
973
|
result = message.result;
|
|
873
974
|
outcome.resultReceived = true;
|
|
@@ -880,12 +981,6 @@ export async function relayThread(input: {
|
|
|
880
981
|
}
|
|
881
982
|
// a turn that produced no streamed text (tool-only turns) still reports.
|
|
882
983
|
if (!postedText && result) await pushText(result);
|
|
883
|
-
if (!postedText && !result && outcome.failed && !outcome.rateLimited) {
|
|
884
|
-
// held back until the depleted-pool probe rules: a "trying again may
|
|
885
|
-
// help" line right before a deferral notice invites a manual re-send
|
|
886
|
-
// of work the daemon is about to resume itself (cubic catch, PR #44).
|
|
887
|
-
pendingFailureLine = "the turn ended without a result - trying again may help";
|
|
888
|
-
}
|
|
889
984
|
} catch (e) {
|
|
890
985
|
outcome.failed = true;
|
|
891
986
|
const detail = (e instanceof Error ? e.message : String(e)).slice(0, 300);
|
|
@@ -900,6 +995,7 @@ export async function relayThread(input: {
|
|
|
900
995
|
};
|
|
901
996
|
|
|
902
997
|
let recoveries = 0;
|
|
998
|
+
let transientRetries = 0;
|
|
903
999
|
const parkDeadline = Date.now() + PARK_MAX_MS;
|
|
904
1000
|
while (true) {
|
|
905
1001
|
// the switch decision runs at the spawn boundary, same as the CLI hooks.
|
|
@@ -956,10 +1052,14 @@ export async function relayThread(input: {
|
|
|
956
1052
|
// the evidence the error text did not carry. A completed result stays
|
|
957
1053
|
// terminal, and a usable pool keeps the plain failure.
|
|
958
1054
|
if (!outcome.resultReceived) {
|
|
1055
|
+
// outranks the transient retry below even when the recovery time is
|
|
1056
|
+
// unknown (cubic review catch): a depleted pool explains the failure,
|
|
1057
|
+
// and respawning against it would just burn doomed attempts.
|
|
1058
|
+
let probeDepleted = false;
|
|
959
1059
|
try {
|
|
960
1060
|
const verdict = await ensureBestAccount();
|
|
961
|
-
|
|
962
|
-
const wake =
|
|
1061
|
+
probeDepleted = verdict.reason === "all-depleted" || verdict.reason === "depleted-wait";
|
|
1062
|
+
const wake = probeDepleted ? verdict.waitUntil ?? null : null;
|
|
963
1063
|
if (wake != null) {
|
|
964
1064
|
outcome.rateLimited = true;
|
|
965
1065
|
outcome.deferUntil = wake + PARK_GRACE_MS;
|
|
@@ -971,8 +1071,43 @@ export async function relayThread(input: {
|
|
|
971
1071
|
// keep the original failure; the probe must never mask it.
|
|
972
1072
|
log("serve.defer_probe_error", { err: (e instanceof Error ? e.message : String(e)).slice(0, 300) });
|
|
973
1073
|
}
|
|
1074
|
+
// a transient non-limit failure (child crash, API blip, errored
|
|
1075
|
+
// result) against a usable pool retries silently into the same
|
|
1076
|
+
// session, mirroring the rate-limit path's invisible short retry
|
|
1077
|
+
// (2026-07-27 report: these turns died terminally, telling the user
|
|
1078
|
+
// "trying again may help" instead of trying again). resultReceived
|
|
1079
|
+
// stays terminal above - a completed answer must never re-run - and
|
|
1080
|
+
// the probe's deferral outranks a retry: an exhausted pool explains
|
|
1081
|
+
// the failure better than "transient".
|
|
1082
|
+
if (!probeDepleted && outcome.deferUntil === null && transientRetries < MAX_TRANSIENT_RETRIES) {
|
|
1083
|
+
transientRetries += 1;
|
|
1084
|
+
breakSegment();
|
|
1085
|
+
log("serve.transient_retry", { attempt: transientRetries });
|
|
1086
|
+
// once a session exists, the failed attempt may have executed
|
|
1087
|
+
// side-effectful tools before dying, so the retry must CONTINUE,
|
|
1088
|
+
// never re-instruct: replaying the original prompt verbatim into
|
|
1089
|
+
// the resumed session reads as "do it again" (codex review catch).
|
|
1090
|
+
// The wrapper mirrors resumeDecision's deferral resume. A pre-init
|
|
1091
|
+
// death ran nothing, so the original replays verbatim there.
|
|
1092
|
+
if (outcome.sessionId !== null) {
|
|
1093
|
+
prompt = `Your previous turn was interrupted by an error mid-run; this session's transcript already holds any work it completed, including tool calls whose side effects already happened. Pick up exactly where you left off and finish the task without re-running completed side-effectful steps. If the work was already complete, just summarize the final state. The original request was:\n\n${input.prompt}`;
|
|
1094
|
+
}
|
|
1095
|
+
if (!(await sleep(RETRY_DELAY_MS))) {
|
|
1096
|
+
// a drain aborted the retry sleep: same as a killed child, the
|
|
1097
|
+
// marker survives (presumedKilled) and the next daemon start
|
|
1098
|
+
// resumes the session where it stopped.
|
|
1099
|
+
await notify("tokenmaxxing is restarting - this turn resumes after the restart.");
|
|
1100
|
+
break;
|
|
1101
|
+
}
|
|
1102
|
+
continue;
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
if (pendingFailureLine !== null) {
|
|
1106
|
+
// same fence-reopen bypass as notify: a diagnostic line opening a
|
|
1107
|
+
// fresh segment must not render inside a reopened code fence.
|
|
1108
|
+
pendingReopenFence = false;
|
|
1109
|
+
await push(transientRetries > 0 ? `${pendingFailureLine} (after ${transientRetries + 1} attempts)` : pendingFailureLine);
|
|
974
1110
|
}
|
|
975
|
-
if (pendingFailureLine !== null) await push(pendingFailureLine);
|
|
976
1111
|
break;
|
|
977
1112
|
}
|
|
978
1113
|
if (recoveries >= MAX_RECOVERIES) {
|