unsnooze 1.19.0 → 1.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -0
- package/README.md +23 -5
- package/package.json +1 -1
- package/src/agents/codex.js +96 -12
- package/src/cli.js +4 -0
- package/src/config.js +6 -0
- package/src/doctor.js +77 -0
- package/src/install.js +2 -2
- package/src/launcher.js +32 -8
- package/src/multiplexers/headless.js +159 -11
- package/src/resumer.js +65 -7
- package/src/time-parser.js +6 -1
- package/src/usage.js +50 -21
- package/src/watcher.js +45 -12
- package/src/watchers/codex.js +190 -24
- package/src/which.js +63 -0
|
@@ -18,7 +18,10 @@
|
|
|
18
18
|
// session sweep skips headless rather than claiming to own anything.
|
|
19
19
|
|
|
20
20
|
import { spawn } from 'node:child_process';
|
|
21
|
-
import {
|
|
21
|
+
import {
|
|
22
|
+
mkdirSync, openSync, closeSync, readSync, statSync, readFileSync, writeFileSync,
|
|
23
|
+
renameSync, unlinkSync, readdirSync,
|
|
24
|
+
} from 'node:fs';
|
|
22
25
|
import { join } from 'node:path';
|
|
23
26
|
|
|
24
27
|
import { HEADLESS_LOG_DIR } from '../config.js';
|
|
@@ -27,6 +30,12 @@ export const SUBMIT_DELAY_MS = 0; // nothing is ever typed
|
|
|
27
30
|
|
|
28
31
|
const PID_PREFIX = 'pid:';
|
|
29
32
|
|
|
33
|
+
// How long a recorded exit stays on disk. A revival is verified ~20s after
|
|
34
|
+
// launch and retried within the hour; a day covers a daemon that was asleep.
|
|
35
|
+
const EXIT_TTL_MS = 24 * 3_600_000;
|
|
36
|
+
// Enough of the child's own output to say why it died, never the whole log.
|
|
37
|
+
const EXIT_OUTPUT_BYTES = 4096;
|
|
38
|
+
|
|
30
39
|
export function parsePidAddress(pane) {
|
|
31
40
|
if (typeof pane !== 'string' || !pane.startsWith(PID_PREFIX)) return null;
|
|
32
41
|
const pid = Number.parseInt(pane.slice(PID_PREFIX.length), 10);
|
|
@@ -60,6 +69,64 @@ export function createHeadless({
|
|
|
60
69
|
env = process.env,
|
|
61
70
|
platform = process.platform,
|
|
62
71
|
} = {}) {
|
|
72
|
+
const exitsDir = join(logDir, 'exits');
|
|
73
|
+
const exitPath = pid => join(exitsDir, `${pid}.json`);
|
|
74
|
+
|
|
75
|
+
function writeExit(pid, record) {
|
|
76
|
+
try {
|
|
77
|
+
mkdirSync(exitsDir, { recursive: true, mode: 0o700 });
|
|
78
|
+
const tmp = join(exitsDir, `.${pid}.${process.pid}.tmp`);
|
|
79
|
+
writeFileSync(tmp, JSON.stringify(record), { mode: 0o600 });
|
|
80
|
+
renameSync(tmp, exitPath(pid));
|
|
81
|
+
} catch { /* evidence only — a revival must never fail on bookkeeping */ }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function readExit(pid) {
|
|
85
|
+
try {
|
|
86
|
+
const parsed = JSON.parse(readFileSync(exitPath(pid), 'utf-8'));
|
|
87
|
+
return parsed && typeof parsed === 'object' && parsed.pid === pid ? parsed : null;
|
|
88
|
+
} catch {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function pruneExits(now = Date.now()) {
|
|
94
|
+
let names;
|
|
95
|
+
try { names = readdirSync(exitsDir); } catch { return; }
|
|
96
|
+
for (const name of names) {
|
|
97
|
+
if (!name.endsWith('.json')) continue;
|
|
98
|
+
const path = join(exitsDir, name);
|
|
99
|
+
try {
|
|
100
|
+
if (now - statSync(path).mtimeMs > EXIT_TTL_MS) unlinkSync(path);
|
|
101
|
+
} catch { /* raced or gone */ }
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// The last lines the child wrote to the shared log, read between the offsets
|
|
106
|
+
// recorded at its launch and at its exit. The upper bound matters: the
|
|
107
|
+
// resumer staggers revivals 8s apart and verifies 20s later, so a revival
|
|
108
|
+
// that died at once is read back after the next one has written its own
|
|
109
|
+
// lines into the same log. (One running alongside it can still interleave.)
|
|
110
|
+
function childOutput(exit) {
|
|
111
|
+
if (!exit.log || !Number.isFinite(exit.from)) return null;
|
|
112
|
+
let fd;
|
|
113
|
+
try {
|
|
114
|
+
const { size } = statSync(exit.log);
|
|
115
|
+
const end = Number.isFinite(exit.to) ? Math.min(exit.to, size) : size;
|
|
116
|
+
const start = Math.max(exit.from, end - EXIT_OUTPUT_BYTES);
|
|
117
|
+
if (end <= start) return null;
|
|
118
|
+
fd = openSync(exit.log, 'r');
|
|
119
|
+
const buf = Buffer.alloc(end - start);
|
|
120
|
+
const n = readSync(fd, buf, 0, buf.length, start);
|
|
121
|
+
const lines = buf.toString('utf-8', 0, n).split(/\r?\n/).map(l => l.trim()).filter(Boolean);
|
|
122
|
+
return lines.length ? lines.slice(-3).join(' | ') : null;
|
|
123
|
+
} catch {
|
|
124
|
+
return null;
|
|
125
|
+
} finally {
|
|
126
|
+
if (fd !== undefined) closeSync(fd);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
63
130
|
const backend = {
|
|
64
131
|
name: 'headless',
|
|
65
132
|
SUBMIT_DELAY_MS,
|
|
@@ -95,24 +162,105 @@ export function createHeadless({
|
|
|
95
162
|
|
|
96
163
|
// A revive is just a detached child. Its output goes to a per-session log
|
|
97
164
|
// because there is no scrollback to read it out of later.
|
|
165
|
+
//
|
|
166
|
+
// The child's exit is recorded next to that log (exits/<pid>.json). With no
|
|
167
|
+
// pane to capture, an empty capture proves nothing — verifyOne used to read
|
|
168
|
+
// it as a cleared banner and mark a launcher that had died with `spawn
|
|
169
|
+
// codex ENOENT` as resumed (#25). The exit record, plus the slice of the
|
|
170
|
+
// log the child wrote, is what paneOutcome() answers with instead.
|
|
98
171
|
async newWindow(sessionName, cwd, launchSpec) {
|
|
99
172
|
// 0700/0600: this log is the entire stdout+stderr of an unattended
|
|
100
173
|
// agent run — the most revealing thing under the state dir.
|
|
101
174
|
mkdirSync(logDir, { recursive: true, mode: 0o700 });
|
|
102
175
|
const logPath = join(logDir, `${sessionName}.log`);
|
|
103
176
|
const fd = openSync(logPath, 'a', 0o600);
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
177
|
+
// The log is shared by every revival into this session name, so remember
|
|
178
|
+
// where this child's output starts.
|
|
179
|
+
let from = 0;
|
|
180
|
+
try { from = statSync(logPath).size; } catch { /* just created */ }
|
|
181
|
+
// The child gets this process's environment — the one backend that
|
|
182
|
+
// passes it through. A resumer spawned by the StopFailure hook runs in
|
|
183
|
+
// claude's, which carries UNSNOOZE_ACTIVE=1: the launcher's "nested
|
|
184
|
+
// call, pass straight through" marker. A revival is a fresh top-level
|
|
185
|
+
// launch; with the marker it dropped launchExtraArgs and, on a spawn
|
|
186
|
+
// failure, exited 1 without saying why.
|
|
187
|
+
const childEnv = { ...env, ...launchSpec.env };
|
|
188
|
+
delete childEnv.UNSNOOZE_ACTIVE;
|
|
189
|
+
let child;
|
|
190
|
+
try {
|
|
191
|
+
child = spawner(launchSpec.file, launchSpec.args || [], {
|
|
192
|
+
cwd,
|
|
193
|
+
detached: true,
|
|
194
|
+
stdio: ['ignore', fd, fd],
|
|
195
|
+
env: childEnv,
|
|
196
|
+
windowsHide: true,
|
|
197
|
+
});
|
|
198
|
+
} finally {
|
|
199
|
+
// The child has its own copy; the daemon's would otherwise stay open
|
|
200
|
+
// for the life of the process, one descriptor per revival.
|
|
201
|
+
try { closeSync(fd); } catch { /* already closed */ }
|
|
202
|
+
}
|
|
203
|
+
const pid = child?.pid;
|
|
204
|
+
const startedAt = Date.now();
|
|
205
|
+
let spawnError = null;
|
|
206
|
+
// Listen before anything below can throw. A spawn that fails — the
|
|
207
|
+
// session's cwd was a worktree that has since been deleted, say — says
|
|
208
|
+
// so with an 'error' event after spawn() has returned, and an 'error'
|
|
209
|
+
// nobody listens for is an uncaught exception: the daemon dies with it.
|
|
210
|
+
if (typeof child?.on === 'function') {
|
|
211
|
+
const record = (code, signal, error) => {
|
|
212
|
+
if (!pid) return;
|
|
213
|
+
// Where this child's output ends in the shared log (childOutput).
|
|
214
|
+
let to = null;
|
|
215
|
+
try { to = statSync(logPath).size; } catch { /* gone */ }
|
|
216
|
+
writeExit(pid, {
|
|
217
|
+
pid, code, signal: signal ?? null, error: error ?? null,
|
|
218
|
+
startedAt, at: Date.now(), log: logPath, from, to,
|
|
219
|
+
});
|
|
220
|
+
};
|
|
221
|
+
child.on('exit', (code, signal) => record(code, signal, null));
|
|
222
|
+
child.on('error', err => {
|
|
223
|
+
spawnError = err;
|
|
224
|
+
record(null, null, err?.message || String(err));
|
|
225
|
+
});
|
|
226
|
+
}
|
|
111
227
|
if (typeof child?.unref === 'function') child.unref();
|
|
112
|
-
if (!
|
|
113
|
-
|
|
228
|
+
if (!pid) {
|
|
229
|
+
// The reason arrives on the next tick; wait for it so the record's
|
|
230
|
+
// lastError says why, not just that there was no pid.
|
|
231
|
+
await new Promise(resolve => setImmediate(resolve));
|
|
232
|
+
throw new Error(`unsnooze: headless launch of ${launchSpec.file} produced no pid`
|
|
233
|
+
+ (spawnError ? ` (${spawnError.message})` : ''));
|
|
114
234
|
}
|
|
115
|
-
|
|
235
|
+
// A recycled pid must not inherit an old exit. Best-effort, like the
|
|
236
|
+
// pruning: an exit record is evidence, never something a launch needs.
|
|
237
|
+
// (No event can have fired yet — nothing above has yielded since spawn.)
|
|
238
|
+
try { unlinkSync(exitPath(pid)); } catch { /* none */ }
|
|
239
|
+
pruneExits();
|
|
240
|
+
return { pane: `${PID_PREFIX}${pid}`, paneOwner: null, session: sessionName };
|
|
241
|
+
},
|
|
242
|
+
|
|
243
|
+
// What became of a revival: { exited: false } while it runs, and once it is
|
|
244
|
+
// gone { exited: true, code, signal, output } from the recorded exit, where
|
|
245
|
+
// output is the tail of what the child itself wrote. null when the process
|
|
246
|
+
// is gone and nothing was recorded (a resumer that was not the parent) —
|
|
247
|
+
// the caller then knows exactly as much as it did before.
|
|
248
|
+
async paneOutcome(pane) {
|
|
249
|
+
const pid = parsePidAddress(pane);
|
|
250
|
+
if (pid === null) return null;
|
|
251
|
+
// The record first: newWindow clears any old one for this pid at spawn,
|
|
252
|
+
// so a record here is our child's exit — even if the pid has since been
|
|
253
|
+
// handed to another process (Windows recycles them quickly), which a
|
|
254
|
+
// liveness check alone would read as our revival still running.
|
|
255
|
+
const exit = readExit(pid);
|
|
256
|
+
if (!exit) return alive(pid) ? { exited: false } : null;
|
|
257
|
+
return {
|
|
258
|
+
exited: true,
|
|
259
|
+
code: exit.code ?? null,
|
|
260
|
+
signal: exit.signal ?? null,
|
|
261
|
+
error: exit.error ?? null,
|
|
262
|
+
output: childOutput(exit),
|
|
263
|
+
};
|
|
116
264
|
},
|
|
117
265
|
|
|
118
266
|
// Nothing to wrap into: the user's own terminal is the session. Returning
|
package/src/resumer.js
CHANGED
|
@@ -9,7 +9,7 @@ import { homedir } from 'node:os';
|
|
|
9
9
|
import { spawnSync } from 'node:child_process';
|
|
10
10
|
import { getMultiplexer, backendCanType } from './multiplexer.js';
|
|
11
11
|
import {
|
|
12
|
-
RESUMER_LOCK, POLL_INTERVAL_MS, STAGGER_MS, VERIFY_DELAY_MS, ensureStateDir,
|
|
12
|
+
RESUMER_LOCK, POLL_INTERVAL_MS, STAGGER_MS, VERIFY_DELAY_MS, HEADLESS_SETTLE_MS, ensureStateDir,
|
|
13
13
|
BUSY_DEFER_MS, MAX_BUSY_DEFERS, MAX_RESUME_ATTEMPTS, READY_TIMEOUT_MS,
|
|
14
14
|
CAPTURE_LINES, PANE_SCAN_LINES, RESUME_SESSION_NAME,
|
|
15
15
|
RESET_MARGIN_MS, FALLBACK_RESET_MS, PROBE_INTERVAL_MS, PROBE_MAX_MS,
|
|
@@ -123,6 +123,15 @@ export function resolveRecordMux(rec) {
|
|
|
123
123
|
return getMultiplexer(rec.mux, { owner: rec.paneOwner });
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
+
// The user's resumeExtraArgs, placed where the adapter's command parses them:
|
|
127
|
+
// appended, unless the adapter names an index (codex's `exec resume` takes
|
|
128
|
+
// its options after `exec`, not after the prompt).
|
|
129
|
+
function withResumeExtraArgs(resume, agentId) {
|
|
130
|
+
const extra = resolveResumeExtraArgs(agentId);
|
|
131
|
+
const at = Number.isInteger(resume.extraArgsAt) ? resume.extraArgsAt : resume.args.length;
|
|
132
|
+
return [...resume.args.slice(0, at), ...extra, ...resume.args.slice(at)];
|
|
133
|
+
}
|
|
134
|
+
|
|
126
135
|
const stopEpisodeAt = rec => rec?.bannerAt ?? rec?.detectedAt;
|
|
127
136
|
|
|
128
137
|
// Newer non-error parent-context Claude usage proves that a user/provider retry
|
|
@@ -366,13 +375,16 @@ function rescheduleProbe(rec, now = Date.now()) {
|
|
|
366
375
|
// never by waiting it out. Typing a wake into it would hit the same wall,
|
|
367
376
|
// so make the stall a visible terminal state instead of a futile resume.
|
|
368
377
|
if (rec.limitType === 'model') {
|
|
378
|
+
// The agent's own remedy: `unsnooze status` shows this line, and
|
|
379
|
+
// Claude's slash commands mean nothing to a Codex workspace wall.
|
|
380
|
+
const remedy = modelRemedy(getAgent(rec.agent));
|
|
369
381
|
setStatus(key, 'failed', {
|
|
370
|
-
lastError:
|
|
382
|
+
lastError: `model limit still active — ${remedy}`,
|
|
371
383
|
probeCount: probeCount + 1,
|
|
372
384
|
}, { expect: ['stopped'] });
|
|
373
385
|
log(`${key}: model limit still active at probe ceiling — needs a human`);
|
|
374
386
|
notify('unsnooze: model limit needs you ⚠️',
|
|
375
|
-
`${rec.cwd}: still limited after probing — ${
|
|
387
|
+
`${rec.cwd}: still limited after probing — ${remedy}`,
|
|
376
388
|
{ context: ctxOf(rec), priority: 4 });
|
|
377
389
|
return 'held';
|
|
378
390
|
}
|
|
@@ -585,10 +597,12 @@ export async function planFor(rec, {
|
|
|
585
597
|
return { ...base, action: 'superseded', target: { key: superseded.key } };
|
|
586
598
|
}
|
|
587
599
|
const target = await reviveTarget(mux, rec);
|
|
588
|
-
|
|
600
|
+
// The same argv reopen() builds: on a backend with no pane the prompt rides
|
|
601
|
+
// in argv, and for codex the command itself changes (`exec resume`).
|
|
602
|
+
const resume = agent.resumeArgs(rec.sessionId, message, { canType: backendCanType(mux) });
|
|
589
603
|
return {
|
|
590
604
|
...base, action: 'reopen', target: { session: target }, message,
|
|
591
|
-
argv: [agent.id, ...resume
|
|
605
|
+
argv: [agent.id, ...withResumeExtraArgs(resume, agent.id)],
|
|
592
606
|
messageViaPane: !!resume.messageViaPane,
|
|
593
607
|
};
|
|
594
608
|
}
|
|
@@ -758,7 +772,7 @@ async function reopen(rec, { mux, resolveMux, agent, resumeMessage, selfCmd, onD
|
|
|
758
772
|
const leaseId = createLeaseId();
|
|
759
773
|
const target = await reviveTarget(mux, rec);
|
|
760
774
|
const launchSpec = {
|
|
761
|
-
file: selfCmd[0], args: [...selfCmd.slice(1), '_run', agent.id, ...resume
|
|
775
|
+
file: selfCmd[0], args: [...selfCmd.slice(1), '_run', agent.id, ...withResumeExtraArgs(resume, agent.id)],
|
|
762
776
|
env: reopenEnv(rec, leaseId, target),
|
|
763
777
|
};
|
|
764
778
|
// reviveTarget can await a multiplexer query. Recheck after it and claim the
|
|
@@ -936,6 +950,44 @@ export async function verifyOne(key, { resolveMux = resolveRecordMux } = {}) {
|
|
|
936
950
|
log(`${key}: limit still active, rescheduled to ${new Date(at).toISOString()} (${source})`);
|
|
937
951
|
return;
|
|
938
952
|
}
|
|
953
|
+
// A pane-less backend answers every capture with '', so "no banner" is not
|
|
954
|
+
// evidence there. Headless records what became of the launcher it spawned;
|
|
955
|
+
// a child that already died non-zero (`spawn codex ENOENT`, the Codex TUI's
|
|
956
|
+
// "stdin is not a terminal") resumed nothing, and marking it resumed would
|
|
957
|
+
// drop a retryable stop from the ledger and announce a wake that never
|
|
958
|
+
// happened (#25). Exit 0 is fine — `claude --resume … "prompt"` runs to
|
|
959
|
+
// completion and exits before the verify delay — and so is running on
|
|
960
|
+
// past HEADLESS_SETTLE_MS. A revival that is still young stays in flight:
|
|
961
|
+
// the loop verifies every resuming record each tick, and once it counts as
|
|
962
|
+
// resumed nothing reads its exit again (the sweep drops the record when
|
|
963
|
+
// the pid goes), so a death at 25s would have been announced as a wake.
|
|
964
|
+
if (typeof mux.paneOutcome === 'function') {
|
|
965
|
+
let outcome = null;
|
|
966
|
+
try { outcome = await mux.paneOutcome(rec.pane); } catch { outcome = null; }
|
|
967
|
+
if (outcome && outcome.exited === false
|
|
968
|
+
&& Date.now() - (rec.lastAttemptAt || 0) < HEADLESS_SETTLE_MS) {
|
|
969
|
+
return 'pending';
|
|
970
|
+
}
|
|
971
|
+
if (outcome?.exited && outcome.code !== 0) {
|
|
972
|
+
const attempts = (rec.attempts || 0) + 1;
|
|
973
|
+
const how = outcome.code != null ? `exit ${outcome.code}`
|
|
974
|
+
: outcome.signal ? `killed by ${outcome.signal}` : 'failed to start';
|
|
975
|
+
const why = outcome.output || outcome.error || '';
|
|
976
|
+
const lastError = `revive died before it could resume (${how}${why ? `: ${why}` : ''})`;
|
|
977
|
+
const applied = transitionStopEpisode(rec, 'stopped', {
|
|
978
|
+
attempts,
|
|
979
|
+
// Same backoff (and same manual exemption) as every other retry.
|
|
980
|
+
resetAt: rec.manual ? Date.now() : Date.now() + retryBackoffMs(attempts),
|
|
981
|
+
lastError, verifyRetries: 0, resumeEpisodeAt: null,
|
|
982
|
+
}, { expect: ['resuming'] });
|
|
983
|
+
if (!applied) {
|
|
984
|
+
releaseSupersededResume(key);
|
|
985
|
+
return 'stale';
|
|
986
|
+
}
|
|
987
|
+
log(`${key}: ${lastError}`);
|
|
988
|
+
return 'retry';
|
|
989
|
+
}
|
|
990
|
+
}
|
|
939
991
|
if (!transitionStopEpisode(rec, 'resumed', {
|
|
940
992
|
lastError: null, verifyRetries: 0, resumeEpisodeAt: null,
|
|
941
993
|
bannerCleared: true,
|
|
@@ -1108,8 +1160,14 @@ export async function runResumer({
|
|
|
1108
1160
|
// Anything over the attempts cap is dead — mark failed so we can exit.
|
|
1109
1161
|
for (const s of dueForDispatch()) {
|
|
1110
1162
|
if ((s.attempts || 0) >= MAX_RESUME_ATTEMPTS) {
|
|
1163
|
+
// Keep the last attempt's reason: this is the state `unsnooze status`
|
|
1164
|
+
// shows after the "gave up" notification, and "exceeded" alone does
|
|
1165
|
+
// not say that every revival died with `spawn codex ENOENT` (#25).
|
|
1111
1166
|
const applied = transitionStopEpisode(s, 'failed', {
|
|
1112
|
-
lastError:
|
|
1167
|
+
lastError: s.lastError
|
|
1168
|
+
? `max resume attempts exceeded — last: ${s.lastError}`
|
|
1169
|
+
: 'max resume attempts exceeded',
|
|
1170
|
+
verifyRetries: 0,
|
|
1113
1171
|
resumeEpisodeAt: null,
|
|
1114
1172
|
}, { expect: ['stopped'] });
|
|
1115
1173
|
if (!applied) continue;
|
package/src/time-parser.js
CHANGED
|
@@ -278,7 +278,12 @@ export function resetAtMs(parsed, {
|
|
|
278
278
|
}
|
|
279
279
|
return t; // unreachable in practice — 9 steps cover a week + DST repeat
|
|
280
280
|
}
|
|
281
|
-
|
|
281
|
+
// A clock time names a whole minute. Codex prints the reset as %-I:%M and
|
|
282
|
+
// drops the seconds, so a dated banner written at 7:36:15 that says "try
|
|
283
|
+
// again at 7:36 AM" means a reset later in that same minute — not 7:36
|
|
284
|
+
// tomorrow, a day late. Only a minute that had fully passed rolls.
|
|
285
|
+
const past = bannerAt != null ? t + 60_000 <= anchor : t <= anchor;
|
|
286
|
+
if (past) {
|
|
282
287
|
// Dated banner, or ambiguous (need both am/pm candidates as real
|
|
283
288
|
// next-occurrences): roll to the next day. Undated non-ambiguous
|
|
284
289
|
// past clock times stay past so the outer guard returns due-now
|
package/src/usage.js
CHANGED
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
} from './config.js';
|
|
16
16
|
import { readState, updateState } from './state.js';
|
|
17
17
|
import { getConfig } from './settings.js';
|
|
18
|
-
import { ROLLOUT_RE } from './agents/codex.js';
|
|
18
|
+
import { ROLLOUT_RE, rolloutId } from './agents/codex.js';
|
|
19
19
|
import { shouldUseTui, formatUsageTui, bar as tuiBar } from './tui.js';
|
|
20
20
|
import { shouldUseDashboard, runDashboard } from './dashboard/run.js';
|
|
21
21
|
|
|
@@ -343,18 +343,17 @@ export function calibrationStopCount(calibration, agent, limitType, pool = null)
|
|
|
343
343
|
return n;
|
|
344
344
|
}
|
|
345
345
|
|
|
346
|
-
// Clamp
|
|
347
|
-
//
|
|
348
|
-
|
|
346
|
+
// Clamp an exact reading to [0, 100]; fall back to the previous one when the
|
|
347
|
+
// current is unparseable. Nothing else: the value is the server's own number
|
|
348
|
+
// (Codex copies it out of a response header, the Claude statusline hands it
|
|
349
|
+
// over verbatim), and there is no client-side estimate in it to smooth. This
|
|
350
|
+
// used to average a >15pp jump with the previous sample "for one tick" — but
|
|
351
|
+
// the report recomputes from the last two samples, and at the moment a user is
|
|
352
|
+
// blocked no further sample ever arrives, so a real 99% sat at 64% for the rest
|
|
353
|
+
// of the window and never crossed the 80/95 warn thresholds (#20 follow-up).
|
|
354
|
+
export function clampUsedPercent(current, previous) {
|
|
349
355
|
if (!Number.isFinite(current)) return Number.isFinite(previous) ? previous : null;
|
|
350
|
-
|
|
351
|
-
if (v > 100) v = 100;
|
|
352
|
-
if (v < 0) v = 0;
|
|
353
|
-
if (Number.isFinite(previous) && v > previous + 15 && previous < 95) {
|
|
354
|
-
// Soft-cap upward spike: average with previous for one tick
|
|
355
|
-
v = (v + previous) / 2;
|
|
356
|
-
}
|
|
357
|
-
return v;
|
|
356
|
+
return Math.min(100, Math.max(0, current));
|
|
358
357
|
}
|
|
359
358
|
|
|
360
359
|
// Dominant model-pool among samples (for Max per-bucket calibration).
|
|
@@ -428,7 +427,7 @@ export function recordExactPctSample(store, exactClaude, { now = Date.now(), kee
|
|
|
428
427
|
if (last && last.pct === pct && now - last.at < 30_000) return store;
|
|
429
428
|
arr.push({
|
|
430
429
|
at: now,
|
|
431
|
-
pct:
|
|
430
|
+
pct: clampUsedPercent(pct, last?.pct),
|
|
432
431
|
resetsAtMs: normalizeResetsAtMs(exactClaude.fiveHour.resets_at),
|
|
433
432
|
});
|
|
434
433
|
store.exactPct.claude5h = arr
|
|
@@ -475,13 +474,23 @@ export function pctSpaceEta(history, {
|
|
|
475
474
|
};
|
|
476
475
|
}
|
|
477
476
|
|
|
478
|
-
|
|
477
|
+
// `rollout` is the uuid of the file the line came from (rolloutId()). A
|
|
478
|
+
// snapshot is only comparable with the previous snapshot of the SAME thread:
|
|
479
|
+
// Codex emits one token_count line per response per rate-limit bucket, every
|
|
480
|
+
// line carries the thread's last-known snapshot, and threads (Desktop tabs,
|
|
481
|
+
// sub-agents) interleave on disk — so "the machine's second-newest line" can be
|
|
482
|
+
// another thread's minutes-old reading, or another bucket entirely.
|
|
483
|
+
export function extractCodexUsage(line, { rollout = null } = {}) {
|
|
479
484
|
if (!line || !line.trim()) return null;
|
|
480
485
|
let entry;
|
|
481
486
|
try { entry = JSON.parse(line); } catch { return null; }
|
|
482
487
|
if (entry?.type !== 'event_msg' || entry.payload?.type !== 'token_count') return null;
|
|
483
488
|
const rl = entry.payload.rate_limits;
|
|
484
489
|
if (!rl || typeof rl !== 'object') return null;
|
|
490
|
+
// Only the account bucket feeds the 5h/weekly lines. Codex defaults a missing
|
|
491
|
+
// limit_id to "codex"; `premium`, `codex_other` and friends are separate
|
|
492
|
+
// meters with their own windows and must not be read as this one moving.
|
|
493
|
+
if (rl.limit_id != null && rl.limit_id !== 'codex') return null;
|
|
485
494
|
|
|
486
495
|
function win(w) {
|
|
487
496
|
if (!w || typeof w !== 'object') return null;
|
|
@@ -501,12 +510,29 @@ export function extractCodexUsage(line) {
|
|
|
501
510
|
return {
|
|
502
511
|
agent: 'codex',
|
|
503
512
|
at: ts,
|
|
513
|
+
rollout: rollout || null,
|
|
504
514
|
primary,
|
|
505
515
|
secondary: win(rl.secondary),
|
|
506
516
|
planType: rl.plan_type || null,
|
|
507
517
|
};
|
|
508
518
|
}
|
|
509
519
|
|
|
520
|
+
// The reading to measure `latest` against: the newest earlier sample from the
|
|
521
|
+
// same rollout. Samples recorded before the rollout was carried (an older
|
|
522
|
+
// daemon's store) fall back to the newest earlier sample of any origin.
|
|
523
|
+
// Strictly earlier: the dashboard hands over the daemon's store and a cold
|
|
524
|
+
// scan of the same files, so `latest`'s own copy sits right before it — and a
|
|
525
|
+
// zero-length interval read as "idle" for as long as the daemon ran.
|
|
526
|
+
export function previousCodexSample(sorted, latest) {
|
|
527
|
+
const idx = sorted.lastIndexOf(latest);
|
|
528
|
+
for (let i = (idx === -1 ? sorted.length : idx) - 1; i >= 0; i--) {
|
|
529
|
+
const s = sorted[i];
|
|
530
|
+
if (s.at >= latest.at) continue;
|
|
531
|
+
if (latest.rollout == null || s.rollout == null || s.rollout === latest.rollout) return s;
|
|
532
|
+
}
|
|
533
|
+
return null;
|
|
534
|
+
}
|
|
535
|
+
|
|
510
536
|
// ---------------------------------------------------------------------------
|
|
511
537
|
// Storage
|
|
512
538
|
// ---------------------------------------------------------------------------
|
|
@@ -821,17 +847,19 @@ export function buildUsageReport({
|
|
|
821
847
|
|
|
822
848
|
// --- Codex ---
|
|
823
849
|
{
|
|
824
|
-
const
|
|
825
|
-
const
|
|
826
|
-
|
|
827
|
-
: null;
|
|
850
|
+
const sorted = [...codexSamples].sort((a, b) => a.at - b.at);
|
|
851
|
+
const latest = sorted.at(-1);
|
|
852
|
+
const prev = latest ? previousCodexSample(sorted, latest) : null;
|
|
828
853
|
if (latest?.primary) {
|
|
829
854
|
const windows = [];
|
|
830
855
|
for (const key of ['primary', 'secondary']) {
|
|
831
856
|
const w = latest[key];
|
|
832
857
|
if (!w) continue;
|
|
833
|
-
|
|
834
|
-
|
|
858
|
+
// A window that changed size between the two readings is not the same
|
|
859
|
+
// window; its old reading says nothing about this one's movement.
|
|
860
|
+
const candidate = prev?.[key];
|
|
861
|
+
const prevW = candidate && candidate.windowMinutes === w.windowMinutes ? candidate : null;
|
|
862
|
+
const pct = clampUsedPercent(w.usedPercent, prevW?.usedPercent);
|
|
835
863
|
const label = w.label || labelWindow(w.windowMinutes);
|
|
836
864
|
// %-space burn for ETA on the primary/short window only
|
|
837
865
|
let burn = { idle: true, burnPerMin: 0, activeMin: 0, warmingUp: false };
|
|
@@ -1107,8 +1135,9 @@ export function collectCodexSamples({
|
|
|
1107
1135
|
match,
|
|
1108
1136
|
onCandidate(path) {
|
|
1109
1137
|
onFile?.(path);
|
|
1138
|
+
const rollout = rolloutId(path);
|
|
1110
1139
|
for (const line of tailLines(path)) {
|
|
1111
|
-
const s = extractCodexUsage(line);
|
|
1140
|
+
const s = extractCodexUsage(line, { rollout });
|
|
1112
1141
|
if (s && s.at >= cut) samples.push(s);
|
|
1113
1142
|
}
|
|
1114
1143
|
},
|
package/src/watcher.js
CHANGED
|
@@ -23,8 +23,10 @@ import {
|
|
|
23
23
|
import { getMultiplexer } from './multiplexer.js';
|
|
24
24
|
import { parseTranscriptLine } from './watchers/claude.js';
|
|
25
25
|
import { parseRolloutLines, rolloutMeta } from './watchers/codex.js';
|
|
26
|
-
import { ROLLOUT_RE } from './agents/codex.js';
|
|
27
|
-
import { parseResetTime, resetAtMs } from './time-parser.js';
|
|
26
|
+
import { ROLLOUT_RE, rolloutId } from './agents/codex.js';
|
|
27
|
+
import { parseResetTime, resetAtMs, sourceRank } from './time-parser.js';
|
|
28
|
+
import { getAgent } from './agents/index.js';
|
|
29
|
+
import { modelRemedy } from './patterns.js';
|
|
28
30
|
import { upsertSession, readState, updateState } from './state.js';
|
|
29
31
|
import { getConfig } from './settings.js';
|
|
30
32
|
import { notify } from './notify.js';
|
|
@@ -88,21 +90,25 @@ export function codexSource({ roots }) {
|
|
|
88
90
|
parse(lines, path, { offset } = {}) {
|
|
89
91
|
const hits = parseRolloutLines(lines, { path, offset });
|
|
90
92
|
if (hits.length === 0) return [];
|
|
91
|
-
const last = hits[hits.length - 1]; // the latest snapshot governs
|
|
93
|
+
const last = hits[hits.length - 1]; // the latest snapshot or limit error governs
|
|
92
94
|
const meta = rolloutMeta(path);
|
|
93
95
|
return [{
|
|
94
96
|
agent: 'codex',
|
|
95
97
|
sessionId: meta.sessionId,
|
|
96
98
|
cwd: meta.cwd,
|
|
97
99
|
limitType: last.limitType,
|
|
98
|
-
|
|
100
|
+
// Banner text from a persisted task_complete error; parsed by
|
|
101
|
+
// dispatchCandidate exactly like a scraped pane when no epoch is known.
|
|
102
|
+
resetLine: last.resetLine || null,
|
|
99
103
|
resetAt: last.resetAt,
|
|
104
|
+
reason: last.reachedType || null,
|
|
100
105
|
origin: meta.originator,
|
|
101
106
|
timestampMs: last.timestampMs,
|
|
102
107
|
}];
|
|
103
108
|
},
|
|
104
|
-
usage(lines) {
|
|
105
|
-
|
|
109
|
+
usage(lines, path) {
|
|
110
|
+
const rollout = rolloutId(path);
|
|
111
|
+
return lines.map(line => extractCodexUsage(line, { rollout })).filter(Boolean);
|
|
106
112
|
},
|
|
107
113
|
};
|
|
108
114
|
}
|
|
@@ -184,16 +190,31 @@ export function dispatchCandidate(c) {
|
|
|
184
190
|
: null;
|
|
185
191
|
if (existing && existing.status === 'cancelled') return;
|
|
186
192
|
if (existing && (existing.status === 'stopped' || existing.status === 'resuming')) {
|
|
193
|
+
let kept = false;
|
|
187
194
|
updateState(state => {
|
|
188
195
|
const s = state.sessions[existing.key];
|
|
189
196
|
if (s && (s.status === 'stopped' || s.status === 'resuming')) {
|
|
190
|
-
|
|
191
|
-
s.
|
|
197
|
+
// A weaker estimate never replaces a stronger one that still stands —
|
|
198
|
+
// the rule monitor.js §7 and upsertSession's merge already apply. One
|
|
199
|
+
// Codex stop's token_count line and its task_complete error can land
|
|
200
|
+
// in different ticks, and a "Try again later." banner must not turn
|
|
201
|
+
// the exact epoch the snapshot gave into a 15-minute probe.
|
|
202
|
+
kept = sourceRank(source) < sourceRank(s.resetSource) && s.resetAt > Date.now();
|
|
203
|
+
if (!kept) {
|
|
204
|
+
s.resetAt = at;
|
|
205
|
+
s.resetSource = source;
|
|
206
|
+
if (c.limitType && c.limitType !== 'unknown') s.limitType = c.limitType;
|
|
207
|
+
// The reason belongs to the stop it came with: a later plain stop
|
|
208
|
+
// must not keep reporting an earlier workspace wall.
|
|
209
|
+
if (c.reason) s.limitReason = c.reason;
|
|
210
|
+
else delete s.limitReason;
|
|
211
|
+
}
|
|
192
212
|
if (bannerAt != null) s.bannerAt = bannerAt;
|
|
193
|
-
if (c.limitType && c.limitType !== 'unknown') s.limitType = c.limitType;
|
|
194
213
|
}
|
|
195
214
|
});
|
|
196
|
-
log(
|
|
215
|
+
log(kept
|
|
216
|
+
? `kept the stronger reset for tracked stop: session=${c.sessionId} (${source} would have replaced it)`
|
|
217
|
+
: `refreshed reset for tracked stop: session=${c.sessionId} resetAt=${new Date(at).toISOString()}`);
|
|
197
218
|
return;
|
|
198
219
|
}
|
|
199
220
|
|
|
@@ -217,6 +238,10 @@ export function dispatchCandidate(c) {
|
|
|
217
238
|
lastError: null,
|
|
218
239
|
};
|
|
219
240
|
if (c.env) record.env = c.env; // e.g. CLAUDE_CONFIG_DIR for sandboxed desktop sessions
|
|
241
|
+
// e.g. Codex's rate_limit_reached_type. Cleared explicitly when absent:
|
|
242
|
+
// upsertSession merges into an older record of the same session.
|
|
243
|
+
if (c.reason) record.limitReason = c.reason;
|
|
244
|
+
else if (existing?.limitReason) record.limitReason = null;
|
|
220
245
|
// Raw reset without margin for calibration window math.
|
|
221
246
|
const rawResetMs = c.resetAt != null
|
|
222
247
|
? c.resetAt
|
|
@@ -235,8 +260,16 @@ export function dispatchCandidate(c) {
|
|
|
235
260
|
upsertSession(record, {
|
|
236
261
|
after: calSample ? (state) => applyCalibrationToState(state, calSample) : null,
|
|
237
262
|
});
|
|
238
|
-
log(`limit stop via transcript: agent=${c.agent} session=${c.sessionId || '?'} origin=${c.origin || '?'} resetAt=${new Date(at).toISOString()} (${source})`);
|
|
239
|
-
|
|
263
|
+
log(`limit stop via transcript: agent=${c.agent} session=${c.sessionId || '?'} origin=${c.origin || '?'} resetAt=${new Date(at).toISOString()} (${source})${c.reason ? ` reason=${c.reason}` : ''}`);
|
|
264
|
+
// A model limit (a Codex workspace wall, say) is not lifted by waiting, so
|
|
265
|
+
// "resumes when the limit resets" would be a promise nothing keeps — say
|
|
266
|
+
// what it needs instead, as the pane monitor does.
|
|
267
|
+
if (record.limitType === 'model') {
|
|
268
|
+
const agent = getAgent(c.agent);
|
|
269
|
+
notify(`${agent.name} hit a model limit — needs you`, `${c.cwd || c.agent}: ${modelRemedy(agent)}`);
|
|
270
|
+
} else {
|
|
271
|
+
notify('limit hit 😴', `${c.cwd || c.agent}: tracked — resumes when the limit resets`);
|
|
272
|
+
}
|
|
240
273
|
}
|
|
241
274
|
|
|
242
275
|
function loadOffsets(path) {
|