quoxcli 0.0.0-stage → 0.2.0

package/lib/core/sessionHeartbeat.js

@@ -0,0 +1,192 @@
+/**
+ * QUOXON J1: the `quox claude` wrapper's session heartbeat. The wrapper starts
+ * ONE detached `quox sessions heartbeat <tmux-session> --loop` per tmux session
+ * it launches; that child beats POST /api/v1/sessions/heartbeat every interval
+ * while the tmux session exists, then calls /api/v1/sessions/end and exits.
+ * The server expires a row after 3 missed beats, so a killed child or a
+ * powered-off box drops out on its own.
+ *
+ * Loop hygiene (house rule: failure accounting, backoff, retirement):
+ *  - a failed beat is counted and logged, never treated as success;
+ *  - consecutive failures back off exponentially, capped at maxBackoffS;
+ *  - the loop retires after maxFailures consecutive failures, when the tmux
+ *    session ends, or when it never appears within the start grace;
+ *  - a pidfile lock keeps one heartbeat per session across reattaches, and is
+ *    released on every exit path.
+ * Pure parts take injected tmux output / held slots / attach lookup.
+ */
+import { spawn as realSpawn, spawnSync as realSpawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, statSync, appendFileSync, truncateSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { slotsBySession, readAttachSafe, readHeldSlotsSafe } from './liveSessionsReport.js';
+
+export const PROBE_FORMAT = '#{session_name}\t#{session_created}\t#{session_attached}\t#{session_activity}\t#{pane_pid}\t#{pane_current_command}';
+export const DEFAULT_INTERVAL_S = 60;
+const HB_DIR = () => join(homedir(), '.quox', 'heartbeat');
+const LOG_CAP_BYTES = 256 * 1024;
+
+const isoSec = (v) => {
+  const n = Number(v);
+  return Number.isFinite(n) && n > 0 ? new Date(n * 1000).toISOString() : null;
+};
+
+/** One line of PROBE_FORMAT output -> probe record, or null. */
+export function parseProbe(out) {
+  const first = String(out || '').split('\n').find((l) => l.trim());
+  if (!first) return null;
+  const c = first.split('\t');
+  if (c.length < 6 || !c[0]) return null;
+  const pid = Number.parseInt(c[4], 10);
+  return {
+    name: c[0],
+    started_at: isoSec(c[1]),
+    attached: Number(c[2]) > 0,
+    last_active: isoSec(c[3]),
+    pid: Number.isInteger(pid) ? pid : null,
+    command: c[5] || null,
+  };
+}
+
+/** The wrapped tmux session's bare name, as the held-slots ledger records it. */
+const bareOf = (name) => String(name).replace(/^quox-/, '');
+
+export function buildBeat({ probe, backend = null, agent = null, slot = null, held = {}, attachLookup = readAttachSafe, intervalS = DEFAULT_INTERVAL_S }) {
+  const by = slotsBySession(held);
+  const bare = bareOf(probe.name);
+  const hit = by[probe.name] || by[bare] || by[bare.replace(/^(codex-|gemini-|tui-)/, '')] || null;
+  // The pane runs claude directly, so its pid IS the attach-kit pid; the slot entry's pid is the fallback.
+  const att = (probe.pid != null && attachLookup(probe.pid)) || (hit && hit.pid != null && attachLookup(hit.pid)) || null;
+  return {
+    name: probe.name,
+    program: backend || probe.command || null,
+    pid: probe.pid,
+    started_at: probe.started_at,
+    last_active: probe.last_active,
+    attached: probe.attached,
+    slot: (hit && hit.slot) || slot || null,
+    agent: (att && att.agent) || agent || null,
+    interval_s: intervalS,
+  };
+}
+
+/** Live probe of one tmux session (exact match). null when it does not exist. */
+export function probeTmux(name, spawnSync = realSpawnSync) {
+  const r = spawnSync('tmux', ['display-message', '-p', '-t', `=${name}`, PROBE_FORMAT], { encoding: 'utf8' });
+  if (!r || r.error || r.status !== 0) return null;
+  const p = parseProbe(r.stdout);
+  return p && p.name === name ? p : null;
+}
+
+export async function runHeartbeatLoop({
+  tmuxName, backend = null, agent = null, slot = null, intervalS = DEFAULT_INTERVAL_S,
+  probe = (n) => probeTmux(n), post, end, held = () => readHeldSlotsSafe(), attachLookup = readAttachSafe,
+  sleep = (ms) => new Promise((r) => setTimeout(r, ms)), log = () => {},
+  maxFailures = 30, maxBackoffS = 600, graceMs = 30_000, gracePollMs = 2000, maxTicks = Infinity,
+  lock = { acquire: () => true, release: () => {} },
+}) {
+  if (!lock.acquire()) {
+    log(`another heartbeat already holds ${tmuxName}; exiting`);
+    return { exit: 'duplicate', beats: 0, failures: 0 };
+  }
+  let beats = 0;
+  let failures = 0;
+  let consecutive = 0;
+  let ticks = 0;
+  let seen = false;
+  let waited = 0;
+  try {
+    while (ticks < maxTicks) {
+      const p = probe(tmuxName);
+      if (!p) {
+        if (!seen && waited < graceMs) { waited += gracePollMs; await sleep(gracePollMs); continue; }
+        if (!seen) { log(`${tmuxName} never appeared; exiting`); return { exit: 'never_started', beats, failures }; }
+        try { await end(); } catch (e) { log(`end failed: ${e && e.status ? `HTTP ${e.status} ` : ''}${e && e.message}`); }
+        log(`${tmuxName} ended after ${beats} beats (${failures} failed)`);
+        return { exit: 'ended', beats, failures };
+      }
+      seen = true;
+      ticks += 1;
+      let beat;
+      try { beat = buildBeat({ probe: p, backend, agent, slot, held: held(), attachLookup, intervalS }); } catch (e) { beat = null; log(`beat build failed: ${e.message}`); }
+      try {
+        if (!beat) throw new Error('no beat');
+        await post(beat);
+        beats += 1;
+        consecutive = 0;
+      } catch (e) {
+        failures += 1;
+        consecutive += 1;
+        log(`beat failed (${consecutive} in a row, ${failures} total): ${e && e.status ? `HTTP ${e.status} ` : ''}${e && e.message}`);
+        if (consecutive >= maxFailures) {
+          log(`retiring after ${consecutive} consecutive failures`);
+          return { exit: 'retired', beats, failures };
+        }
+      }
+      const delayS = consecutive ? Math.min(intervalS * 2 ** Math.min(consecutive, 10), maxBackoffS) : intervalS;
+      await sleep(delayS * 1000);
+    }
+    return { exit: 'max_ticks', beats, failures };
+  } finally {
+    lock.release();
+  }
+}
+
+const safeFile = (name) => String(name).replace(/[^A-Za-z0-9._-]/g, '_');
+
+/** Pidfile lock under ~/.quox/heartbeat/<session>.pid; a dead holder's file is taken over. */
+export function pidLock(name, { dir = HB_DIR(), pid = process.pid, alive = isAlive } = {}) {
+  const file = join(dir, `${safeFile(name)}.pid`);
+  return {
+    acquire() {
+      try {
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        if (existsSync(file)) {
+          const other = Number.parseInt(readFileSync(file, 'utf8'), 10);
+          if (Number.isInteger(other) && other !== pid && alive(other)) return false;
+        }
+        writeFileSync(file, String(pid), { mode: 0o600 });
+        return true;
+      } catch { return false; }
+    },
+    release() {
+      try {
+        if (existsSync(file) && Number.parseInt(readFileSync(file, 'utf8'), 10) === pid) unlinkSync(file);
+      } catch { /* best effort */ }
+    },
+  };
+}
+
+function isAlive(pid) {
+  try { process.kill(pid, 0); return true; } catch (e) { return e && e.code === 'EPERM'; }
+}
+
+/** Append-only per-session log, truncated when it passes LOG_CAP_BYTES. */
+export function fileLogger(name, dir = HB_DIR()) {
+  const file = join(dir, `${safeFile(name)}.log`);
+  return (msg) => {
+    try {
+      mkdirSync(dir, { recursive: true, mode: 0o700 });
+      if (existsSync(file) && statSync(file).size > LOG_CAP_BYTES) truncateSync(file, 0);
+      appendFileSync(file, `${new Date().toISOString()} ${msg}\n`, { mode: 0o600 });
+    } catch { /* logging must never kill the loop */ }
+  };
+}
+
+/**
+ * Start the detached heartbeat child for a tmux session the wrapper launched.
+ * The child is this same quox binary; it survives the wrapper exiting on detach.
+ */
+export function startHeartbeat({ tmuxSession, backend, agent, slot }, { spawn = realSpawn, argv = process.argv, execPath = process.execPath } = {}) {
+  if (!argv[1]) return false;
+  const args = [argv[1], 'sessions', 'heartbeat', tmuxSession, '--loop'];
+  if (backend) args.push('--backend', backend);
+  if (agent) args.push('--agent', agent);
+  if (slot) args.push('--slot', slot);
+  try {
+    const child = spawn(execPath, args, { detached: true, stdio: 'ignore' });
+    if (child && child.on) child.on('error', () => {});
+    if (child && child.unref) child.unref();
+    return true;
+  } catch { return false; }
+}

package/lib/core/sessionsOverview.js

@@ -0,0 +1,160 @@
+/**
+ * QUOXON J2: `quox sessions` overview of GET /api/v1/sessions, grouped by
+ * agent, newest activity first. overviewLines() is plain text + a tone per
+ * line, shared by the CLI renderer (ANSI colour) and the quox conversation
+ * client's `/sessions all` (which colours by tone itself).
+ */
+import { hostname as osHostname } from 'node:os';
+import { line } from './quoxglyph.js';
+import { formatAge } from './claudeLs.js';
+import { loadNames } from '../quoxon/session.js';
+
+export const GLYPH = { working: '◐', waiting: '⏸', resting: '○', failing: '✖' };
+const TONE = { working: 'ok', waiting: 'warn', failing: 'warn', resting: 'dim' };
+const ANSI = { ok: '\x1b[32m', warn: '\x1b[33m', fail: '\x1b[31m', dim: '\x1b[2m', accent: '\x1b[36m', reset: '\x1b[0m' };
+const stripAnsi = (s) => String(s).replace(/\x1b\[[0-9;]*m/g, '');
+
+const clip = (v, n) => {
+  const a = [...String(v ?? '')];
+  return a.length > n ? a.slice(0, n - 1).join('') + '…' : String(v ?? '');
+};
+const pad = (v, n) => clip(v, n).padEnd(n);
+
+function kindLabel(r) {
+  if (r.kind === 'tmux') return r.backend ? `tmux·${r.backend}` : 'tmux';
+  return r.kind;
+}
+
+function hostLabel(r, localHost) {
+  if (!r.host) return r.kind === 'tmux' ? '?' : 'server';
+  return localHost && r.host === localHost ? 'this host' : r.host;
+}
+
+export const RECENT_CONVERSATIONS = 5;
+const KEEP_KINDS = new Set(['tmux', 'slot', 'resident', 'objective', 'loop', 'workflow']);
+
+/** One-line title, clipped on a word boundary when one is near the end, never mid-surrogate. */
+export function clipTitle(raw, max = 34) {
+  const s = String(raw ?? '').replace(/[\x00-\x1f\x7f]/g, ' ').replace(/\s+/g, ' ').trim();
+  const a = [...s];
+  if (a.length <= max) return s;
+  const cut = a.slice(0, max - 1).join('');
+  const sp = cut.lastIndexOf(' ');
+  return (sp >= Math.floor(max / 2) ? cut.slice(0, sp) : cut).replace(/[\s,.;:-]+$/, '') + '…';
+}
+
+const timeOf = (r) => { const t = Date.parse(r.last_active); return Number.isFinite(t) ? t : -Infinity; };
+
+/**
+ * The default view shows what is useful or eventful: every tmux session, slot,
+ * resident and running piece of work, every working/waiting conversation, and
+ * only the RECENT_CONVERSATIONS most recent other conversations. `all` keeps
+ * everything. Conversation names use the local /rename name when known, else
+ * the server title. Regrouped by each row's own agent; "no agent" goes last.
+ * -> { groups, hiddenConversations, shown }
+ */
+export function prepareIndex(index, { all = false, names = {}, keep = RECENT_CONVERSATIONS } = {}) {
+  const rows = (index.groups || []).flatMap((g) => g.rows).map((r) => (r.kind === 'conversation'
+    // The hint is this client's own command, so set it here too (correct against an older server's --print hint).
+    ? { ...r, name: clipTitle((names && names[r.id] && names[r.id].name) || r.name || '(untitled)'), attach_hint: `quox --conversation ${r.id}` }
+    : r));
+  let hidden = 0;
+  let kept = rows;
+  if (!all) {
+    const quiet = rows.filter((r) => r.kind === 'conversation' && r.state !== 'working' && r.state !== 'waiting').sort((a, b) => timeOf(b) - timeOf(a));
+    const keepIds = new Set(quiet.slice(0, keep));
+    hidden = quiet.length - keepIds.size;
+    kept = rows.filter((r) => KEEP_KINDS.has(r.kind) || r.kind !== 'conversation' || r.state === 'working' || r.state === 'waiting' || keepIds.has(r));
+  }
+  const by = new Map();
+  for (const r of [...kept].sort((a, b) => timeOf(b) - timeOf(a))) {
+    const k = r.agent || null;
+    if (!by.has(k)) by.set(k, []);
+    by.get(k).push(r);
+  }
+  const groups = [...by.entries()].map(([agent, rs]) => ({ agent, rows: rs }))
+    .sort((a, b) => (a.agent === null) - (b.agent === null) || timeOf(b.rows[0]) - timeOf(a.rows[0]));
+  return { groups, hiddenConversations: hidden, shown: kept.length };
+}
+
+/** -> [{ text, tone, state? }] */
+export function overviewLines(index, { now = Date.now(), localHost = null, all = false, names = {} } = {}) {
+  const prepared = prepareIndex(index, { all, names });
+  const counts = index.counts || {};
+  const total = counts.total || 0;
+  const out = [
+    { text: stripAnsi(line('quox', 'sessions', { result: `${total} live` })), tone: 'accent' },
+    { text: `${total} live · ${counts.working || 0} working · ${counts.waiting || 0} waiting · ${counts.resting || 0} resting · ${counts.failing || 0} failing`, tone: 'dim' },
+  ];
+  if (!total) out.push({ text: 'No live sessions in this org.', tone: 'dim' });
+  for (const g of prepared.groups) {
+    out.push({ text: `${g.agent || 'no agent'} (${g.rows.length})`, tone: 'accent' });
+    for (const r of g.rows) {
+      const t = Date.parse(r.last_active);
+      const age = Number.isFinite(t) ? formatAge((now - t) / 1000) : '-';
+      const extra = [r.slot ? `slot ${r.slot}` : null, r.state_reason || null].filter(Boolean).join(', ');
+      const text = `  ${GLYPH[r.state] || '?'} ${pad(r.state, 8)} ${pad(r.name, 34)} ${pad(kindLabel(r), 13)} ${pad(hostLabel(r, localHost), 12)} ${pad(age, 5)} ${r.attach_hint || ''}${extra ? `  (${extra})` : ''}`;
+      out.push({ text, tone: TONE[r.state] || 'dim', state: r.state });
+    }
+  }
+  if (prepared.hiddenConversations > 0) {
+    out.push({ text: `  +${prepared.hiddenConversations} more conversations (last 24 h) · quox sessions --all`, tone: 'dim' });
+  }
+  const bad = (index.sources || []).filter((s) => s.state !== 'connected');
+  if (bad.length) {
+    out.push({ text: `sources: ${bad.map((s) => `${s.id}: ${s.state}${s.note ? ` (${s.note})` : ''}`).join(' · ')}`, tone: 'dim' });
+  }
+  return out;
+}
+
+export function renderOverview(index, { now = Date.now(), localHost = osHostname(), color = false, all = false, names = {} } = {}) {
+  return overviewLines(index, { now, localHost, all, names }).map((l) => {
+    if (!color) return l.text;
+    const c = l.state === 'failing' ? ANSI.fail : ANSI[l.tone] || '';
+    return `${c}${l.text}${ANSI.reset}`;
+  }).join('\n');
+}
+
+const errText = (e) => `${e && e.status ? `HTTP ${e.status}: ` : ''}${(e && e.message) || 'request failed'}`;
+
+/**
+ * Bare `quox sessions`. --json prints the raw index; --watch redraws every 2 s
+ * until interrupted. An error is printed with its status (never shown as an
+ * empty list) and exits 1; in --watch it is shown and the next refresh retries.
+ */
+export async function runSessionsOverview({ json = false, watch = false, all = false } = {}, over = {}) {
+  const d = {
+    write: (s) => process.stdout.write(s),
+    sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+    now: () => Date.now(),
+    localHost: osHostname(),
+    color: !!process.stdout.isTTY && !process.env.NO_COLOR,
+    iterations: Infinity,
+    names: undefined,
+    ...over,
+  };
+  const names = () => d.names ?? loadNames();
+  if (!d.fetchIndex) throw new Error('fetchIndex required');
+  if (!watch) {
+    try {
+      const idx = await d.fetchIndex();
+      d.write(json ? JSON.stringify(idx, null, 2) + '\n' : renderOverview(idx, { now: d.now(), localHost: d.localHost, color: d.color, all, names: names() }) + '\n');
+      return 0;
+    } catch (e) {
+      d.write(`quox sessions: ${errText(e)}\n`);
+      return 1;
+    }
+  }
+  for (let i = 0; i < d.iterations; i++) {
+    let body;
+    try {
+      const idx = await d.fetchIndex();
+      body = json ? JSON.stringify(idx) : renderOverview(idx, { now: d.now(), localHost: d.localHost, color: d.color, all, names: names() });
+    } catch (e) {
+      body = `quox sessions: ${errText(e)} (retrying)`;
+    }
+    d.write(`\x1b[2J\x1b[H${body}\n\n(refreshing every 2 s, Ctrl-C to stop)\n`);
+    await d.sleep(2000);
+  }
+  return 0;
+}

package/lib/core/slotKeeper.js

@@ -0,0 +1,282 @@
+/**
+ * Slot hold keeper (SLOTS-PERFECT, quox#662/B).
+ *
+ * Why: the collector frees a held slot 15 minutes after its last heartbeat,
+ * and the only thing heartbeating was a `quox slot wait` the session had to
+ * remember to re-arm. A session that forgot lost its hold silently. The
+ * keeper is a detached background loop, spawned by `slot checkout`, that
+ * heartbeats on a timer and dies with the Claude session it serves.
+ *
+ * Loop hygiene (house rule): failures are counted distinctly from successes,
+ * back off exponentially (30s doubling, cap 5 min) and the keeper retires
+ * after a bounded run of consecutive failures (24; 3 for 401). Everything
+ * environmental (clock, sleep, probe, signals, fs dir) is injected.
+ */
+
+import {
+  openSync, closeSync, writeSync, readFileSync, unlinkSync, appendFileSync,
+  statSync, writeFileSync, mkdirSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { execFile as execFileCb } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { promisify } from 'node:util';
+import { QuoxApiError } from './errors.js';
+
+export const KEEP_INTERVAL_MS = 5 * 60 * 1000;
+export const BACKOFF_BASE_MS = 30 * 1000;
+export const BACKOFF_CAP_MS = 5 * 60 * 1000;
+// 24 consecutive failures at the 5-min backoff cap rides out ~2h of
+// collector/auth outage; each failure is one cheap HTTP call, so outlasting a
+// long outage beats letting the hold lapse (the collector revives a stale
+// slot for its own holder on the next successful heartbeat).
+export const MAX_FAILS = 24;
+export const MAX_AUTH_FAILS = 3;
+export const LOG_MAX_BYTES = 256 * 1024;
+// QUOX-CLAUDE-3: the control queue is polled far more often than the 5-min
+// heartbeat so `!compact` feels immediate. Poll failures back off (doubling,
+// capped at CONTROL_BACKOFF_CAP_TICKS skipped ticks) and are counted
+// distinctly from heartbeat failures; they never retire the keeper (the
+// heartbeat owns hold-loss).
+export const CONTROL_POLL_MS = 20 * 1000;
+export const CONTROL_BACKOFF_CAP_TICKS = 15;
+export const BUSY_WINDOW_MS = 60 * 1000;
+export const NO_TMUX_RESULT = 'session not under tmux; start it with quox claude';
+
+export function keeperDir() {
+  return join(homedir(), '.quox', 'keepers');
+}
+
+/** Real process probe: liveness via kill -0, start ticks via /proc. */
+export function makeProcessProbe(procStartTime) {
+  return {
+    alive(pid) {
+      try { process.kill(pid, 0); return true; } catch (e) { return e && e.code === 'EPERM'; }
+    },
+    starttime: procStartTime,
+  };
+}
+
+/** Is `pid` the same process it was (alive, and start ticks unchanged)? */
+export function sameProcess(probe, pid, starttime) {
+  if (!pid || !probe.alive(pid)) return false;
+  return starttime == null || String(probe.starttime(pid)) === String(starttime);
+}
+
+function isHoldLost(err) {
+  if (!(err instanceof QuoxApiError)) return false;
+  // 409: held by someone else or freed. 404: the slot itself is gone.
+  if (err.status === 409 || err.status === 404) return true;
+  const said = [err.message, err.code, err.body?.error].filter(Boolean).join(' ');
+  return err.status === 403 && /not (the current |slot )?holder/i.test(said);
+}
+
+const execFileP = promisify(execFileCb);
+function defaultExecFile(file, args) {
+  return execFileP(file, args, { timeout: 10000 });
+}
+
+export async function runKeeper(opts) {
+  const {
+    name, session, claude, heartbeat, release, forget, probe,
+    sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
+    rand = Math.random,
+    dir = keeperDir(),
+    // Remote control (all optional; absent pollControl = heartbeat-only keeper).
+    pollControl, ackControl, tmuxPane = null,
+    execFile = defaultExecFile,
+    now = () => Date.now(),
+    installSignals = (cleanup) => {
+      const h = () => { cleanup(); process.exit(0); };
+      process.once('SIGTERM', h);
+      process.once('SIGINT', h);
+    },
+  } = opts;
+
+  mkdirSync(dir, { recursive: true });
+  const lockPath = join(dir, `${name}.pid`);
+  const logPath = join(dir, `${name}.log`);
+  const log = (msg) => {
+    try {
+      let size = 0;
+      try { size = statSync(logPath).size; } catch { /* no log yet */ }
+      if (size > LOG_MAX_BYTES) writeFileSync(logPath, '');
+      appendFileSync(logPath, `${new Date().toISOString()} [${name}] ${msg}\n`);
+    } catch { /* logging must never kill the keeper */ }
+  };
+
+  // Single instance per slot: O_EXCL lockfile, taken over when its pid is dead.
+  let locked = false;
+  for (let attempt = 0; attempt < 2 && !locked; attempt += 1) {
+    try {
+      const fd = openSync(lockPath, 'wx');
+      writeSync(fd, String(process.pid));
+      closeSync(fd);
+      locked = true;
+    } catch (e) {
+      if (e.code !== 'EEXIST') throw e;
+      let other = NaN;
+      try { other = parseInt(readFileSync(lockPath, 'utf-8'), 10); } catch { /* raced away */ }
+      if (Number.isFinite(other) && other !== process.pid && probe.alive(other)) {
+        log(`another keeper (pid ${other}) already running; exiting`);
+        return { code: 0, reason: 'already-running' };
+      }
+      try { unlinkSync(lockPath); } catch { /* raced away */ }
+    }
+  }
+  if (!locked) {
+    log('could not acquire keeper lock; exiting');
+    return { code: 1, reason: 'lock-failed' };
+  }
+
+  const cleanup = () => {
+    try {
+      if (parseInt(readFileSync(lockPath, 'utf-8'), 10) === process.pid) unlinkSync(lockPath);
+    } catch { /* already gone */ }
+  };
+  installSignals(cleanup);
+
+  try {
+    log(`keeper started session=${session} claude=${claude?.pid}:${claude?.starttime}`);
+    let fails = 0;
+    let delay = KEEP_INTERVAL_MS * (1 + rand() * 0.2);
+
+    // Control queue state: ids already executed (never re-run if an ack failed,
+    // only re-acked), and distinct failure accounting with tick-skip backoff.
+    const handled = new Map();
+    const startedAt = now();
+    let lastHeartbeatAt = null;
+    // Busy/idle: only a hash of the pane is kept (pane text can hold secrets
+    // and the status result is posted to Discord). Armed by the first status
+    // request, then refreshed on every control tick.
+    let armed = false;
+    let paneHash = null;
+    let paneChangedAt = null;
+    const samplePane = async () => {
+      if (!tmuxPane) return false;
+      try {
+        const out = await execFile('tmux', ['capture-pane', '-p', '-t', tmuxPane]);
+        const text = typeof out === 'string' ? out : String(out?.stdout ?? '');
+        const h = createHash('sha256').update(text).digest('hex');
+        if (paneHash !== null && h !== paneHash) paneChangedAt = now();
+        paneHash = h;
+        return true;
+      } catch { return false; }
+    };
+    let ctlFails = 0;
+    let ctlSkip = 0;
+    const controlTick = async () => {
+      if (ctlSkip > 0) { ctlSkip -= 1; return; }
+      if (claude && !sameProcess(probe, claude.pid, claude.starttime)) return;
+      try {
+        if (armed) await samplePane();
+        const rows = (await pollControl()) || [];
+        for (const row of rows) {
+          let outcome = handled.get(row.id);
+          if (!outcome) {
+            outcome = await runControlCommand(row);
+            handled.set(row.id, outcome);
+            log(`control ${row.command} ${row.id} from ${row.requested_via || '?'}: ${outcome.status} (${outcome.result})`);
+          }
+          await ackControl(row.id, outcome.status, outcome.result);
+          handled.delete(row.id);
+        }
+        if (ctlFails > 0) log(`control poll recovered after ${ctlFails} failure(s)`);
+        ctlFails = 0;
+      } catch (err) {
+        ctlFails += 1;
+        ctlSkip = Math.min(2 ** (ctlFails - 1) - 1, CONTROL_BACKOFF_CAP_TICKS);
+        log(`control poll FAILED (${ctlFails} consecutive, skipping ${ctlSkip} tick(s)): ${err.status || ''} ${err.message}`);
+      }
+    };
+    const fmtDur = (sec) => {
+      const s = Math.max(0, Math.floor(sec));
+      const d = Math.floor(s / 86400); const h = Math.floor((s % 86400) / 3600); const m = Math.floor((s % 3600) / 60);
+      return d ? `${d}d${String(h).padStart(2, '0')}h` : `${h}h${String(m).padStart(2, '0')}m`;
+    };
+    const statusResult = async () => {
+      const parts = [`session=${session}`, `slot=${name}`];
+      let claudeUp = null;
+      if (claude) {
+        const alive = sameProcess(probe, claude.pid, claude.starttime);
+        if (alive) {
+          try {
+            const out = await execFile('ps', ['-o', 'etimes=', '-p', String(claude.pid)]);
+            const n = parseInt(String(typeof out === 'string' ? out : out?.stdout).trim(), 10);
+            if (Number.isFinite(n)) claudeUp = n;
+          } catch { /* uptime unknown */ }
+        }
+        parts.push(`claude pid ${claude.pid} ${alive ? 'alive' : 'gone'}${alive && claudeUp != null ? ` up ${fmtDur(claudeUp)}` : ''}`);
+      }
+      if (!tmuxPane) {
+        parts.push('not under tmux');
+      } else {
+        parts.push(`pane=${tmuxPane}`);
+        const wasArmed = armed;
+        armed = true;
+        const ok = await samplePane();
+        if (ok) {
+          if (!wasArmed) parts.push('activity unknown (first check)');
+          else parts.push(paneChangedAt !== null && now() - paneChangedAt <= BUSY_WINDOW_MS ? 'busy' : 'idle');
+        }
+      }
+      parts.push(lastHeartbeatAt === null ? 'last heartbeat none yet' : `last heartbeat ${new Date(lastHeartbeatAt).toISOString()}`);
+      parts.push(`keeper up ${fmtDur((now() - startedAt) / 1000)}`);
+      return parts.join(', ').slice(0, 500);
+    };
+    const runControlCommand = async (row) => {
+      if (row.command === 'status') return { status: 'done', result: await statusResult() };
+      if (row.command !== 'compact') return { status: 'failed', result: `unsupported command ${row.command}` };
+      if (!tmuxPane) return { status: 'failed', result: NO_TMUX_RESULT };
+      try {
+        await execFile('tmux', ['send-keys', '-t', tmuxPane, '/compact', 'Enter']);
+        return { status: 'done', result: `sent /compact to ${tmuxPane}` };
+      } catch (err) {
+        return { status: 'failed', result: `tmux send-keys failed: ${err.message}` };
+      }
+    };
+
+    for (;;) {
+      let remaining = delay;
+      while (pollControl && remaining > CONTROL_POLL_MS) {
+        await sleep(CONTROL_POLL_MS);
+        remaining -= CONTROL_POLL_MS;
+        await controlTick();
+      }
+      await sleep(remaining);
+      if (pollControl) await controlTick();
+
+      if (claude && !sameProcess(probe, claude.pid, claude.starttime)) {
+        log(`claude session gone (pid ${claude.pid}); releasing ${name} and exiting`);
+        try { await release(); } catch (e) { log(`release failed: ${e.message}`); }
+        return { code: 0, reason: 'session-gone' };
+      }
+
+      try {
+        await heartbeat();
+        lastHeartbeatAt = now();
+        if (fails > 0) log(`heartbeat recovered after ${fails} failure(s)`);
+        fails = 0;
+        log('heartbeat ok');
+        delay = KEEP_INTERVAL_MS * (1 + rand() * 0.2);
+      } catch (err) {
+        if (isHoldLost(err)) {
+          log(`HOLD LOST: ${name} is no longer held by ${session}; keeper exiting`);
+          try { forget(); } catch { /* best effort */ }
+          return { code: 0, reason: 'hold-lost' };
+        }
+        fails += 1;
+        const limit = err instanceof QuoxApiError && err.status === 401 ? MAX_AUTH_FAILS : MAX_FAILS;
+        log(`heartbeat FAILED (${fails}/${limit}): ${err.status || ''} ${err.message}`);
+        if (fails >= limit) {
+          log(`RETIRED after ${fails} consecutive failures; hold will lapse`);
+          return { code: 1, reason: 'retired' };
+        }
+        delay = Math.min(BACKOFF_BASE_MS * 2 ** (fails - 1), BACKOFF_CAP_MS);
+      }
+    }
+  } finally {
+    cleanup();
+  }
+}