viber-channel 0.8.19 → 0.8.21

package/lib/runner_roster.ts

@@ -0,0 +1,521 @@
+/**
+ * runner_roster.ts — the machine's presence, as the runner reports it (#627 step-03).
+ *
+ * Each runner heartbeat carries a ROSTER built from the agents' presence records
+ * (`presence_record.ts`) and ONE enumeration of the machine's processes:
+ *
+ *   present  the record is active and its pid still has the SAME start time
+ *   closed   the record says the agent ended on a DECIDED cause (see `endVerdict`)
+ *   down     the agent is dead without a decided end: crash, reboot, window closed…
+ *
+ * Nothing is reported when the runner cannot tell — never a guess:
+ *   - the enumeration failed its positive control (it must contain the runner itself);
+ *   - the record is unreadable, or its start time is not known yet;
+ *   - the directory could not be read (then NO closed/down at all this beat).
+ *
+ * ⚠ The END POLICY lives here and nowhere else (plan 627, step-01 review): an agent
+ * records a fact, the runner decides. JP's rule: grey rather than lose.
+ *
+ * A closed/down record is resent on every beat until the server acknowledges it
+ * (`presence_applied: true` in a 2xx): only then is the file removed. So a failed
+ * beat, or a server that has not applied it, loses nothing.
+ */
+import { mkdirSync, readdirSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import {
+  type PresenceEndReason,
+  type PresenceRecord,
+  presenceDir,
+  RECORD_FILE_RE,
+  readPresenceRecord,
+} from "./presence_record.js";
+import { enumerateStartTimes } from "./process_start.js";
+import { runnerIdentity } from "./runner_exec.js";
+import * as reg from "./runner_registry.js";
+import type { SeenAgent } from "./runner_registry.js";
+
+/** Most instances one beat may carry; the server refuses more (step-04). */
+export const ROSTER_MAX = 200;
+
+/**
+ * The largest heartbeat body the server accepts (step-04 sets it on the route).
+ * Derived, not guessed: 200 entries at their maximum size must fit — pinned by a
+ * test that builds exactly that body. The historical limit was 8 KiB, which about
+ * 90 agents would overflow, taking the RUNNER's own beat down with them (Opus).
+ */
+export const HEARTBEAT_MAX_BODY_BYTES = 32 * 1024;
+
+export type EndVerdict = "closed" | "down";
+
+/**
+ * THE end policy. Closed (removed from the web) only for a cause somebody decided;
+ * every ambiguous end — a pipe closed, a signal, a failure — is down (greyed,
+ * relaunchable). A Windows reboot closes claude's stdin exactly like `/exit` does:
+ * that is why `stdin-end` is down.
+ */
+export function endVerdict(reason: PresenceEndReason): EndVerdict {
+  switch (reason) {
+    case "revoked":
+    case "lease-lost":
+    case "once-completed":
+      return "closed";
+    case "stdin-end":
+    case "signal":
+    case "shutdown":
+      return "down";
+  }
+}
+
+export interface RosterPresent {
+  instance_id: string;
+  conversation_id: string | null;
+  /** The process start time the runner verified. Two present instances sharing a
+   *  label (a relaunch overlapping its old process): Python lets the LATEST win. */
+  started_at: number;
+}
+
+export interface Roster {
+  present: RosterPresent[];
+  closed: string[];
+  down: string[];
+}
+
+export interface CollectedRoster {
+  roster: Roster;
+  /** The live records, for the registry join (`syncFromRecords`). */
+  seen: SeenAgent[];
+  /** Record file of each closed/down instance, with the EXACT content that was read:
+   *  removed once acknowledged, and only if the file still holds that content. */
+  endedFiles: Map<string, { path: string; content: string }>;
+  /** Why this beat reported nothing it could not prove. */
+  notes: string[];
+}
+
+/** The process table: pid → start time, or `undefined` when the instrument did not see. */
+export type ProcessTable = Map<number, number | undefined>;
+
+export interface CollectDeps {
+  identityBaseUrl: string;
+  lockDir?: string;
+  /** ONE enumeration of the machine (`enumerateStartTimes`); undefined = not seen. */
+  enumerate: () => Promise<ProcessTable | undefined>;
+}
+
+function emptyCollected(note?: string): CollectedRoster {
+  return {
+    roster: { present: [], closed: [], down: [] },
+    seen: [],
+    endedFiles: new Map(),
+    notes: note === undefined ? [] : [note],
+  };
+}
+
+type Classified =
+  | { kind: "present"; record: PresenceRecord }
+  | { kind: "closed" | "down"; record: PresenceRecord }
+  | { kind: "unknown"; why: string };
+
+/** Classify ONE record against the process table. PURE — the witness's subject. */
+export function classifyRecord(record: PresenceRecord, table: ProcessTable): Classified {
+  if (record.state === "ended") {
+    // A decided or ambiguous END is a fact written by the agent itself: it holds
+    // even if the process is still unwinding.
+    return { kind: endVerdict(record.ended_reason as PresenceEndReason), record };
+  }
+  if (record.start_time_ms === null) return { kind: "unknown", why: `${record.instance_id}: start time not known yet` };
+  // Absent from the table → dead. Sound ONLY because the table passed its positive
+  // control (`enumerateStartTimes` saw the runner itself): a partial table never gets here.
+  if (!table.has(record.pid)) return { kind: "down", record };
+  const observed = table.get(record.pid);
+  if (observed === undefined) return { kind: "unknown", why: `${record.instance_id}: pid ${record.pid} has no readable start time` };
+  // ZERO tolerance: both numbers come from the same CIM expression (process_start.ts).
+  return observed === record.start_time_ms ? { kind: "present", record } : { kind: "down", record };
+}
+
+/** Read every record of this backend and classify it. Never throws. */
+export async function collectRoster(deps: CollectDeps): Promise<CollectedRoster> {
+  const dir = presenceDir(deps.identityBaseUrl, deps.lockDir);
+  let names: string[];
+  try {
+    names = readdirSync(dir).filter((n) => RECORD_FILE_RE.test(n));
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") return emptyCollected();
+    return emptyCollected(`presence directory unreadable (${String(err)}): nothing reported`);
+  }
+  if (names.length === 0) return emptyCollected();
+
+  const table = await deps.enumerate();
+  if (table === undefined) return emptyCollected("process enumeration failed its control: nothing reported");
+
+  const out = emptyCollected();
+  for (const name of names) {
+    const path = join(dir, name);
+    const read = readPresenceRecord(path);
+    if (read.kind !== "ok") {
+      if (read.kind === "unreadable") out.notes.push(`${name}: unreadable (${read.error})`);
+      continue;
+    }
+    const c = classifyRecord(read.record, table);
+    if (c.kind === "unknown") {
+      out.notes.push(c.why);
+      continue;
+    }
+    const { record } = c;
+    if (c.kind === "present") {
+      out.roster.present.push({
+        instance_id: record.instance_id,
+        conversation_id: record.conversation_id,
+        started_at: record.start_time_ms as number, // non-null: classifyRecord checked it
+      });
+      out.seen.push({
+        label: record.label,
+        project_id: record.project_id,
+        instance_id: record.instance_id,
+        conversation_id: record.conversation_id,
+        created_at: record.created_at,
+        job_name: record.job_name,
+      });
+    } else {
+      out.roster[c.kind].push(record.instance_id);
+      out.endedFiles.set(record.instance_id, { path, content: JSON.stringify(record) });
+    }
+  }
+  return capRoster(out);
+}
+
+/** Keep the beat under `ROSTER_MAX`: ended entries first (they must not be lost), then present. */
+function capRoster(c: CollectedRoster): CollectedRoster {
+  const { present, closed, down } = c.roster;
+  const total = present.length + closed.length + down.length;
+  if (total <= ROSTER_MAX) return c;
+  let room = ROSTER_MAX;
+  const keptClosed = closed.slice(0, room);
+  room -= keptClosed.length;
+  const keptDown = down.slice(0, room);
+  room -= keptDown.length;
+  const keptPresent = present.slice(0, room);
+  const kept = new Set([...keptClosed, ...keptDown]);
+  for (const id of [...c.endedFiles.keys()]) if (!kept.has(id)) c.endedFiles.delete(id);
+  // The registry must only learn what this beat actually told the server (Codex).
+  const sentPresent = new Set(keptPresent.map((p) => p.instance_id));
+  c.seen = c.seen.filter((s) => sentPresent.has(s.instance_id));
+  c.roster = { present: keptPresent, closed: keptClosed, down: keptDown };
+  c.notes.push(`roster capped at ${ROSTER_MAX} (had ${total}); the rest goes next beat`);
+  return c;
+}
+
+/** The server's answer to a roster beat (step-04 contract). */
+export interface RosterAck {
+  /** true only when Python applied the presence; false/absent → resend next beat. */
+  presence_applied: boolean;
+  /** The server's id of this beat: the identifier the agents count (step-06). */
+  beat_id: string | null;
+  /** Lease state per present instance: true / false / null (indeterminate). */
+  instances: Map<string, boolean | null>;
+  /** #627 (decision A): replaced instances whose conversations are not handed over yet. */
+  handover_pending: Set<string>;
+}
+
+/** Parse a 2xx body. Anything unexpected reads as "not applied, no lease info". */
+export function parseRosterAck(body: unknown): RosterAck {
+  const ack: RosterAck = { presence_applied: false, beat_id: null, instances: new Map(), handover_pending: new Set() };
+  if (typeof body !== "object" || body === null) return ack;
+  const b = body as Record<string, unknown>;
+  ack.presence_applied = b.presence_applied === true;
+  ack.beat_id = typeof b.beat_id === "string" && b.beat_id !== "" ? b.beat_id : null;
+  if (Array.isArray(b.handover_pending)) {
+    for (const id of b.handover_pending) if (typeof id === "string") ack.handover_pending.add(id);
+  }
+  if (typeof b.instances === "object" && b.instances !== null) {
+    for (const [id, v] of Object.entries(b.instances as Record<string, unknown>)) {
+      const lease = typeof v === "object" && v !== null ? (v as Record<string, unknown>).lease_active : undefined;
+      if (lease === true || lease === false || lease === null) ack.instances.set(id, lease);
+    }
+  }
+  return ack;
+}
+
+/** One instance's coverage, read by its agent (step-06). */
+export interface Coverage {
+  schema_version: 1;
+  instance_id: string;
+  lease_active: boolean | null;
+  /** EXACTLY the server's value: the agent counts each id once. */
+  beat_id: string;
+  /** The conversation the lease was renewed for. */
+  conversation_id: string | null;
+  /** LOCAL clock of this write: freshness is judged on it, never on beat_id. */
+  written_at: number;
+}
+
+/** `<instance_id>.coverage.json` — does not match `RECORD_FILE_RE`, by construction. */
+export function coveragePath(identityBaseUrl: string, instanceId: string, lockDir?: string): string {
+  return join(presenceDir(identityBaseUrl, lockDir), `${instanceId}.coverage.json`);
+}
+
+function atomicWrite(path: string, value: unknown): void {
+  mkdirSync(dirname(path), { recursive: true });
+  const tmp = `${path}.${process.pid}.tmp`;
+  writeFileSync(tmp, JSON.stringify(value), "utf8");
+  try {
+    renameSync(tmp, path);
+  } catch (err) {
+    try {
+      unlinkSync(tmp);
+    } catch {
+      // already gone
+    }
+    throw err;
+  }
+}
+
+export interface ApplyAckDeps {
+  identityBaseUrl: string;
+  lockDir?: string;
+  now: () => number;
+  log: (line: string) => void;
+}
+
+/**
+ * After a 2xx: write each present instance's coverage (when the server gave its
+ * lease and a beat id), and — only when the presence was APPLIED — remove the
+ * acknowledged closed/down records. Returns the ids whose record was removed.
+ */
+export function applyAck(collected: CollectedRoster, ack: RosterAck, deps: ApplyAckDeps): string[] {
+  if (ack.beat_id !== null) {
+    for (const p of collected.roster.present) {
+      if (!ack.instances.has(p.instance_id)) continue;
+      const coverage: Coverage = {
+        schema_version: 1,
+        instance_id: p.instance_id,
+        lease_active: ack.instances.get(p.instance_id) ?? null,
+        beat_id: ack.beat_id,
+        conversation_id: p.conversation_id,
+        written_at: deps.now(),
+      };
+      try {
+        atomicWrite(coveragePath(deps.identityBaseUrl, p.instance_id, deps.lockDir), coverage);
+      } catch (err) {
+        deps.log(`[runner] coverage write failed for ${p.instance_id}: ${String(err)}`);
+      }
+    }
+  }
+  if (!ack.presence_applied) return [];
+  const removed: string[] = [];
+  for (const [id, { path, content }] of collected.endedFiles) {
+    // The file may have been rewritten while the beat was in flight: remove only
+    // the version the server acknowledged. Compared on the WHOLE record, not on
+    // `written_at`, which two writes can share within one millisecond (Codex).
+    const now = readPresenceRecord(path);
+    if (now.kind !== "ok" || JSON.stringify(now.record) !== content) continue;
+    try {
+      unlinkSync(path);
+      removed.push(id);
+    } catch (err) {
+      deps.log(`[runner] could not remove acknowledged record ${id}: ${String(err)}`);
+    }
+    try {
+      unlinkSync(coveragePath(deps.identityBaseUrl, id, deps.lockDir));
+    } catch {
+      // no coverage for it: nothing to clean
+    }
+  }
+  return removed;
+}
+
+export interface RosterBeatAuth {
+  base_url: string;
+  runner_id: string;
+  runner_token: string;
+  identity_base_url?: string;
+}
+
+export interface RosterBeatDeps {
+  fetchImpl?: typeof fetch;
+  enumerate?: () => Promise<ProcessTable | undefined>;
+  lockDir?: string;
+  now?: () => number;
+  log?: (line: string) => void;
+  headers?: () => Record<string, string>;
+}
+
+/**
+ * ONE runner heartbeat carrying the roster, and everything a 2xx makes true locally:
+ * coverage files, acknowledged records removed, the registry kept in step, and
+ * `runner.json`. Returns "revoked" on a 401 so the daemon tears down.
+ *
+ * A failed or non-2xx beat changes NOTHING on disk: the coverage then expires on its
+ * own and the ended records go again next beat.
+ */
+export async function runRosterBeat(auth: RosterBeatAuth, deps: RosterBeatDeps = {}): Promise<"ok" | "revoked"> {
+  // SINGLE-FLIGHT. The enumeration alone can take up to 30 s — one heartbeat period —
+  // so an overlapping beat is ordinary, and two answers arriving in reverse order
+  // would rewrite an older coverage over a newer one (Codex, step-03 review). A tick
+  // that finds a beat in flight is skipped: the running one covers it.
+  if (beatInFlight) {
+    noteOnce("a heartbeat was skipped: the previous one is still in flight", deps.log ?? ((l: string) => process.stderr.write(`${l}\n`)));
+    return "ok";
+  }
+  beatInFlight = true;
+  try {
+    return await rosterBeatOnce(auth, deps);
+  } finally {
+    beatInFlight = false;
+  }
+}
+let beatInFlight = false;
+let unappliedBeats = 0;
+/** #627 (decision A): pending beats before the runner says the handover is slow. */
+export const HANDOVER_WARN_AFTER = 10;
+const handoverAttempts = new Map<string, number>();
+/** Consecutive unacknowledged beats (with ended records) before the runner says so. */
+export const UNAPPLIED_WARN_AFTER = 10;
+
+async function rosterBeatOnce(auth: RosterBeatAuth, deps: RosterBeatDeps): Promise<"ok" | "revoked"> {
+  const log = deps.log ?? ((l: string) => process.stderr.write(`${l}\n`));
+  const now = deps.now ?? Date.now;
+  const identity = runnerIdentity(auth);
+  const collected = await collectRoster({
+    identityBaseUrl: identity,
+    lockDir: deps.lockDir,
+    enumerate: deps.enumerate ?? (() => enumerateStartTimes()),
+  });
+  for (const note of collected.notes) noteOnce(note, log);
+  // #627 step-07: an instance REPLACED by a relaunch is reported closed until the
+  // server acknowledges it — so no greyed ghost stays behind the new agent.
+  const { closes: pendingCloses, replaced } = pendingClosesOf(identity, deps.lockDir, collected.roster);
+  collected.roster.closed.push(...pendingCloses);
+
+  let resp: Response;
+  try {
+    resp = await (deps.fetchImpl ?? fetch)(`${auth.base_url}/api/runners/${auth.runner_id}/heartbeat`, {
+      method: "POST",
+      // never follow a redirect while carrying the bearer (Codex review, #460)
+      redirect: "error",
+      headers: {
+        Authorization: `Bearer ${auth.runner_token}`,
+        "Content-Type": "application/json",
+        ...(deps.headers?.() ?? {}),
+      },
+      body: JSON.stringify({
+        capabilities: { runtimes: ["claude-code", "codex", "gemma"], platform: process.platform, version: 1 },
+        // #627 (JP, decision A): the new instance of each pair takes the old one's
+        // conversations — the server re-invites it once it is online.
+        roster: replaced.length > 0 ? { ...collected.roster, replaced } : collected.roster,
+      }),
+    });
+  } catch {
+    return "ok"; // best-effort: nothing acknowledged, nothing changed
+  }
+  if (resp.status === 401) return "revoked";
+  if (resp.status < 200 || resp.status >= 300) {
+    // Not swallowed: a 4xx here means the server refused the beat — the RUNNER itself
+    // then goes offline after 90 s (Opus, step-03 review).
+    noteOnce(`heartbeat refused: HTTP ${resp.status}`, log);
+    return "ok";
+  }
+
+  const ack = parseRosterAck(await resp.json().catch(() => null));
+  // A beat whose ended records are never acknowledged is not dangerous (coverage and
+  // leases are still served) but it is SILENT: say it once it lasts (Opus, step-05).
+  if (collected.endedFiles.size > 0 && !ack.presence_applied) {
+    unappliedBeats++;
+    if (unappliedBeats === UNAPPLIED_WARN_AFTER) {
+      log(`[runner] roster not applied by the server for ${unappliedBeats} beats in a row — ended records are kept and resent`);
+    }
+  } else {
+    unappliedBeats = 0;
+  }
+  const removed = applyAck(collected, ack, { identityBaseUrl: identity, lockDir: deps.lockDir, now, log });
+  const closedNow = removed.filter((id) => collected.roster.closed.includes(id));
+  const result = reg.updateRegistry(reg.registryPath(identity, deps.lockDir), (r) => {
+    const synced = reg.syncFromRecords(r, collected.seen);
+    const closed = reg.markClosed(r, closedNow, now());
+    let cleared = false;
+    if (ack.presence_applied) {
+      for (const e of Object.values(r.entries)) {
+        // An older predecessor (handover_from) is done once the server no longer
+        // answers it pending — same rule as `pending_close` below.
+        if (e.handover_from?.length) {
+          const kept = e.handover_from.filter((id) => !pendingCloses.includes(id) || ack.handover_pending.has(id));
+          if (kept.length !== e.handover_from.length) {
+            e.handover_from = kept;
+            cleared = true;
+          }
+        }
+        if (e.pending_close && pendingCloses.includes(e.pending_close)) {
+          const id = e.pending_close;
+          // #627 (decision A): the server could not yet hand this agent's conversations
+          // to its replacement — keep the close and resend the pair (Opus I1). NEVER given
+          // up (Codex): a dropped pair would leave the conversations with nobody and no way
+          // to retry. It is bounded by the replacement's life: the pair travels only while
+          // the new agent is present. A long wait is said once, not silently absorbed.
+          if (ack.handover_pending.has(id)) {
+            const n = (handoverAttempts.get(id) ?? 0) + 1;
+            handoverAttempts.set(id, n);
+            if (n === HANDOVER_WARN_AFTER) {
+              log(`[runner] relaunch: conversations of ${id.slice(0, 8)} still not handed over after ${n} beats — retrying`);
+            }
+            continue;
+          }
+          handoverAttempts.delete(id);
+          e.pending_close = null;
+          cleared = true;
+        }
+      }
+    }
+    return synced || closed || cleared;
+  });
+  if (!result.ok) noteOnce(`registry not updated: ${result.error}`, log);
+  try {
+    reg.writeRunnerState(reg.runnerStatePath(identity, deps.lockDir), { runner_id: auth.runner_id, last_beat_ok_at: now() });
+  } catch (err) {
+    log(`[runner] runner.json write failed: ${String(err)}`);
+  }
+  return "ok";
+}
+
+/** A note logged at every 30 s beat would drown the log: once per distinct text. */
+const loggedNotes = new Set<string>();
+function noteOnce(note: string, log: (line: string) => void): void {
+  if (loggedNotes.has(note)) return;
+  if (loggedNotes.size > 500) loggedNotes.clear();
+  loggedNotes.add(note);
+  log(`[runner] roster: ${note}`);
+}
+
+/** One relaunch: the instance it replaced, and the one that now runs. */
+export interface ReplacedPair {
+  old: string;
+  new: string;
+}
+
+/**
+ * Replaced instances still to report closed — never one the roster already carries.
+ * #627 (JP, decision A): a close waits until its REPLACEMENT is present in this same
+ * roster, and travels with the pair, so the server can hand the old instance's
+ * conversations to the new one at the one moment the old is retired.
+ */
+export function pendingClosesOf(
+  identity: string,
+  lockDir: string | undefined,
+  roster: Roster,
+): { closes: string[]; replaced: ReplacedPair[] } {
+  const read = reg.readRegistry(reg.registryPath(identity, lockDir));
+  if (read.kind !== "ok") return { closes: [], replaced: [] };
+  const present = new Set(roster.present.map((p) => p.instance_id));
+  const carried = new Set([...present, ...roster.closed, ...roster.down]);
+  const room = Math.max(0, ROSTER_MAX - carried.size);
+  const closes: string[] = [];
+  const replaced: ReplacedPair[] = [];
+  for (const e of Object.values(read.registry.entries)) {
+    const next = e.instance_id;
+    if (!next || !present.has(next)) continue; // its replacement is not running yet
+    for (const id of [e.pending_close, ...(e.handover_from ?? [])]) {
+      if (!id || id === next || carried.has(id) || closes.includes(id) || closes.length >= room) continue;
+      closes.push(id);
+      replaced.push({ old: id, new: next });
+    }
+  }
+  return { closes, replaced };
+}