quoxcli 0.0.0-stage → 0.3.1

package/lib/core/residentAlarm.js

@@ -0,0 +1,282 @@
+/**
+ * Resident alarm — QUOXPERTICUS-REVISIT P5 "warden watchdog + loud alarm".
+ *
+ * While the warden pumps (residentPump.js) instead of answering, it also
+ * watches the resident it wards: is the tmux pane still there, has a
+ * permission dialog been stuck too long to ever clear itself, has the
+ * provider started refusing turns. Three independent conditions, each with
+ * its own escalating alert schedule (10m, 30m, then hourly) so a wedged
+ * resident is loud rather than silently absent, and each condition clears
+ * itself the moment its detector stops firing (no human-ack mechanism yet:
+ * clearing = the underlying thing is gone).
+ *
+ * Every I/O dependency (tmux session list, pane read, release/redeploy,
+ * the alert transport, the turn-receipt store, the clock) is injected, same
+ * contract as supervisorEngine.js and residentPump.js, so this runs under
+ * vitest with no real tmux host and no real Telegram bridge.
+ */
+
+import { ERROR_NOTICE_WINDOW_MS } from './supervisorEngine.js';
+
+// Gaps BETWEEN successive alerts for the SAME still-active condition:
+// immediate on first detection (see shouldSendAlarm), then +10m, then +30m,
+// then hourly forever until the condition clears.
+export const ALARM_SCHEDULE_MS = [10 * 60 * 1000, 30 * 60 * 1000, ERROR_NOTICE_WINDOW_MS];
+
+// A stuck permission dialog only becomes an alarm condition once it has sat
+// unresolved this long. Below this age it is exactly what residentPump.js
+// already logs: held input, not yet an incident.
+export const STUCK_DIALOG_ALARM_THRESHOLD_MS = 10 * 60 * 1000;
+
+// Pane-text scan patterns for a provider/credits failure. Deliberately
+// simple substring/regex matches (this is pane TEXT, not a structured
+// provider error object the way detectTurnError in supervisorEngine.js
+// parses a headless turn's JSON output) — same spirit, different signal.
+export const PROVIDER_ISSUE_PATTERNS = [
+  { pattern: 'out of usage credits', regex: /out of usage credits/i },
+  { pattern: 'usage limit', regex: /usage limit/i },
+  { pattern: 'provider error', regex: /provider error/i },
+];
+
+export function scanPaneForProviderIssue(paneText) {
+  const text = String(paneText || '');
+  for (const { pattern, regex } of PROVIDER_ISSUE_PATTERNS) {
+    if (regex.test(text)) return { matched: true, pattern };
+  }
+  return { matched: false, pattern: null };
+}
+
+export function createInitialAlarmCondition() {
+  return { active: false, first_detected_at: null, last_alerted_at: null, alert_count: 0 };
+}
+
+export function createInitialAlarmState() {
+  return {
+    pane_dead_attempted: false,
+    pane_dead_redeploy_failed: false,
+    conditions: {
+      pane_dead: createInitialAlarmCondition(),
+      stuck_dialog: createInitialAlarmCondition(),
+      provider_error: createInitialAlarmCondition(),
+    },
+  };
+}
+
+/**
+ * Advance one condition's state from a single boolean detection this pass.
+ * A detection while already active is a no-op (first_detected_at sticks to
+ * the original onset). No detection while active is a full reset: clearing
+ * a condition means the next detection is treated as a fresh onset, with
+ * its own fresh alert schedule.
+ */
+export function updateAlarmCondition(conditionState, detected, now) {
+  const current = conditionState || createInitialAlarmCondition();
+  if (detected) {
+    if (current.active) return current;
+    return { ...current, active: true, first_detected_at: now };
+  }
+  if (current.active) return createInitialAlarmCondition();
+  return current;
+}
+
+/**
+ * Whether condition `conditionState` should alert right now. The first
+ * alert for a freshly-active condition fires immediately (no delay: a dead
+ * pane or a provider outage is news the moment it is confirmed). Every
+ * alert after that waits at least the schedule's gap for however many
+ * alerts have already fired, capped at the schedule's last (hourly) entry.
+ */
+export function shouldSendAlarm(conditionState, now, schedule = ALARM_SCHEDULE_MS) {
+  if (!conditionState || !conditionState.active) return false;
+  if (conditionState.last_alerted_at == null) return true;
+  const idx = Math.min(Math.max((conditionState.alert_count || 0) - 1, 0), schedule.length - 1);
+  const gap = schedule[idx];
+  return (now - conditionState.last_alerted_at) >= gap;
+}
+
+export function recordAlarmSent(conditionState, now) {
+  return { ...conditionState, last_alerted_at: now, alert_count: (conditionState.alert_count || 0) + 1 };
+}
+
+/**
+ * Send one alarm through both transports in order, per spec: tg-send.sh
+ * first (the owner's phone), `quox slot say` second as the fallback when
+ * tg-send.sh is unavailable or fails — never silent, and the absence of the
+ * script is itself treated as "alarm-transport-unavailable", not a reason
+ * to skip the slot notice.
+ */
+export async function sendAlarm({ tgSend, postSlotNotice, text, sessionName = 'WARDEN', log = () => {} }) {
+  let tgSent = false;
+  if (tgSend) {
+    try {
+      await tgSend(sessionName, text);
+      tgSent = true;
+    } catch (err) {
+      log('warn', `alarm: tg-send.sh failed, falling back to slot say: ${err.message}`);
+    }
+  } else {
+    log('warn', 'alarm: alarm-transport-unavailable (tg-send.sh not found), falling back to slot say');
+  }
+  if (!tgSent) {
+    try {
+      await postSlotNotice(text);
+    } catch (err) {
+      log('error', `alarm: slot say fallback also failed: ${err.message}`);
+    }
+  }
+  return { tg_sent: tgSent };
+}
+
+/**
+ * Run one warden watchdog pass.
+ *
+ * Injected dependencies:
+ *   listSessions()        -> array of the host's tmux sessions ({name, ...})
+ *   readPane()             -> current pane text (only called while the pane
+ *                             is confirmed alive)
+ *   releaseSlot()           -> release the resident's slot (best-effort)
+ *   redeploy()               -> run the redeploy path (reuses `quox dev
+ *                             resident deploy`; throws on failure)
+ *   claimFallback()          -> attempt to claim the slot as this warden's
+ *                             own fallback voice (only called when redeploy
+ *                             fails); returns {claimed: bool}
+ *   postSlotNotice(text)     -> slot `say`
+ *   recordTurn(fields)       -> turn receipt (outcome/error_note only;
+ *                             NEVER a new outcome value — see
+ *                             devSupervisorTurnsStore.js OUTCOMES)
+ *   tgSend(session, text)    -> shells out to tg-send.sh; absent/failing is
+ *                             handled by sendAlarm above, never thrown here
+ *   stuckDialogSinceMs        -> residentPump.js's state.stuck_dialog_since
+ *                             for this slot (single source of truth for
+ *                             dialog detection; this module only ages it)
+ */
+export async function runWardenPass({
+  hostId,
+  slot,
+  sessionName,
+  listSessions,
+  readPane,
+  releaseSlot,
+  redeploy,
+  claimFallback,
+  postSlotNotice,
+  recordTurn,
+  tgSend,
+  stuckDialogSinceMs = null,
+  state,
+  log = () => {},
+  now = () => Date.now(),
+  schedule = ALARM_SCHEDULE_MS,
+  stuckDialogThresholdMs = STUCK_DIALOG_ALARM_THRESHOLD_MS,
+}) {
+  const base = state || createInitialAlarmState();
+  const nextState = {
+    pane_dead_attempted: base.pane_dead_attempted ?? false,
+    pane_dead_redeploy_failed: base.pane_dead_redeploy_failed ?? false,
+    conditions: {
+      pane_dead: base.conditions?.pane_dead || createInitialAlarmCondition(),
+      stuck_dialog: base.conditions?.stuck_dialog || createInitialAlarmCondition(),
+      provider_error: base.conditions?.provider_error || createInitialAlarmCondition(),
+    },
+  };
+
+  const nowMs = now();
+  const report = { pane_alive: null, redeployed: false, redeploy_error: null, fallback_claimed: false, alerts_sent: [] };
+
+  const sessions = await listSessions();
+  const paneAlive = !!(sessions || []).find((s) => s.name === sessionName);
+  report.pane_alive = paneAlive;
+
+  let paneText = null;
+
+  if (!paneAlive) {
+    if (!nextState.pane_dead_attempted) {
+      nextState.pane_dead_attempted = true;
+      log('warn', `warden: resident session ${sessionName} on ${hostId} is not present (pane dead); releasing the slot and redeploying.`);
+      try {
+        await releaseSlot();
+      } catch (err) {
+        log('warn', `warden: releasing ${slot} failed (continuing): ${err.message}`);
+      }
+      try {
+        await redeploy();
+        report.redeployed = true;
+        nextState.pane_dead_redeploy_failed = false;
+        nextState.conditions.pane_dead = createInitialAlarmCondition();
+        try {
+          await postSlotNotice(`Resident on ${hostId} lost its pane; redeployed automatically.`);
+        } catch (err) {
+          log('warn', `warden: redeploy notice failed to post: ${err.message}`);
+        }
+        try {
+          await recordTurn({ outcome: 'error', error_note: 'pane_dead_auto_redeployed' });
+        } catch (err) {
+          log('warn', `warden: redeploy receipt failed to post: ${err.message}`);
+        }
+      } catch (err) {
+        report.redeploy_error = err.message;
+        nextState.pane_dead_redeploy_failed = true;
+        log('error', `warden: redeploy failed for ${hostId}: ${err.message}`);
+        try {
+          await postSlotNotice(`Resident on ${hostId} lost its pane and automatic redeploy FAILED (${err.message}). A human should check this host.`);
+        } catch (err2) {
+          log('warn', `warden: failure notice failed to post: ${err2.message}`);
+        }
+        try {
+          const fb = await claimFallback();
+          report.fallback_claimed = !!(fb && fb.claimed);
+        } catch (err2) {
+          log('warn', `warden: fallback claim attempt failed: ${err2.message}`);
+        }
+      }
+    }
+    if (nextState.pane_dead_redeploy_failed) {
+      nextState.conditions.pane_dead = updateAlarmCondition(nextState.conditions.pane_dead, true, nowMs);
+    }
+  } else {
+    if (nextState.pane_dead_attempted || nextState.conditions.pane_dead.active) {
+      log('info', `warden: resident session ${sessionName} on ${hostId} is present again; pane-dead condition cleared.`);
+    }
+    nextState.pane_dead_attempted = false;
+    nextState.pane_dead_redeploy_failed = false;
+    nextState.conditions.pane_dead = updateAlarmCondition(nextState.conditions.pane_dead, false, nowMs);
+
+    try {
+      paneText = await readPane();
+    } catch (err) {
+      log('warn', `warden: pane read failed on ${hostId} (skipping provider-error scan this pass): ${err.message}`);
+    }
+  }
+
+  // Stuck dialog: this module never detects the dialog itself, it only ages
+  // what residentPump.js already found (single source of truth).
+  const dialogStuck = stuckDialogSinceMs != null && (nowMs - stuckDialogSinceMs) >= stuckDialogThresholdMs;
+  nextState.conditions.stuck_dialog = updateAlarmCondition(nextState.conditions.stuck_dialog, dialogStuck, nowMs);
+
+  if (paneText != null) {
+    const scan = scanPaneForProviderIssue(paneText);
+    nextState.conditions.provider_error = updateAlarmCondition(nextState.conditions.provider_error, scan.matched, nowMs);
+  }
+
+  const conditionAlerts = [
+    ['pane_dead', `Resident on ${hostId}: pane is dead and automatic redeploy failed. A human should check this host.`],
+    ['stuck_dialog', `Resident on ${hostId}: a permission dialog has been stuck for over ${Math.round(stuckDialogThresholdMs / 60000)} minutes and will never resolve itself; a human must approve or cancel it.`],
+    ['provider_error', `Resident on ${hostId}: a provider/credits error was detected in its pane. A human should check its account/billing status.`],
+  ];
+
+  for (const [key, message] of conditionAlerts) {
+    const condition = nextState.conditions[key];
+    if (shouldSendAlarm(condition, nowMs, schedule)) {
+      const sendResult = await sendAlarm({ tgSend, postSlotNotice, text: message, sessionName: `WARDEN-${hostId}`, log });
+      nextState.conditions[key] = recordAlarmSent(condition, nowMs);
+      try {
+        await recordTurn({ outcome: 'error', error_note: `alarm_${key}` });
+      } catch (err) {
+        log('warn', `warden: alarm receipt failed for ${key}: ${err.message}`);
+      }
+      report.alerts_sent.push({ condition: key, tg_sent: sendResult.tg_sent });
+    }
+  }
+
+  return { state: nextState, ...report };
+}

package/lib/core/residentAuth.js

@@ -0,0 +1,243 @@
+/**
+ * Resident agent authentication for QuoxCLI (quox#453 phase 15)
+ *
+ * A dev server's resident agent authenticates with an Ed25519 key on disk
+ * instead of a password, zero human minting. The exchange is a signed
+ * challenge:
+ *
+ *   1. POST {baseUrl}/api/auth/resident/challenge {org_id, host_id}
+ *        -> {challenge_id, challenge_b64, expires_in}
+ *   2. sign the raw challenge bytes with the Ed25519 private key at key_path
+ *   3. POST {baseUrl}/api/auth/resident/token {challenge_id, org_id, host_id, signature_b64}
+ *        -> {access_token, token_type, expires_in, user}
+ *
+ * Bootstrap file on a dev host (not secret itself, points at the real key):
+ *   ~/.config/quox/resident-profile.json
+ *   {kind: 'resident', org_id, host_id, key_id, key_path, instance_url}
+ *
+ * Neither the private key nor any resulting token is ever printed or logged
+ * by this module.
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { sign, createPrivateKey } from 'node:crypto';
+import { QuoxError, QuoxAuthError, QuoxTransientAuthError } from './errors.js';
+
+export function defaultResidentProfilePath() {
+  return join(homedir(), '.config', 'quox', 'resident-profile.json');
+}
+
+export function expandHome(p) {
+  if (!p) return p;
+  if (p === '~') return homedir();
+  if (p.startsWith('~/')) return join(homedir(), p.slice(2));
+  return p;
+}
+
+/**
+ * Load and validate the resident bootstrap profile.
+ * @param {string} [explicitPath] - overrides the default ~/.config/quox/resident-profile.json
+ */
+export function loadResidentProfile(explicitPath) {
+  const path = expandHome(explicitPath) || defaultResidentProfilePath();
+
+  if (!existsSync(path)) {
+    throw new QuoxError(
+      `No resident profile found at ${path}. A resident agent needs this bootstrap file `
+      + `({kind, org_id, host_id, key_id, key_path, instance_url}) before it can authenticate. `
+      + `Run --resident-profile <path> to point at a different one.`
+    );
+  }
+
+  let raw;
+  try {
+    raw = readFileSync(path, 'utf-8');
+  } catch (err) {
+    throw new QuoxError(`Could not read resident profile at ${path}: ${err.message}`);
+  }
+
+  let data;
+  try {
+    data = JSON.parse(raw);
+  } catch (err) {
+    throw new QuoxError(`Resident profile at ${path} is not valid JSON: ${err.message}`);
+  }
+
+  const required = ['org_id', 'host_id', 'key_path'];
+  const missing = required.filter((key) => !data[key]);
+  if (missing.length > 0) {
+    throw new QuoxError(
+      `Resident profile at ${path} is missing required field(s): ${missing.join(', ')}.`
+    );
+  }
+
+  return { ...data, kind: 'resident', _path: path };
+}
+
+/**
+ * Prefer the bootstrap file's instance_url when the active CLI profile has
+ * no explicit URL of its own: a resident agent bootstraps its own instance
+ * pointer, it shouldn't need a human to also run `quox config`.
+ */
+export function resolveResidentBaseUrl(config, profile) {
+  // quox#455 gap 1: the resident endpoints live on the instance EDGE
+  // (https://<host>:3000/api/auth/resident/*, proxied to auth). NEVER
+  // derive them by port-swapping to :3101: auth binds loopback on the
+  // control box, so :3101 is connection-refused from every remote host.
+  if (process.env.QUOX_AUTH_URL) return process.env.QUOX_AUTH_URL;
+  if (process.env.QUOX_URL) return process.env.QUOX_URL;
+  const activeProfile = typeof config.getActiveProfile === 'function' ? config.getActiveProfile() : null;
+  const hasExplicitUrl = !!(activeProfile && activeProfile.url && activeProfile.url !== 'http://localhost:3000');
+  if (hasExplicitUrl) return activeProfile.url;
+  if (profile?.instance_url) return profile.instance_url;
+  // Local default profile on the control box itself: auth is directly
+  // reachable on loopback :3101, keep the historical local-dev behaviour.
+  return config.getServiceUrl('auth');
+}
+
+/**
+ * True only when `urlStr` is https AND its host is a bare IP or a
+ * .local/LAN-style single-label name: the shapes that can never carry a
+ * real, publicly-issued certificate, so a self-signed edge is the expected
+ * (not the exceptional) case. A public FQDN with a real cert must never get
+ * this baked in, even if it happens to be reachable over the LAN.
+ */
+function isSelfSignedLikelyHost(urlStr) {
+  let u;
+  try {
+    u = new URL(urlStr);
+  } catch {
+    return false;
+  }
+  if (u.protocol !== 'https:') return false;
+  const host = u.hostname;
+  if (!host) return false;
+  if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host)) return true; // bare IPv4
+  if (host.includes(':')) return true; // bare IPv6 (URL strips the brackets)
+  if (/\.local$/i.test(host)) return true; // mDNS / LAN-style
+  if (!host.includes('.')) return true; // single-label hostname, never a public FQDN
+  return false;
+}
+
+/**
+ * Derive the systemd unit environment for a resident's headless supervisor
+ * from its bootstrap profile (item 1, live outage 2026-08-23: a unit run
+ * directly by systemd inherits none of the caller's shell environment, so
+ * every re-auth against a self-signed LAN edge died with a bare "fetch
+ * failed"). Baked in once, at install time, from the same profile the
+ * resident already trusts for its own identity: QUOX_URL and QUOX_AUTH_URL
+ * from instance_url, QUOX_COLLECTOR_URL from collector_url, and
+ * QUOX_INSECURE only when isSelfSignedLikelyHost(instance_url) is true.
+ * Returns {} for a missing profile or a profile with neither URL: the
+ * caller is responsible for warning in that case, this function only ever
+ * derives, never decides whether the result is enough.
+ */
+export function deriveSupervisorEnv(profile) {
+  const env = {};
+  if (!profile) return env;
+  if (profile.instance_url) {
+    env.QUOX_URL = profile.instance_url;
+    env.QUOX_AUTH_URL = profile.instance_url;
+    if (isSelfSignedLikelyHost(profile.instance_url)) env.QUOX_INSECURE = '1';
+  }
+  if (profile.collector_url) env.QUOX_COLLECTOR_URL = profile.collector_url;
+  return env;
+}
+
+const TRANSIENT_STATUSES = new Set([429, 502, 503, 504]);
+
+// Retry-After is delta-seconds or an HTTP date; returns ms or null.
+function parseRetryAfterMs(res) {
+  const raw = res?.headers && typeof res.headers.get === 'function' ? res.headers.get('retry-after') : null;
+  if (!raw) return null;
+  const secs = Number(raw);
+  if (Number.isFinite(secs) && secs >= 0) return Math.round(secs * 1000);
+  const at = Date.parse(raw);
+  return Number.isFinite(at) ? Math.max(0, at - Date.now()) : null;
+}
+
+function exchangeFailure(what, res, data) {
+  const msg = `${what} failed (${res.status}): ${data?.error || res.statusText || 'unknown error'}`;
+  if (TRANSIENT_STATUSES.has(res.status)) return new QuoxTransientAuthError(msg, parseRetryAfterMs(res));
+  return new QuoxAuthError(msg);
+}
+
+async function safeJson(res) {
+  try {
+    return await res.json();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Run the challenge/sign/token exchange and return the token response.
+ * Never logs the private key or the returned token.
+ *
+ * @param {object} opts
+ * @param {string} opts.baseUrl
+ * @param {object} opts.profile - result of loadResidentProfile()
+ * @param {typeof fetch} [opts.fetchImpl]
+ */
+export async function residentLogin({ baseUrl, profile, fetchImpl = fetch }) {
+  const { org_id, host_id, key_path } = profile;
+  const keyPath = expandHome(key_path);
+
+  if (!existsSync(keyPath)) {
+    throw new QuoxError(
+      `Resident identity key not found at ${keyPath}. Provision the Ed25519 key before logging in.`
+    );
+  }
+
+  let keyPem;
+  try {
+    keyPem = readFileSync(keyPath, 'utf-8');
+  } catch (err) {
+    throw new QuoxError(`Could not read resident identity key at ${keyPath}: ${err.message}`);
+  }
+
+  let privateKey;
+  try {
+    privateKey = createPrivateKey(keyPem);
+  } catch (err) {
+    throw new QuoxError(`Resident identity key at ${keyPath} is not a valid private key: ${err.message}`);
+  }
+
+  // Step 1: request a fresh challenge.
+  const challengeRes = await fetchImpl(`${baseUrl}/api/auth/resident/challenge`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ org_id, host_id }),
+  });
+  const challengeData = await safeJson(challengeRes);
+  if (!challengeRes.ok) {
+    throw exchangeFailure('Resident challenge request', challengeRes, challengeData);
+  }
+  const { challenge_id, challenge_b64 } = challengeData || {};
+  if (!challenge_id || !challenge_b64) {
+    throw new QuoxError('Resident challenge response is missing challenge_id or challenge_b64.');
+  }
+
+  // Step 2: sign the raw challenge bytes with the Ed25519 key.
+  const challengeBytes = Buffer.from(challenge_b64, 'base64');
+  const signatureBytes = sign(null, challengeBytes, privateKey);
+  const signature_b64 = signatureBytes.toString('base64');
+
+  // Step 3: exchange the signature for an access token.
+  const tokenRes = await fetchImpl(`${baseUrl}/api/auth/resident/token`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ challenge_id, org_id, host_id, signature_b64 }),
+  });
+  const tokenData = await safeJson(tokenRes);
+  if (!tokenRes.ok) {
+    throw exchangeFailure('Resident token exchange', tokenRes, tokenData);
+  }
+  if (!tokenData?.access_token) {
+    throw new QuoxError('Resident token response is missing access_token.');
+  }
+
+  return tokenData;
+}