quoxcli 0.3.1 → 0.4.0

package/lib/commands/kumaSla.js

@@ -0,0 +1,174 @@
+/**
+ * quox kuma sla ... : signed, verifiable uptime reports (NQ-KUMA-7).
+ *
+ * quox kuma sla [--month YYYY-MM | --from <iso> --to <iso>] [--out <file>] [--json]   generate + sign + store
+ * quox kuma sla list                                                                   this org's stored reports
+ * quox kuma sla verify <id|file> [--public-key <base64>]                               recompute the hash, check the signature
+ *
+ * `verify <id>` asks the collector, which checks against its pinned WARD key (trust anchored).
+ * `verify <file>` is fully OFFLINE: it recomputes sha256 over the canonical report and checks the Ed25519
+ * signature with node:crypto. Against the key embedded in the file this proves integrity only (anyone can
+ * mint a keypair); pass --public-key <raw 32 byte base64 WARD key> to pin the signer yourself.
+ *
+ * Exit codes: 0 ok / verified, 1 error, 2 verification FAILED.
+ */
+
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { createHash, createPublicKey, verify as cryptoVerify } from 'node:crypto';
+import { resolveEndpoint } from '../api/endpoints.js';
+import { line as glyphLine } from '../core/quoxglyph.js';
+
+// Same canonical form as services/collector/lib/bundleSigner.js canonicalize (sorted keys, no whitespace).
+export function canonicalize(val) {
+  if (val === null || val === undefined) return JSON.stringify(val);
+  if (typeof val !== 'object') return JSON.stringify(val);
+  if (Array.isArray(val)) return `[${val.map((v) => canonicalize(v)).join(',')}]`;
+  const keys = Object.keys(val).filter((k) => val[k] !== undefined).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalize(val[k])}`).join(',')}}`;
+}
+
+const SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex');
+const keyFromRaw = (b64) => {
+  try {
+    const raw = Buffer.from(String(b64 || ''), 'base64');
+    if (raw.length !== 32) return null;
+    return createPublicKey({ key: Buffer.concat([SPKI_PREFIX, raw]), format: 'der', type: 'spki' });
+  } catch { return null; }
+};
+
+/** Offline verification of an export bundle. Mirrors kumaSlaReports.verifyBundle. */
+export function verifyBundleOffline(bundle, { publicKey } = {}) {
+  const out = { valid: false, hash_ok: false, binding_ok: false, signature_ok: false, key_source: 'none', trust_anchored: false };
+  if (!bundle || typeof bundle !== 'object' || !bundle.report || typeof bundle.report !== 'object') return { ...out, reason: 'malformed_bundle' };
+  const sig = bundle.signature;
+  if (!sig || !sig.signature || !sig.message) return { ...out, reason: 'missing_signature' };
+  const recomputed = createHash('sha256').update(canonicalize(bundle.report), 'utf-8').digest('hex');
+  out.recomputed_sha256 = recomputed;
+  out.hash_ok = recomputed === bundle.report_sha256;
+  if (!out.hash_ok) return { ...out, reason: 'report_hash_mismatch' };
+  const r = bundle.report;
+  const expected = r.period ? {
+    kind: 'quox.kuma.sla.report', report_id: bundle.report_id, org_id: r.org_id,
+    period_from: r.period.from, period_to: r.period.to, generated_at: r.generated_at, report_sha256: recomputed,
+  } : null;
+  out.binding_ok = !!expected && canonicalize(expected) === canonicalize(sig.message);
+  if (!out.binding_ok) return { ...out, reason: 'signature_not_bound_to_report' };
+  if (sig.sig_type !== 'ed25519') return { ...out, reason: `unverifiable_sig_type:${sig.sig_type}` };
+  let key = null; let source = 'none';
+  if (publicKey) { key = keyFromRaw(publicKey); source = 'pinned'; if (!key) return { ...out, reason: 'bad_public_key' }; }
+  else { key = keyFromRaw(sig.public_key); source = 'embedded'; }
+  if (!key) return { ...out, reason: 'no_verification_key' };
+  out.key_source = source;
+  try {
+    out.signature_ok = cryptoVerify(null, Buffer.from(canonicalize(sig.message), 'utf-8'), key, Buffer.from(String(sig.signature), 'base64'));
+  } catch (err) { return { ...out, reason: `verify_error:${err.message}` }; }
+  if (!out.signature_ok) return { ...out, reason: 'signature_mismatch' };
+  out.valid = true;
+  out.trust_anchored = source === 'pinned' && !sig.warning;
+  if (sig.warning) out.warning = sig.warning;
+  return out;
+}
+
+const dur = (s) => {
+  if (s === null || s === undefined) return '-';
+  if (s < 60) return `${s}s`;
+  if (s < 3600) return `${Math.floor(s / 60)}m ${s % 60}s`;
+  return `${Math.floor(s / 3600)}h ${Math.floor((s % 3600) / 60)}m`;
+};
+
+export function registerKumaSla(kuma, { client, auth, jsonOut, guard, KumaFail, pad }) {
+  const slaFail = (verb, ep, err) => {
+    const status = err && (err.status ?? err.statusCode);
+    const msg = (err && err.message) || String(err);
+    const head = `quox kuma ${verb}: ${ep.method} ${ep.path} failed${status ? ` with HTTP ${status}` : ''}`;
+    if (status === 422) return new KumaFail(`${head}: Uptime Kuma is not configured for this org (add the uptime_kuma credential in the vault).`);
+    if (status === 404) return new KumaFail(`${head}: not found in this org.`);
+    return new KumaFail(`${head}: ${msg}`);
+  };
+
+  const sla = kuma
+    .command('sla')
+    .description('Signed, verifiable uptime (SLA) report from Kuma heartbeats; `sla verify <id|file>` checks one')
+    .option('--month <YYYY-MM>', 'calendar month (UTC), e.g. 2026-09')
+    .option('--from <iso>', 'range start (with --to)')
+    .option('--to <iso>', 'range end, exclusive')
+    .option('--out <file>', 'also write the signed bundle here for offline verification')
+    .addHelpText('after', '\nExit codes: 0 ok, 1 error, 2 `sla verify` found the report does NOT verify.')
+    .action(guard('sla', async (options) => {
+      auth.requireAuth();
+      if (!options.month && !(options.from && options.to)) throw new KumaFail('quox kuma sla: give --month YYYY-MM, or both --from and --to.');
+      const ep = resolveEndpoint('mon.kuma.sla.create');
+      const body = options.month ? { month: options.month } : { from: options.from, to: options.to };
+      let data;
+      try { data = await client.post(ep.path, { service: ep.service, body }); } catch (err) { throw slaFail('sla', ep, err); }
+      if (!data || data.success === false || data.error || !data.report || !data.signature) {
+        throw new KumaFail(`quox kuma sla: ${ep.method} ${ep.path} returned an error body: ${(data && data.error) || 'response had no report'}`);
+      }
+      if (options.out) {
+        writeFileSync(options.out, `${JSON.stringify({ report_id: data.report_id, report: data.report, report_sha256: data.report_sha256, signature: data.signature }, null, 2)}\n`);
+      }
+      if (jsonOut()) { console.log(JSON.stringify(data, null, 2)); return; }
+      const r = data.report; const ro = r.rollup;
+      console.log(glyphLine('kuma', 'sla', { result: `${r.period.label} · ${ro.uptime_pct === null ? 'no data' : `${ro.uptime_pct}%`}`, status: ro.uptime_pct === null ? 'warn' : 'ok' }));
+      console.log(`Report ${data.report_id}   sha256 ${data.report_sha256}`);
+      console.log(`Period ${r.period.from} to ${r.period.effective_to}${r.period.partial ? '  (partial: the period has not ended)' : ''}`);
+      const rows = [['MONITOR', 'UPTIME', 'OUTAGES', 'DOWNTIME', 'LONGEST', 'MTTR', 'MAINT', 'GAPS']];
+      for (const m of r.monitors) {
+        rows.push([m.name || m.monitor_id, m.status === 'no_data' ? 'no data' : `${m.uptime_pct}%`, m.outages.count, dur(m.outages.total_downtime_seconds), dur(m.outages.longest_seconds), dur(m.outages.mttr_seconds), m.beats.maintenance, m.coverage.gaps.length]);
+      }
+      console.log(`\n${pad(rows).join('\n')}`);
+      console.log(`\nOrg: ${ro.uptime_pct === null ? 'no data' : `${ro.uptime_pct}%`} over ${ro.monitors_with_data}/${ro.monitors_total} monitors with data; ${ro.outages.count} ${ro.outages.count === 1 ? 'outage' : 'outages'}, ${dur(ro.outages.total_downtime_seconds)} down, MTTR ${dur(ro.outages.mttr_seconds)}.`);
+      if (ro.monitors_no_data) console.log(`${ro.monitors_no_data} ${ro.monitors_no_data === 1 ? 'monitor has' : 'monitors have'} no heartbeats in the period: reported as no data, not as 100%.`);
+      console.log(data.trust_anchored ? 'Signed with the WARD key and witnessed.' : 'WARNING: signed with an ephemeral key (no persistent WARD key on this collector): integrity only, not trust anchored.');
+      console.log(`Verify:  quox kuma sla verify ${options.out || data.report_id}`);
+    }));
+
+  sla
+    .command('list')
+    .description('This org\'s stored SLA reports')
+    .action(guard('sla list', async () => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('mon.kuma.sla.list');
+      let data;
+      try { data = await client.get(ep.path, { service: ep.service }); } catch (err) { throw slaFail('sla list', ep, err); }
+      if (!data || data.success === false || data.error || !Array.isArray(data.reports)) {
+        throw new KumaFail(`quox kuma sla list: ${ep.method} ${ep.path} returned an error body: ${(data && data.error) || 'response had no reports array'}`);
+      }
+      if (jsonOut()) { console.log(JSON.stringify(data, null, 2)); return; }
+      console.log(glyphLine('kuma', 'sla list', { result: `${data.reports.length} ${data.reports.length === 1 ? 'report' : 'reports'}`, status: 'ok' }));
+      if (!data.reports.length) { console.log('No reports yet. Generate one: quox kuma sla --month YYYY-MM'); return; }
+      const rows = [['REPORT', 'PERIOD', 'CREATED', 'SHA256']];
+      for (const r of data.reports) rows.push([r.report_id, r.period, r.created_at, String(r.report_sha256).slice(0, 16)]);
+      console.log(pad(rows).join('\n'));
+    }));
+
+  sla
+    .command('verify')
+    .description('Recompute the report hash and check the signature (id: via the collector, pinned WARD key; file: fully offline)')
+    .argument('<id|file>', 'a report id from `quox kuma sla list`, or a bundle file written by `sla --out`')
+    .option('--public-key <base64>', 'offline: pin this raw 32 byte Ed25519 WARD public key instead of trusting the key in the file')
+    .action(guard('sla verify', async (target, options) => {
+      let result;
+      if (existsSync(target)) {
+        let bundle;
+        try { bundle = JSON.parse(readFileSync(target, 'utf-8')); } catch (err) { throw new KumaFail(`quox kuma sla verify: ${target} is not valid JSON: ${err.message}`); }
+        result = { report_id: bundle && bundle.report_id, mode: 'offline', ...verifyBundleOffline(bundle, { publicKey: options.publicKey }) };
+      } else {
+        auth.requireAuth();
+        const ep = resolveEndpoint('mon.kuma.sla.verify', { id: target });
+        let data;
+        try { data = await client.get(ep.path, { service: ep.service }); } catch (err) { throw slaFail('sla verify', ep, err); }
+        if (!data || data.success === false || data.error || typeof data.valid !== 'boolean') {
+          throw new KumaFail(`quox kuma sla verify: ${ep.method} ${ep.path} returned an error body: ${(data && data.error) || 'response had no verdict'}`);
+        }
+        result = { mode: 'collector', ...data };
+      }
+      if (jsonOut()) { console.log(JSON.stringify(result, null, 2)); } else {
+        console.log(glyphLine('kuma', 'sla verify', { result: result.valid ? `verified (${result.mode})` : `NOT verified: ${result.reason}`, status: result.valid ? 'ok' : 'error' }));
+        console.log(`hash ${result.hash_ok ? 'matches' : 'DOES NOT match'}   binding ${result.binding_ok ? 'ok' : 'FAILED'}   signature ${result.signature_ok ? 'valid' : 'INVALID'}   key ${result.key_source}${result.trust_anchored ? ' (trust anchored)' : ' (not trust anchored)'}`);
+        if (result.valid && !result.trust_anchored) console.log('Integrity holds, but the signer was not checked against the WARD key. Pin it with --public-key, or verify by id through the collector.');
+        if (result.warning) console.log(`Warning: ${result.warning}`);
+      }
+      if (!result.valid) process.exitCode = 2;
+    }));
+}

package/lib/commands/licence.js

@@ -109,9 +109,21 @@ export function registerLicenceCommands(program, ctx, { owner = VENDOR_OWNER } =
     .description('List a plugin band holders; --release-dead frees non-live holders (dry run unless --yes)')
     .argument('<pluginId>', 'plugin id, e.g. notesq')
     .option('--release-dead', 'release holders that are not live orgs (deleted orgs, user: ids)')
+    .option('--release <orgId>', 'release a LIVE org seat (its install of the plugin is disabled); dry run unless --yes')
+    .option('--to <orgId>', 'with --release: hand the freed seat to this org in the same call')
     .option('--yes', 'actually release (otherwise dry run)')
     .action(async (pluginId, opts) => {
       auth.requireAuth();
+      if (opts.to && !opts.release) {
+        const e = new Error('--to only works together with --release <orgId>');
+        e.exitCode = 2;
+        throw e;
+      }
+      if (opts.release && opts.releaseDead) {
+        const e = new Error('use either --release or --release-dead, not both');
+        e.exitCode = 2;
+        throw e;
+      }
       const lep = resolveEndpoint('setup.licensing.seats.list', { pluginId });
       const list = await client.get(lep.path, { service: lep.service });
       let rel = null;
@@ -119,7 +131,15 @@ export function registerLicenceCommands(program, ctx, { owner = VENDOR_OWNER } =
         const rep = resolveEndpoint('setup.licensing.seats.releaseDead', { pluginId });
         rel = await client.post(rep.path, { service: rep.service, body: { dry_run: !opts.yes } });
       }
-      if (json()) { console.log(JSON.stringify({ ...list, release: rel }, null, 2)); return; }
+      let live = null;
+      if (opts.release) {
+        const rp = resolveEndpoint('setup.licensing.seats.release', { pluginId });
+        live = await client.post(rp.path, {
+          service: rp.service,
+          body: { org_id: opts.release, ...(opts.to ? { to_org_id: opts.to } : {}), dry_run: !opts.yes },
+        });
+      }
+      if (json()) { console.log(JSON.stringify({ ...list, release: rel, live_release: live }, null, 2)); return; }
       printInfo(`${pluginId}: ${list.holders.length} holder(s), cap ${list.allowance ?? '(no band)'}`);
       formatOutput({
         columns: ['id', 'name', 'status'],
@@ -133,6 +153,24 @@ export function registerLicenceCommands(program, ctx, { owner = VENDOR_OWNER } =
           printSuccess(`Released ${rel.released.length} non-live holder(s): ${rel.released.join(', ') || '(none)'}`);
         }
       }
+      if (live) {
+        const who = `${live.org_name || live.org_id} (${live.org_id})`;
+        const loses = live.will_disable_install
+          ? `its install of ${pluginId} goes disabled and the plugin stops working there`
+          : `it loses access to ${pluginId} (no enabled install to disable)`;
+        if (live.dry_run) {
+          printInfo(`DRY RUN: would release the ${pluginId} seat held by ${who}: ${loses}.`);
+          if (live.would_assign_to) printInfo(`DRY RUN: the freed seat would go to ${live.would_assign_to}.`);
+          else printInfo(`Seats are first-come: add --to <orgId> to hand the freed seat to a specific org, otherwise the first org to use ${pluginId} takes it.`);
+          printInfo(`Re-run with --yes to release.`);
+        } else if (live.released) {
+          printSuccess(`Released the ${pluginId} seat held by ${who}: ${loses}.`);
+          if (live.assigned_to) printSuccess(`Seat now held by ${live.assigned_to}.`);
+          else if (opts.to) printWarning(`The seat was freed but ${opts.to} could not take it (see the band with: quox licence seats ${pluginId}).`);
+        } else {
+          printWarning(`Nothing released: ${who} no longer holds a ${pluginId} seat.`);
+        }
+      }
     });
 
   if (owner) owner.registerLicenceOwnerCommands(licence, { json, fetchImpl });

package/lib/commands/loop.js

@@ -0,0 +1,291 @@
+/**
+ * Loop commands (QLOOP-LIVE): observe a QLoop loop as it runs.
+ *
+ *   quox loop watch <id> [--spool|--quiet] [--since n]   live, one QuoxGlyph line per event
+ *   quox loop status <id>                                snapshot
+ *   quox loop list                                       running + recent loops (org)
+ *
+ * `watch` reconnects with ?since=<last event_seq> when the stream drops, and exits with the loop's
+ * outcome: 0 completed, 1 failed, 2 halted, 3 interrupted, 4 connection lost for good.
+ * Events come from GET /api/v1/loops/:id/events (design: ~/quoxon-work/qloop-live-design.md).
+ */
+
+import { formatOutput, getOutputFormat } from '../core/output.js';
+import { resolveEndpoint } from '../api/endpoints.js';
+import { loopLine, renderEvent, exitCodeFor, elapsedLabel } from '../core/loopGlyph.js';
+import { QuoxApiError, QuoxAuthError, QuoxUserError } from '../core/errors.js';
+
+const MAX_SILENT_RETRIES = 10;
+const BACKOFF_BASE_MS = 1000;
+const BACKOFF_CAP_MS = 10000;
+
+const sleepReal = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/** SQLite "YYYY-MM-DD HH:MM:SS" (UTC, no zone) or ISO to epoch ms; null if unreadable. */
+function toMs(ts) {
+  if (!ts) return null;
+  const iso = /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$/.test(ts) ? `${ts.replace(' ', 'T')}Z` : ts;
+  const n = Date.parse(iso);
+  return Number.isNaN(n) ? null : n;
+}
+
+const SETTLE_MAX_MS = 90000;
+const SETTLE_POLL_MS = 4000;
+
+/**
+ * After loop.finished: wait (bounded) for the loop's receipts to be WITNESSED by WARD, asking the snapshot
+ * (which re-checks WARD and records what it finds), printing each receipt as it flips. Returns the finished
+ * event with the final witnessed count. A snapshot that cannot be read ends the wait and says so; it never
+ * reads as "0 witnessed".
+ */
+async function settleReceipts({ client, id, finished, write, sleep, ctx, maxMs, signal, seen = new Set() }) {
+  const ep = resolveEndpoint('loops.get', { id: encodeURIComponent(id) });
+  let witnessed = finished.witnessed;
+  write(loopLine('receipts', `${witnessed}/${finished.receipts} witnessed, waiting for WARD to sign the covering tip`, 'busy'));
+  let waited = 0;
+  while (witnessed < finished.receipts && waited < maxMs && !(signal && signal.aborted)) {
+    await sleep(SETTLE_POLL_MS);
+    waited += SETTLE_POLL_MS;
+    let snap;
+    try { snap = await client.get(ep.path, { service: ep.service }); } catch (err) {
+      write(loopLine('receipts', `could not re-check WARD (${oneLineErr(err)}); \`quox loop status ${id}\` will`, 'warn'));
+      break;
+    }
+    const items = (snap && snap.receipts && snap.receipts.items) || [];
+    for (const r of items) {
+      if (r.status === 'WITNESSED' && !seen.has(r.envelope_id)) {
+        seen.add(r.envelope_id);
+        write(loopLine('receipt', `${String(r.envelope_id)} WITNESSED`, 'ok'));
+      }
+    }
+    witnessed = snap && snap.receipts ? snap.receipts.witnessed : witnessed;
+  }
+  if (witnessed < finished.receipts) write(loopLine('receipts', `${finished.receipts - witnessed} still pending after ${Math.round(waited / 1000)}s; they are witnessed when the next tip signs (quox loop status ${id})`, 'warn'));
+  return { ...finished, witnessed };
+}
+
+/**
+ * Follow one loop to its end. Returns the process exit code. Exported for tests.
+ * @param {object} p
+ * @param {object} p.client   cli client (stream)
+ * @param {string} p.id
+ * @param {object} p.snapshot  loops.get result (title, max_turns, started_at)
+ * @param {{spool?: boolean, quiet?: boolean, since?: number, json?: boolean, noSettle?: boolean, settleMs?: number}} p.opts
+ * @param {(line: string) => void} p.write
+ * @param {(ms: number) => Promise<void>} [p.sleep]
+ * @param {AbortSignal} [p.signal]
+ */
+export async function watchLoop({ client, id, snapshot, opts = {}, write, sleep = sleepReal, signal }) {
+  const ep = resolveEndpoint('loops.events', { id: encodeURIComponent(id) });
+  let lastSeq = Number(opts.since) || 0;
+  let startedMs = toMs(snapshot && snapshot.started_at);
+  const ctx = { title: snapshot && snapshot.title, maxTurns: snapshot && snapshot.max_turns, startedMs, spool: !!opts.spool };
+  const showQuiet = new Set(['loop.started', 'loop.observe', 'loop.approval', 'loop.finished']);
+  let failures = 0;
+  let backoff = BACKOFF_BASE_MS;
+  const witnessedShown = new Set(); // receipts already printed from loop.ward events, so the settle phase does not repeat them
+
+  while (!signal || !signal.aborted) {
+    let gotEvent = false;
+    let ended = null;
+    try {
+      for await (const ev of client.stream(ep.path, { service: ep.service, query: { since: lastSeq }, signal })) {
+        // `event: end` frame (an old loop that never logged a finish): no type, a reason
+        if (ev && !ev.type && ev.reason) { ended = ev; break; }
+        if (!ev || typeof ev.type !== 'string' || !ev.type.startsWith('loop.')) continue;
+        gotEvent = true;
+        if (typeof ev.event_seq === 'number') {
+          if (ev.event_seq <= lastSeq) continue; // replay overlap after a reconnect
+          lastSeq = ev.event_seq;
+        }
+        if (ev.type === 'loop.ward' && ev.status === 'WITNESSED') witnessedShown.add(ev.envelope_id);
+        if (!ctx.startedMs && ev.type === 'loop.started') ctx.startedMs = Date.parse(ev.at);
+        if (opts.json) write(JSON.stringify(ev));
+        else if (ev.type !== 'loop.finished' && !(opts.quiet && !showQuiet.has(ev.type))) for (const l of renderEvent(ev, ctx)) write(l);
+        if (ev.type === 'loop.finished') {
+          // the loop is over; its receipts are WITNESSED when WARD's next signed tip covers them (about a minute),
+          // so say so and follow them in rather than printing a final count that is about to be stale
+          const last = (!opts.json && !opts.noSettle && ev.receipts > ev.witnessed)
+            ? await settleReceipts({ client, id, finished: ev, write, sleep, ctx, maxMs: opts.settleMs ?? SETTLE_MAX_MS, signal, seen: witnessedShown })
+            : ev;
+          if (!opts.json) for (const l of renderEvent(last, ctx).slice(-1)) write(l);
+          if (!opts.json && !opts.quiet) write(`  quox loop status ${id}   # receipts and proof`);
+          return exitCodeFor(ev.state);
+        }
+      }
+    } catch (err) {
+      // a refusal is not a dropped connection: 401/403/404 and friends end the watch now with the server's words
+      if (err instanceof QuoxAuthError || (err instanceof QuoxApiError && err.status >= 400 && err.status < 500 && err.status !== 429)) throw err;
+      if (signal && signal.aborted) return 130;
+      failures += 1;
+      if (failures > MAX_SILENT_RETRIES) {
+        write(loopLine('connection lost', `could not reach the loop stream after ${MAX_SILENT_RETRIES} tries; resume with: quox loop watch ${id} --since ${lastSeq}`, 'fail'));
+        return 4;
+      }
+      if (!opts.quiet && !opts.json) write(loopLine('reconnecting', `stream dropped (${oneLineErr(err)}); resuming after event ${lastSeq}`, 'warn'));
+      await sleep(backoff);
+      backoff = Math.min(BACKOFF_CAP_MS, backoff * 2);
+      continue;
+    }
+    if (ended) {
+      write(loopLine(ctx.title || 'loop', `ended without a recorded finish (${ended.status || 'unknown'}): ${ended.reason}`, 'warn'));
+      return ended.status === 'completed' ? 0 : 1;
+    }
+    // stream closed by the server (restart, proxy idle limit) before loop.finished: resume, quietly counting
+    if (gotEvent) { failures = 0; backoff = BACKOFF_BASE_MS; } else failures += 1;
+    if (failures > MAX_SILENT_RETRIES) {
+      write(loopLine('connection lost', `the loop stream keeps closing with nothing new; resume with: quox loop watch ${id} --since ${lastSeq}`, 'fail'));
+      return 4;
+    }
+    await sleep(gotEvent ? 250 : backoff);
+    if (!gotEvent) backoff = Math.min(BACKOFF_CAP_MS, backoff * 2);
+  }
+  return 130;
+}
+
+function oneLineErr(err) {
+  return String((err && err.message) || err).replace(/\s+/g, ' ').slice(0, 80);
+}
+
+export function statusLines(s, { diagnostics = false } = {}) {
+  const lines = [];
+  const turns = `turn ${s.turn}${s.max_turns ? `/${s.max_turns}` : ''}`;
+  lines.push(loopLine(s.title || s.objective_id, `${s.state}, ${turns}`, s.state === 'running' ? 'busy' : (s.state === 'completed' ? 'ok' : (s.state === 'completed_with_errors' || s.state === 'halted' || s.state === 'interrupted' ? 'warn' : (s.finished ? 'fail' : 'busy')))));
+  lines.push(`  id        ${s.objective_id}`);
+  lines.push(`  host      ${s.host || '-'}   backend ${s.backend || '-'}   scope ${(s.scope || []).join(',') || '-'}`);
+  if (s.until) lines.push(`  until     ${s.until}${s.every_seconds ? `   (a turn every ${s.every_seconds}s)` : ''}`);
+  else if (s.every_seconds) lines.push(`  every     ${s.every_seconds}s`);
+  if (s.workspace) lines.push(`  workspace ${s.workspace}`);
+  const r = s.receipts || { total: 0, witnessed: 0 };
+  lines.push(`  receipts  ${r.witnessed}/${r.total} witnessed${s.ward_check_error ? `   (WARD could not be asked just now: ${s.ward_check_error})` : ''}`);
+  if (s.observe) lines.push(`  observe   ${JSON.stringify(s.observe)}`);
+  if (s.reason && s.finished) lines.push(`  reason    ${s.reason}`);
+  if (diagnostics && s.diagnostics) lines.push(`  diagnostics ${s.diagnostics}`);
+  if ((s.last_turns || []).length) {
+    lines.push('  last turns');
+    for (const t of s.last_turns) {
+      const tools = (t.tools || []).map((x) => `${x.tool}${x.denied ? ' denied' : (x.ok ? '' : ' FAILED')}`).join(', ');
+      lines.push(`    ${t.turn}  ${t.summary}${tools ? `  [${tools}]` : ''}  ${t.envelope_id ? `${String(t.envelope_id)} ${t.ward}` : ''}`.trimEnd());
+    }
+  }
+  if (s.started_at) {
+    const st = toMs(s.started_at);
+    const en = toMs(s.completed_at) || Date.now();
+    if (st) lines.push(`  elapsed   ${elapsedLabel(en - st)}`);
+  }
+  return lines;
+}
+
+export function registerLoopCommands(program, ctx) {
+  const { client, auth } = ctx;
+
+  const loop = program
+    .command('loop')
+    .description('Observe QLoop loops as they run (watch, status, list)');
+
+  loop
+    .command('watch')
+    .description('Follow a loop live: one QuoxGlyph line per event, reconnecting on drop. Exits 0 completed, 1 failed, 2 halted, 3 interrupted, 4 connection lost')
+    .argument('<id>', 'loop (objective) ID')
+    .option('--spool', 'also stream the loop\'s output text under each turn line')
+    .option('--quiet', 'only the headline lines: started, observe points (milestones, errors, approvals, stalls), the final line')
+    .option('--since <n>', 'resume after this event_seq (printed by a previous watch; default: from the start)')
+    .option('--no-wait-receipts', 'print the final line as soon as the loop ends instead of waiting (up to 90s) for its receipts to be witnessed')
+    .action(async (id, options) => {
+      auth.requireAuth();
+      if (options.spool && options.quiet) throw new QuoxUserError('--spool and --quiet cancel each other out; pick one.');
+      const since = options.since !== undefined ? Number(options.since) : 0;
+      if (!Number.isInteger(since) || since < 0) throw new QuoxUserError('--since needs a whole number event_seq (0 or more).');
+      const snap = resolveEndpoint('loops.get', { id: encodeURIComponent(id) });
+      const snapshot = await client.get(snap.path, { service: snap.service });
+      const ac = new AbortController();
+      const onSigint = () => ac.abort();
+      process.once('SIGINT', onSigint);
+      let code;
+      try {
+        code = await watchLoop({
+          client, id, snapshot,
+          opts: { spool: options.spool, quiet: options.quiet, since, json: getOutputFormat() === 'json', noSettle: options.waitReceipts === false },
+          write: (l) => console.log(l),
+          signal: ac.signal,
+        });
+      } finally {
+        process.removeListener('SIGINT', onSigint);
+      }
+      if (code === 130) console.log(`  stopped watching; the loop keeps running. Resume: quox loop watch ${id}`);
+      if (code) process.exitCode = code;
+    });
+
+  loop
+    .command('status')
+    .description('Snapshot of a loop: state, turn n/max, host, backend, scope, last turns, receipts')
+    .argument('<id>', 'loop (objective) ID')
+    .option('--diagnostics', 'also show the raw diagnostics behind the finish reason (hidden by default)')
+    .action(async (id, options = {}) => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('loops.get', { id: encodeURIComponent(id) });
+      const data = await client.get(ep.path, { service: ep.service });
+      if (getOutputFormat() === 'json') {
+        const out = options.diagnostics || !data || typeof data !== 'object' ? data : { ...data, diagnostics: undefined };
+        console.log(JSON.stringify(out, null, 2)); return;
+      }
+      for (const l of statusLines(data, { diagnostics: !!options.diagnostics })) console.log(l);
+    });
+
+  loop
+    .command('list')
+    .description('Running and recent loops in this org')
+    .option('--limit <n>', 'max results', '20')
+    .action(async (options) => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('loops.list');
+      const data = await client.get(ep.path, { service: ep.service, query: { limit: options.limit } });
+      formatOutput({
+        columns: ['objective_id', 'title', 'state', 'turn', 'backend', 'created_at'],
+        headers: ['ID', 'Title', 'State', 'Turn', 'Backend', 'Created'],
+        rows: (data.loops || []).map((l) => ({ ...l, turn: l.max_turns ? `${l.turn}/${l.max_turns}` : String(l.turn) })),
+      }, { format: program.opts().output, emptyMessage: 'No loops in this org yet.' });
+    });
+
+  // SECURITY containment (2026-10-26): which orgs may run loops in-server (core-local) while a security review is open.
+  // Instance-admin only (the server enforces it; a non-admin gets a 403). Stored server-side in instance_config.
+  const coreLocal = loop
+    .command('core-local')
+    .description('Instance admin: which workspaces may run loops inside the server (limited while a security review is open)');
+
+  coreLocal
+    .command('list')
+    .description('Show the workspaces allowed to run loops inside the server')
+    .action(async () => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('admin.instance.core_local_orgs.list');
+      const r = await client.get(ep.path, { service: ep.service });
+      if (getOutputFormat() === 'json') { console.log(JSON.stringify(r, null, 2)); return; }
+      if (!(r.orgs || []).length) { console.log('No workspace may run loops inside the server (loops need a dev server).'); return; }
+      for (const o of r.orgs) console.log(o);
+    });
+
+  coreLocal
+    .command('add')
+    .description('Allow a workspace to run loops inside the server (instance admin)')
+    .argument('<orgId>', 'workspace (org) id')
+    .action(async (orgId) => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('admin.instance.core_local_orgs.add');
+      const r = await client.post(ep.path, { service: ep.service, body: { org_id: orgId } });
+      if (getOutputFormat() === 'json') { console.log(JSON.stringify(r, null, 2)); return; }
+      console.log(`Added ${orgId}. ${(r.orgs || []).length} workspace(s) may now run loops inside the server.`);
+    });
+
+  coreLocal
+    .command('remove')
+    .description('Stop a workspace running loops inside the server (instance admin)')
+    .argument('<orgId>', 'workspace (org) id')
+    .action(async (orgId) => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('admin.instance.core_local_orgs.remove', { orgId: encodeURIComponent(orgId) });
+      const r = await client.delete(ep.path, { service: ep.service });
+      if (getOutputFormat() === 'json') { console.log(JSON.stringify(r, null, 2)); return; }
+      console.log(`Removed ${orgId}. ${(r.orgs || []).length} workspace(s) may still run loops inside the server.`);
+    });
+}

package/lib/commands/objective.js

@@ -12,9 +12,10 @@
  *                        live loop gate: budget, risk, interval).
  */
 
-import { formatOutput, formatDetail, printSuccess } from '../core/output.js';
+import { formatOutput, formatDetail, printSuccess, printWarning } from '../core/output.js';
 import { parseJSON } from './helpers.js';
 import { resolveEndpoint } from '../api/endpoints.js';
+import { QuoxUserError } from '../core/errors.js';
 
 /**
  * Format a timestamp as a short relative-time string ("5m ago").
@@ -75,6 +76,13 @@ export function registerObjectiveCommands(program, ctx) {
     .option('--loop-backend <backend>', 'claude|codex, loop mode only (default claude)')
     .option('--capability-scope <tools>', 'comma-separated tool names the loop may call (repeatable); empty scope escalates every tool', collectScope)
     .option('--system-prompt <text>', 'per-objective system prompt, carried verbatim to the run')
+    .option('--observe-every <n>', 'loop mode: emit a status line every N turns while it runs (watch with `quox loop watch <id>`)')
+    .option('--observe-milestones', 'loop mode: lift MILESTONE: lines from the loop output into headline updates')
+    .option('--until <text>', 'loop mode: the done criterion in plain words. The loop runs turn after turn until the model ends a turn with "DONE: <evidence>" (or --max-iterations / --max-duration-ms stops it)')
+    .option('--every-seconds <n>', 'loop mode: the runner waits N seconds (30-3600) between turns; the model is never asked to sleep')
+    .option('--workspace <name>', 'loop mode: name the durable workspace dir, so a follow-up loop can work in the same files (default: one per loop)')
+    .option('--done-file <path>', 'loop mode: DONE only counts once this file exists in the loop workspace; the runner checks it itself (repeatable)', (v, prev = []) => prev.concat(v))
+    .option('--done-command <cmd>', 'loop mode: DONE only counts once this command exits 0 in the loop workspace; must be one the loop scope lets it run (repeatable)', (v, prev = []) => prev.concat(v))
     .action(async (goal, options) => {
       auth.requireAuth();
       const body = { title: goal };
@@ -98,6 +106,28 @@ export function registerObjectiveCommands(program, ctx) {
         if (scope.length > 0) body.capability_scope = scope;
       }
       if (options.systemPrompt) body.system_prompt = options.systemPrompt;
+      if (options.until) body.until = options.until;
+      if (options.everySeconds !== undefined) {
+        const n = Number(options.everySeconds);
+        if (!Number.isInteger(n) || n < 30 || n > 3600) throw new QuoxUserError('--every-seconds needs a whole number of seconds between 30 and 3600.');
+        body.every_seconds = n;
+      }
+      if (options.workspace) body.workspace = options.workspace;
+      const doneChecks = [
+        ...(options.doneFile || []).map((path) => ({ kind: 'file_exists', path })),
+        ...(options.doneCommand || []).map((command) => ({ kind: 'command_ok', command })),
+      ];
+      if (doneChecks.length) body.done_check = doneChecks;
+      if (options.observeEvery !== undefined || options.observeMilestones) {
+        const observe = {};
+        if (options.observeEvery !== undefined) {
+          const n = Number(options.observeEvery);
+          if (!Number.isInteger(n) || n < 1) throw new QuoxUserError('--observe-every needs a whole number of turns (1 or more).');
+          observe.every_turns = n;
+        }
+        if (options.observeMilestones) observe.milestones = true;
+        body.observe = observe;
+      }
 
       const data = await client.post(resolveEndpoint('objectives.create').path, { service: resolveEndpoint('objectives.create').service, body });
       printSuccess(`Objective created: ${data.id}`);
@@ -155,6 +185,17 @@ export function registerObjectiveCommands(program, ctx) {
       printSuccess(`Objective ${id}: ${data.status}${data.message ? ` — ${data.message}` : ''}`);
     });
 
+  objective
+    .command('retry')
+    .description('Retry a failed or partially completed objective: failed and skipped steps are reset and the plan comes back for approval')
+    .argument('<id>', 'objective ID')
+    .action(async (id) => {
+      auth.requireAuth();
+      const ep = resolveEndpoint('objectives.retry', { id });
+      const data = await client.post(ep.path, { service: ep.service });
+      printSuccess(`Objective ${id}: ${data.steps_reset ?? 0} step(s) reset, plan is back for approval. Run \`quox objective approve ${id}\` to run them again.`);
+    });
+
   objective
     .command('checkpoints')
     .description('List pending HITL checkpoints across the org (approval queue)')
@@ -191,6 +232,12 @@ export function registerObjectiveCommands(program, ctx) {
       if (options.notes) body.notes = options.notes;
       const params = { id: objectiveId, checkpointId };
       const data = await client.post(resolveEndpoint('objectives.checkpoint-approve', params).path, { service: resolveEndpoint('objectives.checkpoint-approve', params).service, body });
+      // LOOPQ-HALT: "approved" is not "running". When the server says the loop did not start, say so in words.
+      if (data.started === false) {
+        printWarning(`Checkpoint ${checkpointId}: ${data.status}, but the objective did NOT start (objective ${data.objective_status || 'pending'}).`);
+        if (data.hint) printWarning(data.hint);
+        return;
+      }
       printSuccess(`Checkpoint ${checkpointId}: ${data.status}${data.objective_status ? ` (objective ${data.objective_status})` : ''}`);
     });