session-orchestrator 4.0.0 → 4.1.0

package/scripts/lib/telemetry-flush-health-banner.mjs

@@ -0,0 +1,211 @@
+/**
+ * telemetry-flush-health-banner.mjs — #1255
+ *
+ * Surfaces the ONE telemetry-flush outcome nobody could see: a flush the
+ * sandbox guard refused.
+ *
+ * ## Why this module exists
+ *
+ * `scripts/lib/telemetry/sync.mjs` fails CLOSED since 4.0.0 — when its
+ * environment probe cannot complete its checks it returns
+ * `{ sandbox: true, reason: 'sandbox:probe-failed' }` and sends nothing. That
+ * is the right default, and `sync.mjs` stays deliberately SILENT about it (it
+ * runs inside the SessionEnd teardown budget, where a banner has no reader).
+ *
+ * The refusal therefore reaches exactly one sink: `hooks/on-session-end.mjs`
+ * emits `orchestrator.telemetry.flush` with `{ outcome, reason }` into
+ * `.orchestrator/metrics/events.jsonl`. Measured 2026-09-07 (W1-D6 census):
+ * NO consumer read that reason back out — a repo could refuse every flush for
+ * weeks and the only visible difference from a healthy repo was an absence.
+ * That is `.claude/rules/host-resources.md` § HR-105 ("a rule you cannot
+ * falsify is not a rule") applied to the flush path.
+ *
+ * This probe closes the loop on the NEXT session start, which is the first
+ * moment a human is actually looking: it reads the LAST flush record and warns
+ * when that record is a sandbox refusal. Newest-wins by construction — a later
+ * successful flush silently clears the warning, so the banner reports the
+ * CURRENT state of the channel, never its history.
+ *
+ * Never throws. Never mutates input. No network.
+ *
+ * @module scripts/lib/telemetry-flush-health-banner
+ */
+
+import { existsSync, openSync, readSync, fstatSync, closeSync } from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Bytes of `events.jsonl` read from the END of the file.
+ *
+ * The ledger is append-only and grows without bound (33k+ records in this repo
+ * alone), so a full `readFileSync` on a SessionStart hook's critical path is
+ * the wrong shape — this probe shares a 2s budget with ~18 siblings.
+ *
+ * NAMED CEILING (BV-004): a flush record older than the last 64 KB of the
+ * ledger is invisible to this probe and reads as "no flush recorded" → `null`
+ * (silent). At the observed record width (~120-400 bytes) that window holds
+ * roughly 150-500 events, and a flush is emitted once per session close — so
+ * the window covers the last flush unless ~150+ events landed after it without
+ * one, which cannot happen inside a single session's teardown.
+ * REVISIT TRIGGER: if the per-session event volume ever exceeds ~150 records
+ * between two session ends, raise this or index the ledger — do not silently
+ * accept the truncation.
+ */
+export const TAIL_BYTES = 64 * 1024;
+
+/** The event name `hooks/on-session-end.mjs` emits for every flush attempt. */
+const FLUSH_EVENT = 'orchestrator.telemetry.flush';
+
+/**
+ * Bound an untrusted ledger `reason` before it is interpolated into a terminal
+ * banner: strip C0/DEL control bytes (a record could carry an ANSI escape) and
+ * cap the length. Mirrors the sibling bound in
+ * `scripts/lib/session-start-probes.mjs` (`.slice(0, 200)` on a probe error).
+ *
+ * @param {unknown} reason
+ * @returns {string}
+ */
+function sanitizeReason(reason) {
+  // eslint-disable-next-line no-control-regex -- stripping control bytes IS the job
+  return String(reason).replace(/[\x00-\x1f\x7f]/g, '').slice(0, 120);
+}
+
+/**
+ * Read the last `TAIL_BYTES` of a file as UTF-8.
+ *
+ * Returns a DISCRIMINATED result rather than `string|null`, because "the
+ * ledger could not be read" and "the ledger holds nothing alarming" are the
+ * two states this probe exists to keep apart (HR-105): collapsing an EACCES
+ * onto `null` renders an unreadable channel as a healthy one.
+ *
+ * @param {string} file
+ * @returns {{text: string} | {missing: true} | {error: string}}
+ */
+function readTail(file) {
+  let fd;
+  try {
+    fd = openSync(file, 'r');
+    const { size } = fstatSync(fd);
+    const length = Math.min(size, TAIL_BYTES);
+    const start = size - length;
+    const buf = Buffer.allocUnsafe(length);
+    let read = 0;
+    while (read < length) {
+      const n = readSync(fd, buf, read, length - read, start + read);
+      if (n <= 0) break;
+      read += n;
+    }
+    return { text: buf.subarray(0, read).toString('utf8') };
+  } catch (err) {
+    if (err?.code === 'ENOENT') return { missing: true };
+    return { error: typeof err?.code === 'string' ? err.code : 'EUNKNOWN' };
+  } finally {
+    if (fd !== undefined) {
+      try {
+        closeSync(fd);
+      } catch {
+        /* best effort */
+      }
+    }
+  }
+}
+
+/**
+ * Classify the telemetry-flush channel from the LAST recorded flush event.
+ *
+ * Scans the ledger tail BACKWARDS and stops at the first
+ * `orchestrator.telemetry.flush` record it can parse — newest wins, so an
+ * older refusal followed by a successful flush produces no banner.
+ *
+ * Returns `null` (silent) when: `repoRoot` is not a string, the ledger is
+ * ABSENT, no flush record sits in the tail window, or the newest flush
+ * record's `reason` does not start with `sandbox:`. A malformed line is
+ * SKIPPED (it is not a flush record we can read), never a throw — the ledger's
+ * last line is routinely a partial write when another process is appending.
+ *
+ * An UNREADABLE ledger (EACCES/EIO/…, i.e. anything other than "not there") is
+ * NOT silent: it returns a `ledger-unreadable` warning. "The guard state
+ * cannot be confirmed" must not display like "the last flush was fine" — that
+ * collapse is the exact HR-105 defect this module's header cites as its reason
+ * to exist, one layer down.
+ *
+ * This probe reports the LAST outcome regardless of its age (no clock seam):
+ * a refusal does not expire on its own — only a later successful flush clears
+ * it.
+ *
+ * @param {{repoRoot: string}} opts
+ * @returns {null | {severity: 'warn', reason: string, message: string}}
+ */
+export function checkTelemetryFlushHealth({ repoRoot } = {}) {
+  try {
+    if (!repoRoot || typeof repoRoot !== 'string') return null;
+
+    const file = path.join(repoRoot, '.orchestrator', 'metrics', 'events.jsonl');
+    if (!existsSync(file)) return null;
+
+    const tail = readTail(file);
+    if (tail.missing) return null;
+    if (tail.error) {
+      return {
+        severity: 'warn',
+        reason: 'ledger-unreadable',
+        message:
+          '⚠ Telemetry: flush-health unknown — .orchestrator/metrics/events.jsonl ' +
+          `could not be read (${sanitizeReason(tail.error)}); ` +
+          'the sandbox guard state cannot be confirmed.',
+      };
+    }
+    if (!tail.text) return null;
+
+    const lines = tail.text.split('\n');
+    for (let i = lines.length - 1; i >= 0; i -= 1) {
+      const line = lines[i].trim();
+      if (!line) continue;
+
+      let record;
+      try {
+        record = JSON.parse(line);
+      } catch {
+        // Malformed (or a tail-truncated first line) — not readable as a flush
+        // record, so it cannot be the newest one. Keep scanning backwards.
+        continue;
+      }
+      if (!record || typeof record !== 'object') continue;
+      if (record.event !== FLUSH_EVENT) continue;
+
+      // First flush record found scanning backwards = the newest one.
+      const rawReason = record.reason;
+      if (typeof rawReason !== 'string' || !rawReason.startsWith('sandbox:')) return null;
+
+      // The ledger is an untrusted string source for banner purposes: bound the
+      // reason on BOTH surfaces (the field a consumer may render itself, and
+      // the message we render) rather than only on the one we happen to own.
+      const reason = sanitizeReason(rawReason);
+
+      return {
+        severity: 'warn',
+        reason,
+        message:
+          `⚠ Telemetry: last flush refused by the sandbox guard (${reason}) — ` +
+          'the guard could not complete its checks and failed closed; no ping was sent. ' +
+          'See docs/telemetry.md § Sandbox guard.',
+      };
+    }
+
+    return null;
+  } catch {
+    // Defensive catch-all — a banner probe must never throw (fail-open).
+    return null;
+  }
+}
+
+/**
+ * Convenience renderer: the banner message string, or `''` when silent.
+ *
+ * @param {{repoRoot: string}} opts
+ * @returns {string}
+ */
+export function renderBanner({ repoRoot } = {}) {
+  const result = checkTelemetryFlushHealth({ repoRoot });
+  return result ? result.message : '';
+}

package/scripts/lib/validate/check-codex-skills.mjs

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+// Independently inspect Codex discovery artifacts; generator freshness alone
+// cannot detect a generator that consistently emits an invalid skill contract.
+import { existsSync, readFileSync, readdirSync, realpathSync } from 'node:fs';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import yaml from 'js-yaml';
+
+const SKILL_ROOT = '.codex-plugin/skills';
+const FRONTMATTER_KEYS = new Set([
+  'name', 'description', 'license', 'compatibility', 'metadata', 'allowed-tools',
+]);
+
+/** @param {unknown} value */
+function isRecord(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+/**
+ * Read emitted artifacts against their source contract without importing the
+ * generator or comparing its prose. Paths are resolved as installed files.
+ * @param {string} pluginRoot
+ * @returns {{skills: string[], violations: string[]}}
+ */
+export function validateCodexSkills(pluginRoot) {
+  const root = realpathSync(pluginRoot);
+  const violations = [];
+  const sources = new Map();
+  const skills = [];
+  const label = (file) => relative(root, file).split(sep).join('/');
+
+  function text(file) {
+    try {
+      const actual = realpathSync(file);
+      if (!actual.startsWith(`${root}${sep}`)) {
+        violations.push(`${label(file)}: outside the plugin package`);
+        return null;
+      }
+      return readFileSync(file, 'utf8');
+    } catch (error) {
+      violations.push(`${label(file)}: missing or unreadable (${error.code ?? error.message})`);
+      return null;
+    }
+  }
+
+  function mapping(raw, file) {
+    if (raw === null) return null;
+    try {
+      const parsed = yaml.load(raw, { schema: yaml.CORE_SCHEMA });
+      if (!isRecord(parsed)) throw new Error('expected a YAML mapping');
+      return parsed;
+    } catch (error) {
+      violations.push(`${label(file)}: invalid YAML (${error.message.split('\n')[0]})`);
+      return null;
+    }
+  }
+
+  function frontmatter(file) {
+    const raw = text(file);
+    if (raw === null) return null;
+    const match = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/.exec(raw);
+    if (!match) {
+      violations.push(`${label(file)}: missing YAML frontmatter`);
+      return null;
+    }
+    const fm = mapping(match[1], file);
+    return fm && { fm, body: raw.slice(match[0].length) };
+  }
+
+  // Codex's Agent Plugins loader fixes skills to ./skills; its native overlay
+  // cannot override skills or version. A valid native manifest alone is not
+  // enough when a standard root manifest intercepts discovery (CLI 0.153.3/4).
+  const rootManifestFile = join(root, 'plugin.json');
+  if (existsSync(rootManifestFile)) {
+    try {
+      const manifest = JSON.parse(text(rootManifestFile));
+      if (typeof manifest?.$schema === 'string' && manifest.$schema.startsWith('https://agent-plugins.org/schemas/')) {
+        violations.push('Root Agent Plugins manifest overrides native Codex skills and cache version; use the native harness manifests');
+      }
+    } catch (error) {
+      violations.push(`Root plugin manifest: invalid JSON (${error.message})`);
+    }
+  }
+
+  const manifestFile = join(root, '.codex-plugin/plugin.json');
+  try {
+    const raw = text(manifestFile);
+    const manifest = raw === null ? null : JSON.parse(raw);
+    if (manifest?.skills !== `./${SKILL_ROOT}/`) {
+      violations.push(`Codex manifest skills must register only ./${SKILL_ROOT}/`);
+    }
+    if (!Array.isArray(manifest?.commands) || manifest.commands.length !== 0) {
+      violations.push('Codex manifest commands must be an explicit empty array to disable automatic command migration');
+    }
+  } catch (error) {
+    violations.push(`Codex manifest: invalid JSON (${error.message})`);
+  }
+
+  // Skill first, command second: a command owns the public entry on overlap.
+  for (const kind of ['skill', 'command']) {
+    const directory = join(root, kind === 'skill' ? 'skills' : 'commands');
+    if (!existsSync(directory)) {
+      violations.push(`${label(directory)}: canonical source directory is missing`);
+      continue;
+    }
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      const name = kind === 'command' ? entry.name.replace(/\.md$/, '') : entry.name;
+      if (name.startsWith('.') || name.startsWith('_')) continue;
+      if (kind === 'command' && !entry.name.endsWith('.md')) continue;
+      const file = kind === 'command' ? join(directory, entry.name) : join(directory, name, 'SKILL.md');
+      if (kind === 'skill' && !existsSync(file)) continue;
+      const source = frontmatter(file);
+      sources.set(name, { kind, file, fm: source?.fm });
+    }
+  }
+  if (sources.size === 0) violations.push('No canonical commands or skills found');
+
+  const generated = join(root, SKILL_ROOT);
+  const actualNames = existsSync(generated)
+    ? readdirSync(generated, { withFileTypes: true })
+      .filter((entry) => (entry.isDirectory() || entry.isSymbolicLink()) && existsSync(join(generated, entry.name, 'SKILL.md')))
+      .map((entry) => entry.name)
+    : [];
+  for (const name of [...new Set([...sources.keys(), ...actualNames])].sort()) {
+    const source = sources.get(name);
+    if (!source) {
+      violations.push(`${SKILL_ROOT}/${name}: no canonical source`);
+      continue;
+    }
+    const file = join(generated, name, 'SKILL.md');
+    const artifact = frontmatter(file);
+    if (!artifact) continue;
+    skills.push(name);
+    const { fm, body } = artifact;
+    if (fm.name !== name) violations.push(`${label(file)}: name must equal ${name}`);
+    if (typeof fm.description !== 'string' || !fm.description.trim() || fm.description.length > 1024) {
+      violations.push(`${label(file)}: description must be a non-empty string of at most 1024 characters`);
+    }
+    for (const key of Object.keys(fm)) {
+      if (!FRONTMATTER_KEYS.has(key)) violations.push(`${label(file)}: unsupported frontmatter key ${key}`);
+    }
+    if (fm.metadata !== undefined && (!isRecord(fm.metadata) || Object.values(fm.metadata).some((value) => typeof value !== 'string'))) {
+      violations.push(`${label(file)}: metadata must contain string values`);
+    }
+    const links = [...body.matchAll(/\]\(([^\s)]+)\)/g)].map((match) => match[1]);
+    if (!links.some((target) => !target.includes(':') && !target.startsWith('/') && resolve(dirname(file), target) === source.file)) {
+      violations.push(`${label(file)}: missing canonical link to ${label(source.file)}`);
+    }
+    if (source.kind === 'command' && source.fm) {
+      const disabled = source.fm['disable-model-invocation'];
+      if (disabled !== undefined && typeof disabled !== 'boolean') {
+        violations.push(`${label(source.file)}: disable-model-invocation must be boolean`);
+      }
+      const policyFile = join(generated, name, 'agents/openai.yaml');
+      const sidecar = mapping(text(policyFile), policyFile);
+      if (sidecar && (typeof sidecar.policy?.allow_implicit_invocation !== 'boolean'
+        || sidecar.policy.allow_implicit_invocation !== (disabled !== true))) {
+        violations.push(`${label(policyFile)}: allow_implicit_invocation must preserve the command's boolean policy`);
+      }
+    }
+  }
+  return { skills, violations };
+}
+
+async function main(pluginRoot) {
+  if (!pluginRoot) throw new Error('Usage: check-codex-skills.mjs <plugin-root>');
+  console.log('--- Check: Codex command skills and generated freshness ---');
+  const { skills, violations } = validateCodexSkills(pluginRoot);
+  if (violations.length === 0) {
+    // Invoke the function directly: a child CLI that silently skips its entry
+    // point must never turn a missing freshness comparison into success.
+    const { generateCodexSurface } = await import('../../generate-codex-skills.mjs');
+    const fresh = generateCodexSurface({ pluginRoot, check: true });
+    if (!fresh.ok || !Array.isArray(fresh.files) || fresh.files.length === 0) {
+      violations.push(...(fresh.drift ?? []), ...(fresh.errors ?? []));
+      if (violations.length === 0) violations.push('Codex generator did not report a successful artifact comparison');
+    }
+  }
+  for (const violation of violations) console.log(`  FAIL: ${violation}`);
+  if (violations.length === 0) console.log(`  PASS: ${skills.length} unique Codex skills, command policies and generated files in sync`);
+  console.log(`Results: ${violations.length === 0 ? 1 : 0} passed, ${violations.length} failed`);
+  if (violations.length) console.log('Remedy: node scripts/generate-codex-skills.mjs');
+  process.exitCode = violations.length ? 1 : 0;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(realpathSync(process.argv[1])).href) {
+  main(process.argv[2]).catch((error) => {
+    console.error(`FAIL: Codex skills check: ${error.message}`);
+    process.exitCode = 1;
+  });
+}