session-orchestrator 3.16.0

package/scripts/lib/eval/session-resolve.mjs

@@ -0,0 +1,137 @@
+/**
+ * eval/session-resolve.mjs — deterministic session selection + time-window
+ * attribution helpers for the aiat-llm-eval engine (Epic #803, S3).
+ *
+ * The engine scores ONE completed orchestrator session. Two questions must be
+ * answered deterministically and honestly before any dimension is graded:
+ *
+ *   1. WHICH session? — resolveSession() implements the operator-approved
+ *      cascade (Decision #2, revised #822): explicit id > newest record (source
+ *      order) that is either status:'completed' or non-abandoned with completed
+ *      work. Abandoned records are ALWAYS skipped.
+ *
+ *   2. WHOSE events? — quality_gate events in events.jsonl carry NO session_id,
+ *      so they are attributed to a session by its wall-clock window
+ *      [started_at, completed_at] (Decision #1: attribution = time-window).
+ *      findPeerOverlap() detects when that window overlaps ANY other session's
+ *      window — a contaminated window means gate-attribution is unsafe and the
+ *      affected dimensions downgrade to `cannot-determine` (never guess).
+ *
+ * Pure module: no clock reads, no I/O — it operates on already-parsed record
+ * arrays. Determinism is load-bearing for the CLI `--verify` re-evaluation path.
+ */
+
+/** Thrown by resolveSession when no eligible session can be selected. */
+export class SessionResolutionError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = 'SessionResolutionError';
+  }
+}
+
+function isPlainObject(v) {
+  return v !== null && typeof v === 'object' && !Array.isArray(v);
+}
+
+/**
+ * Resolve the session to evaluate.
+ *
+ * @param {object[]} records — parsed sessions.jsonl records, in source order.
+ * @param {string} [sessionId] — explicit session_id; when absent the cascade runs.
+ * @returns {{ record: object, resolvedVia: 'explicit'|'cascade-completed'|'cascade-fallback' }}
+ * @throws {SessionResolutionError} when nothing eligible is found.
+ */
+export function resolveSession(records, sessionId) {
+  if (!Array.isArray(records) || records.length === 0) {
+    throw new SessionResolutionError('no session records found in sessions.jsonl');
+  }
+
+  // Explicit selection: the LAST record carrying this session_id (records may be
+  // rewritten/backfilled across a session's life; the latest is authoritative).
+  if (sessionId) {
+    let match = null;
+    for (const r of records) {
+      if (isPlainObject(r) && r.session_id === sessionId) match = r;
+    }
+    if (!match) {
+      throw new SessionResolutionError(`session not found: ${sessionId}`);
+    }
+    return { record: match, resolvedVia: 'explicit' };
+  }
+
+  // Cascade (#822 — single backward scan, newest-wins): walk records from
+  // newest to oldest and return the FIRST one that qualifies via either tier,
+  // rather than exhausting tier (a) — status:'completed' — over the WHOLE
+  // array before ever considering tier (b). status:'completed' is a sparse
+  // legacy field (absent from every record written after 2026-07-04 by the
+  // current Phase-3.7 writer), so exhausting it first could select an
+  // arbitrarily old record and skip many newer valid ones. Both resolvedVia
+  // labels are preserved for downstream callers/tests.
+  for (let i = records.length - 1; i >= 0; i--) {
+    const r = records[i];
+    if (!isPlainObject(r)) continue;
+    if (r.status === 'abandoned') continue;
+    if (r.status === 'completed') {
+      return { record: r, resolvedVia: 'cascade-completed' };
+    }
+    // status may be absent (absent !== 'abandoned') — qualifies when it
+    // otherwise shows evidence of completed work.
+    if (!r.completed_at) continue;
+    const complete = isPlainObject(r.agent_summary) ? r.agent_summary.complete ?? 0 : 0;
+    const rate = isPlainObject(r.effectiveness) ? r.effectiveness.completion_rate : undefined;
+    if (complete > 0 || (rate !== undefined && rate !== null)) {
+      return { record: r, resolvedVia: 'cascade-fallback' };
+    }
+  }
+
+  throw new SessionResolutionError(
+    'no completed session found (all records abandoned or without evidence of completed work)',
+  );
+}
+
+/**
+ * Compute the wall-clock window [start, end] (epoch ms) for a session record.
+ * Returns null when either boundary is missing / unparseable — the caller then
+ * treats window-attributed dimensions as unmeasurable rather than guessing.
+ *
+ * @param {object} record
+ * @returns {{ start: number, end: number } | null}
+ */
+export function computeWindow(record) {
+  if (!isPlainObject(record)) return null;
+  const start = Date.parse(record.started_at);
+  const end = Date.parse(record.completed_at);
+  if (Number.isNaN(start) || Number.isNaN(end)) return null;
+  return { start, end };
+}
+
+/**
+ * Detect peer sessions whose window overlaps the resolved session's window.
+ *
+ * Overlap uses STRICT inequality (a.start < b.end && b.start < a.end) so two
+ * back-to-back sessions that merely touch at a shared boundary are NOT counted
+ * as overlapping. Records sharing the resolved session_id (duplicate/backfill
+ * rewrites of the SAME session) are excluded, as are records without a valid
+ * window.
+ *
+ * @param {object[]} records — all sessions.jsonl records.
+ * @param {object} resolved — the resolved session record.
+ * @returns {{ count: number, peers: string[] }} unique overlapping session_ids.
+ */
+export function findPeerOverlap(records, resolved) {
+  const win = computeWindow(resolved);
+  if (!win || !Array.isArray(records)) return { count: 0, peers: [] };
+
+  const peers = new Set();
+  for (const r of records) {
+    if (!isPlainObject(r)) continue;
+    if (r.session_id === resolved.session_id) continue;
+    const w = computeWindow(r);
+    if (!w) continue;
+    if (win.start < w.end && w.start < win.end) {
+      peers.add(typeof r.session_id === 'string' ? r.session_id : '(unknown)');
+    }
+  }
+  const list = [...peers];
+  return { count: list.length, peers: list };
+}

package/scripts/lib/eval/sink.mjs

@@ -0,0 +1,77 @@
+/**
+ * eval/sink.mjs — append-only sink + reader for the eval journal
+ * (.orchestrator/metrics/eval.jsonl), the Single Source of Truth for the
+ * aiat-llm-eval standard (Epic #803, S2). The HTML report is a derived view;
+ * this journal is authoritative.
+ *
+ * Design (mirrors scripts/harness-audit.mjs appendAuditRecord): a metrics append
+ * MUST NEVER break its caller. The session-end eval phase is advisory and must
+ * never block /close, so appendEvalRecord swallows BOTH validation and
+ * filesystem errors — it emits a stderr WARN and returns a result object, and
+ * NEVER throws. Callers that want hard validation call validateEvalRecord
+ * (from ./schema.mjs) directly.
+ *
+ * Read path reuses readJsonlFile from scripts/lib/io.mjs (missing file → [],
+ * malformed lines skipped) — never throws.
+ */
+
+import { mkdirSync, appendFileSync } from 'node:fs';
+import path from 'node:path';
+import { validateEvalRecord, normalizeEvalRecord } from './schema.mjs';
+import { readJsonlFile } from '../io.mjs';
+
+/** Default repo-relative path for the eval journal. */
+export const DEFAULT_EVAL_JSONL_PATH = '.orchestrator/metrics/eval.jsonl';
+
+/**
+ * Append a single session-eval record to the eval journal. NEVER throws.
+ *
+ * Steps:
+ *   1. validateEvalRecord — stamps schema_version, enforces the full contract
+ *      (incl. the no-global-score rule). On failure: stderr WARN + return
+ *      { ok:false, reason:'validation', error }. The malformed record is NOT written.
+ *   2. Serialize the validated record to JSON + newline.
+ *   3. mkdirSync(dirname, recursive) + appendFileSync (append-only). On FS
+ *      failure: stderr WARN + return { ok:false, reason:'fs-error', error }.
+ *
+ * @param {object} record — candidate session-eval record.
+ * @param {{ path?: string }} [opts] — target path (absolute preferred); defaults
+ *        to DEFAULT_EVAL_JSONL_PATH (repo-relative).
+ * @returns {{ ok: true, record: object, path: string }
+ *          | { ok: false, reason: 'validation'|'fs-error', error: string }}
+ */
+export function appendEvalRecord(record, { path: filePath = DEFAULT_EVAL_JSONL_PATH } = {}) {
+  let validated;
+  try {
+    validated = validateEvalRecord(record);
+  } catch (err) {
+    const msg = err?.message ?? String(err);
+    process.stderr.write(`[eval-sink] WARN: record failed validation, not written: ${msg}\n`);
+    return { ok: false, reason: 'validation', error: msg };
+  }
+
+  try {
+    const line = JSON.stringify(validated) + '\n';
+    mkdirSync(path.dirname(filePath), { recursive: true });
+    appendFileSync(filePath, line, 'utf8');
+    return { ok: true, record: validated, path: filePath };
+  } catch (err) {
+    const msg = err?.message ?? String(err);
+    process.stderr.write(`[eval-sink] WARN: could not append to ${filePath}: ${msg}\n`);
+    return { ok: false, reason: 'fs-error', error: msg };
+  }
+}
+
+/**
+ * Read all session-eval records from the eval journal. Never throws — a missing
+ * file yields [] and malformed JSONL lines are skipped (readJsonlFile with
+ * skipInvalid). Each record is passed through normalizeEvalRecord so callers get
+ * a uniform shape (kpi nulls, judge-dim defaults, stamped versions).
+ *
+ * @param {string} [filePath] — absolute path to the eval journal.
+ * @returns {object[]} normalized session-eval records, in source order.
+ */
+export function readEvalRecords(filePath = DEFAULT_EVAL_JSONL_PATH) {
+  const rows = readJsonlFile(filePath, { skipInvalid: true });
+  return rows.map((r) => normalizeEvalRecord(r));
+}

package/scripts/lib/events-rotation.mjs

@@ -0,0 +1,86 @@
+/**
+ * events-rotation.mjs — size-based rotation for .orchestrator/metrics/events.jsonl.
+ *
+ * Contract: called from the session-start hook. Returns a result object; never
+ * throws on fs errors (rotation failure must not break session-start). Uses
+ * synchronous fs calls (the hook is short-lived; async adds complexity).
+ *
+ * Issue #251 (CRITICAL). Rotation fires only at session-start, not per-append —
+ * per-append overhead is wasteful given ~6 KiB/day growth.
+ *
+ * Rename safety (POSIX): atomic rename is safe with in-flight writers. Old fds
+ * continue writing to the original inode (now `events.jsonl.1`); new writers
+ * will open the new file on next append.
+ */
+
+import { statSync, renameSync, unlinkSync, existsSync } from 'node:fs';
+
+/**
+ * Rotate the events log if it exceeds `maxSizeMb`.
+ *
+ * Shift scheme: `.N-1` → `.N`, …, `.1` → `.2`, active `events.jsonl` → `.1`.
+ * The oldest backup (`events.jsonl.{maxBackups}`) is deleted before shifting.
+ *
+ * @param {object} opts
+ * @param {string}  opts.logPath     — absolute path to `events.jsonl`
+ * @param {number}  opts.maxSizeMb   — integer 1..1024
+ * @param {number}  opts.maxBackups  — integer 1..20
+ * @param {boolean} opts.enabled     — if false, returns early
+ * @returns {{rotated: boolean, reason?: string, archivedAs?: string, sizeBefore?: number, maxBackups?: number, error?: string}}
+ */
+export function maybeRotate({ logPath, maxSizeMb, maxBackups, enabled } = {}) {
+  // --- Input validation (throw — programmer error, not runtime fs failure) ---
+  if (typeof logPath !== 'string' || logPath.length === 0) {
+    throw new Error(`events-rotation: logPath must be a non-empty string, got ${typeof logPath}`);
+  }
+  if (!Number.isInteger(maxSizeMb) || maxSizeMb < 1 || maxSizeMb > 1024) {
+    throw new Error(`events-rotation: maxSizeMb must be integer 1..1024, got ${maxSizeMb}`);
+  }
+  if (!Number.isInteger(maxBackups) || maxBackups < 1 || maxBackups > 20) {
+    throw new Error(`events-rotation: maxBackups must be integer 1..20, got ${maxBackups}`);
+  }
+
+  if (enabled === false) {
+    return { rotated: false, reason: 'disabled' };
+  }
+
+  try {
+    if (!existsSync(logPath)) {
+      return { rotated: false, reason: 'no-file' };
+    }
+
+    const sizeBefore = statSync(logPath).size;
+    const threshold = maxSizeMb * 1024 * 1024;
+    if (sizeBefore < threshold) {
+      return { rotated: false, reason: 'under-threshold' };
+    }
+
+    // Drop the oldest backup if it exists — this keeps the ring bounded.
+    const oldestPath = `${logPath}.${maxBackups}`;
+    if (existsSync(oldestPath)) {
+      unlinkSync(oldestPath);
+    }
+
+    // Shift `.N-1` → `.N`, …, `.1` → `.2`.
+    for (let i = maxBackups - 1; i >= 1; i--) {
+      const src = `${logPath}.${i}`;
+      const dst = `${logPath}.${i + 1}`;
+      if (existsSync(src)) {
+        renameSync(src, dst);
+      }
+    }
+
+    // Rotate active file to `.1`. POSIX atomic rename.
+    renameSync(logPath, `${logPath}.1`);
+
+    return {
+      rotated: true,
+      archivedAs: `${logPath}.1`,
+      sizeBefore,
+      maxBackups,
+    };
+  } catch (err) {
+    // Never throw — rotation failure must not break session-start.
+    return { rotated: false, reason: 'error', error: err?.message ?? String(err) };
+  }
+}

package/scripts/lib/events-schema.mjs

@@ -0,0 +1,81 @@
+/**
+ * events-schema.mjs — canonical schema + naming-convention validator for
+ * `.orchestrator/metrics/events.jsonl` records.
+ *
+ * The orchestrator emits ONE event stream via `emitEvent()` (see events.mjs).
+ * Every record has the shape `{ timestamp, event, ...payload }`. Orchestrator-owned
+ * events follow the dotted namespace `orchestrator.<domain>.<verb>` (see
+ * docs/events-schema.md). Third-party / legacy event names (e.g. `tmux-layout.*`)
+ * are accepted as-is — this validator only enforces the convention on the
+ * `orchestrator.` namespace we own.
+ *
+ * Pure functions, no filesystem access — safe to import anywhere.
+ */
+
+/** ISO-8601 UTC timestamp with trailing Z (e.g. 2026-05-28T14:35:13.123Z). */
+const ISO_8601_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/;
+
+/** Prefix marking an orchestrator-owned event. */
+export const ORCHESTRATOR_PREFIX = 'orchestrator.';
+
+/**
+ * Dotted orchestrator-domain event name: `orchestrator.<domain>.<verb>[.<...>]`.
+ * Lowercase alphanumeric segments; underscores allowed WITHIN a segment
+ * (e.g. `quality_gate`, `propose_invoked`). Requires at least three segments
+ * (orchestrator + domain + verb).
+ */
+export const ORCHESTRATOR_EVENT_RE =
+  /^orchestrator\.[a-z0-9]+(?:_[a-z0-9]+)*(?:\.[a-z0-9]+(?:_[a-z0-9]+)*)+$/;
+
+/**
+ * True when `value` is an ISO-8601 UTC timestamp string that also parses to a
+ * real date.
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+export function isIso8601(value) {
+  return (
+    typeof value === 'string' &&
+    ISO_8601_RE.test(value) &&
+    !Number.isNaN(Date.parse(value))
+  );
+}
+
+/**
+ * Validate a single events.jsonl record against the canonical schema.
+ *
+ * Rules:
+ *   - record must be a plain (non-array) object;
+ *   - `timestamp` is required and must be an ISO-8601 UTC string;
+ *   - `event` is required and must be a non-empty string;
+ *   - events in the `orchestrator.` namespace MUST match
+ *     `orchestrator.<domain>.<verb>` (lowercase, dotted);
+ *   - non-orchestrator event names are accepted as-is (legacy / third-party).
+ *
+ * @param {unknown} record
+ * @returns {{ valid: boolean, errors: string[] }}
+ */
+export function validateEventRecord(record) {
+  if (record === null || typeof record !== 'object' || Array.isArray(record)) {
+    return { valid: false, errors: ['record must be a non-array object'] };
+  }
+
+  const errors = [];
+
+  if (!isIso8601(record.timestamp)) {
+    errors.push('timestamp must be an ISO-8601 UTC string ending in Z');
+  }
+
+  if (typeof record.event !== 'string' || record.event.length === 0) {
+    errors.push('event must be a non-empty string');
+  } else if (
+    record.event.startsWith(ORCHESTRATOR_PREFIX) &&
+    !ORCHESTRATOR_EVENT_RE.test(record.event)
+  ) {
+    errors.push(
+      `orchestrator-domain event "${record.event}" must match orchestrator.<domain>.<verb> (lowercase, dotted, ≥3 segments)`,
+    );
+  }
+
+  return { valid: errors.length === 0, errors };
+}

package/scripts/lib/events.mjs

@@ -0,0 +1,80 @@
+/**
+ * events.mjs — JSONL event emission + optional webhook POST for session-orchestrator.
+ *
+ * Replaces scripts/lib/events.sh. Windows-safe (no hardcoded path separators).
+ * Uses native fetch (Node 20+) and fs.promises — no external dependencies.
+ *
+ * Part of v3.0.0 migration (Epic #124, issue #133).
+ * Issue #228: removed hardcoded personal-domain default URL. Clank Event Bus URL
+ * must now be supplied explicitly via CLANK_EVENT_URL when CLANK_EVENT_SECRET is set.
+ */
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { SO_PROJECT_DIR, SO_SHARED_DIR } from './platform.mjs';
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Returns the absolute path to .orchestrator/metrics/events.jsonl in the project root.
+ * @returns {string}
+ */
+export function eventsFilePath() {
+  return path.join(SO_PROJECT_DIR, SO_SHARED_DIR, 'metrics', 'events.jsonl');
+}
+
+/**
+ * Append a JSONL event record and optionally POST to the Clank Event Bus webhook.
+ *
+ * Writes `{ts, event, ...payload}` as a single JSON line to
+ * `.orchestrator/metrics/events.jsonl` (creates parent directory if needed).
+ * If both `CLANK_EVENT_SECRET` and `CLANK_EVENT_URL` are set, fires an async
+ * fire-and-forget POST to `CLANK_EVENT_URL` with a 3-second timeout. Network
+ * errors are swallowed. Write errors propagate to the caller. No personal-domain
+ * default URL exists — both vars must be set explicitly (#228).
+ *
+ * @param {string} type — event type (e.g. "orchestrator.session.started")
+ * @param {object} [payload={}] — additional fields shallow-merged into the record
+ * @param {object} [opts={}] — emission options.
+ * @param {string} [opts.filePath] — override the destination JSONL path. Defaults
+ *   to `eventsFilePath()` (the project's `.orchestrator/metrics/events.jsonl`).
+ *   Used by `scripts/emit-event.mjs --file` so shell callers (e.g.
+ *   compute-grounding-injection.sh) can target a pre-resolved EVENTS_JSONL path
+ *   without depending on platform.mjs CWD/env resolution (#611).
+ * @returns {Promise<void>}
+ */
+export async function emitEvent(type, payload = {}, opts = {}) {
+  // Build the JSONL record: ts + event come first, payload spreads last.
+  const record = { timestamp: new Date().toISOString(), event: type, ...payload };
+  const line = JSON.stringify(record) + '\n';
+
+  // Ensure the destination directory exists before appending. The default
+  // resolution is unchanged for 2-arg callers — only an explicit opts.filePath
+  // overrides it (#611, additive).
+  const filePath = opts.filePath ?? eventsFilePath();
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.appendFile(filePath, line, 'utf8');
+
+  // Fire-and-forget webhook POST — only when secret AND URL are configured.
+  // No personal-domain default: CLANK_EVENT_URL must be set explicitly (#228).
+  if (process.env.CLANK_EVENT_SECRET && process.env.CLANK_EVENT_URL) {
+    const url = process.env.CLANK_EVENT_URL;
+    const body = JSON.stringify({
+      event_type: type,
+      source: 'session-orchestrator',
+      payload,
+    });
+
+    fetch(`${url}/api/webhooks/events`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${process.env.CLANK_EVENT_SECRET}`,
+      },
+      body,
+      signal: AbortSignal.timeout(3000),
+    }).catch(() => {});
+  }
+}