session-orchestrator 3.16.0 → 3.17.0

package/scripts/lib/telemetry/sync.mjs

@@ -0,0 +1,312 @@
+/**
+ * telemetry/sync.mjs — batch build + offline-tolerant sync for anonymous usage
+ * telemetry (Epic #841, Issue #844 / S3 FA3; PRD
+ * docs/prd/2026-07-20-anonymous-usage-telemetry.md §3-FA3).
+ *
+ * This module is the SEND path. It ties together the W2 primitives:
+ *   - consent.mjs  — resolveConsent (the outermost gate), telemetry.json read/write
+ *   - schema.mjs   — buildUsagePing + projectUsagePing (whitelist projection)
+ *   - anon-id.mjs  — ensureAnonId (lazy mint + 90-day rotation)
+ *   - queue.mjs    — the bounded NDJSON offline queue
+ *
+ * ── Outermost-seam gating (load-bearing privacy invariant) ───────────────────
+ * `resolveConsent()` is the FIRST statement of `flush()`. When it returns
+ * `send !== true` the function returns immediately — nothing below the gate is
+ * reachable: no fetch, no queue write, and NO anon-ID minting. The anon-ID is
+ * minted lazily inside `buildBatch()`, which `flush()` calls ONLY after the gate
+ * has passed. This makes "no ID exists until an affirmative-consent send is
+ * actually attempted" a structural guarantee, not a discipline.
+ *
+ * ── Fire-and-forget, never-throw ─────────────────────────────────────────────
+ * A flush never throws and never blocks a session beyond the POST timeout. On
+ * any send failure (network, timeout, non-2xx) the batch lands in the host-local
+ * queue (bounded, oldest-dropped) and the session closes with zero user-facing
+ * error.
+ *
+ * Node ESM. The only network dependency is the global `fetch`.
+ */
+
+import path from 'node:path';
+
+import {
+  resolveConsent,
+  readTelemetryState,
+  writeTelemetryState,
+  TELEMETRY_JSON_PATH,
+} from './consent.mjs';
+import { buildUsagePing, projectUsagePing } from './schema.mjs';
+import { ensureAnonId } from './anon-id.mjs';
+import { peekAll, enqueue, clear, queueStats } from './queue.mjs';
+import { loadOwnerConfig } from '../owner-yaml.mjs';
+import { readJsonlFile } from '../io.mjs';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** Public ingest endpoint. Overridable per-process via SO_TELEMETRY_ENDPOINT (tests/staging). */
+export const TELEMETRY_ENDPOINT = 'https://telemetry.session-orchestrator.com/v1/records';
+
+/** Fire-and-forget POST timeout (ms). */
+export const POST_TIMEOUT_MS = 3000;
+
+/** Daily-fallback horizon: only flush a backlog older than this. */
+const DAILY_FLUSH_MS = 24 * 60 * 60 * 1000;
+
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+
+/** Resolve the effective ingest endpoint (env override wins). */
+function resolveEndpoint(env) {
+  const override = (env?.SO_TELEMETRY_ENDPOINT || '').trim();
+  return override || TELEMETRY_ENDPOINT;
+}
+
+/**
+ * The default network sender: one POST carrying the whole batch array as the
+ * JSON body (the server accepts arrays), with an AbortSignal timeout. Resolves
+ * on a 2xx status, rejects on anything else (which routes the caller into the
+ * offline queue).
+ *
+ * @param {object} opts
+ * @param {NodeJS.ProcessEnv} opts.env
+ * @param {number} opts.timeoutMs
+ * @returns {(batches: object[]) => Promise<void>}
+ */
+function defaultSender({ env, timeoutMs }) {
+  const endpoint = resolveEndpoint(env);
+  return async (batches) => {
+    const res = await fetch(endpoint, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(batches),
+      signal: AbortSignal.timeout(timeoutMs),
+    });
+    if (!res.ok) {
+      throw new Error(`telemetry endpoint returned HTTP ${res.status}`);
+    }
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Batch build
+// ---------------------------------------------------------------------------
+
+/**
+ * Build ONE whitelist-projected usage-ping record from the local JSONL streams.
+ *
+ * Reads `<metricsDir>/sessions.jsonl` + `<metricsDir>/skill-invocations.jsonl`
+ * (metricsDir defaults to `<cwd>/.orchestrator/metrics`). The LAST sessions.jsonl
+ * record defines the session window: skill-invocations whose `timestamp >=` its
+ * `started_at` are included. When no session record exists, the ping falls back
+ * to `session_type: 'other'`, `duration_bucket: '<15m'`, and the invocations of
+ * the last 24 hours.
+ *
+ * anon-ID handling (persist=true, the send path): `ensureAnonId` runs on the
+ * telemetry.json record; a created/rotated ID is persisted via
+ * `writeTelemetryState`. With `persist=false` (the CLI `show` preview) NOTHING is
+ * minted or written — an existing ID is echoed, otherwise a placeholder string is
+ * shown. This preserves the lazy-ID invariant even for `show`.
+ *
+ * INVARIANT: `flush()` calls this ONLY after the consent gate has passed, so the
+ * (persisting) anon-ID mint is never reachable under `send !== true`.
+ *
+ * Never throws — an internal failure returns `{ record: null, reason }`.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.metricsDir]       Metrics dir (default `<cwd>/.orchestrator/metrics`).
+ * @param {NodeJS.ProcessEnv} [opts.env]   Env source (default process.env).
+ * @param {object} [opts.ownerConfig]      Parsed owner.yaml (default: loaded here).
+ * @param {{skills: Set<string>, commands: Set<string>}} [opts.roster] Roster (default: loaded by schema).
+ * @param {string} [opts.now]              ISO timestamp for sent_at + rotation clock.
+ * @param {string} [opts.statePath]        telemetry.json path override (test injection).
+ * @param {boolean} [opts.persist=true]    Mint+persist the anon-ID (false ⇒ preview only).
+ * @returns {{ record: object|null, reason?: string }}
+ */
+export function buildBatch({
+  metricsDir,
+  env = process.env,
+  ownerConfig,
+  roster,
+  now,
+  statePath,
+  persist = true,
+} = {}) {
+  try {
+    const dir = metricsDir || path.join(process.cwd(), '.orchestrator', 'metrics');
+    const nowIso = now || new Date().toISOString();
+
+    const sessions = readJsonlFile(path.join(dir, 'sessions.jsonl'), { skipInvalid: true });
+    const invocations = readJsonlFile(path.join(dir, 'skill-invocations.jsonl'), { skipInvalid: true });
+
+    const sessionRecord = sessions.length > 0 ? sessions[sessions.length - 1] : null;
+
+    let windowInvocations;
+    let sessionForPing;
+    if (sessionRecord && typeof sessionRecord.started_at === 'string' && !Number.isNaN(Date.parse(sessionRecord.started_at))) {
+      const startMs = Date.parse(sessionRecord.started_at);
+      windowInvocations = invocations.filter((rec) => {
+        const t = Date.parse(rec?.timestamp);
+        return !Number.isNaN(t) && t >= startMs;
+      });
+      sessionForPing = sessionRecord;
+    } else {
+      // No usable session record → 24h window + synthetic session (schema
+      // fallbacks yield session_type 'other' / duration_bucket '<15m').
+      const cutoff = (Number.isNaN(Date.parse(nowIso)) ? Date.now() : Date.parse(nowIso)) - DAILY_FLUSH_MS;
+      windowInvocations = invocations.filter((rec) => {
+        const t = Date.parse(rec?.timestamp);
+        return !Number.isNaN(t) && t >= cutoff;
+      });
+      sessionForPing = {};
+    }
+
+    const cfg = ownerConfig ?? loadOwnerConfig().config;
+
+    const ping = buildUsagePing({
+      sessionRecord: sessionForPing,
+      skillInvocations: windowInvocations,
+      ownerConfig: cfg,
+      env,
+      now: nowIso,
+      roster,
+    });
+
+    const target = statePath || TELEMETRY_JSON_PATH;
+    const { record: stateRecord } = readTelemetryState({ path: target });
+
+    if (persist) {
+      const { record: nextState, anon_id, created, rotated } = ensureAnonId(stateRecord, { now: nowIso });
+      if (created || rotated) {
+        writeTelemetryState(nextState, { path: target });
+      }
+      ping.anon_id = anon_id;
+    } else {
+      ping.anon_id =
+        typeof stateRecord.anon_id === 'string' && stateRecord.anon_id.trim() !== ''
+          ? stateRecord.anon_id
+          : '(generated on first send)';
+    }
+
+    return { record: projectUsagePing(ping) };
+  } catch (err) {
+    return { record: null, reason: `build-error: ${err?.message ?? String(err)}` };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Flush
+// ---------------------------------------------------------------------------
+
+/**
+ * Attempt to flush telemetry: gate on consent, build the batch, drain the
+ * offline queue together with the new record in ONE send, and empty the queue on
+ * success. Never throws, never blocks beyond `timeoutMs`.
+ *
+ * @param {object} [opts]
+ * @param {NodeJS.ProcessEnv} [opts.env]   Env source (default process.env).
+ * @param {number} [opts.timeoutMs]        POST timeout (default POST_TIMEOUT_MS).
+ * @param {(batches: object[]) => Promise<void>} [opts.sender] Injected sender (default: network POST).
+ * @param {string} [opts.metricsDir]       Metrics dir override.
+ * @param {string} [opts.statePath]        telemetry.json path override.
+ * @param {string} [opts.queuePath]        queue path override.
+ * @param {string} [opts.now]              ISO timestamp (sent_at, last_flush_at, rotation clock).
+ * @param {object} [opts.ownerConfig]      Parsed owner.yaml (default: loaded here). Inject to
+ *                                         isolate a test from the host's real owner.yaml fleet flag.
+ * @returns {Promise<{ sent: boolean, queued: boolean, state: string, reason: string }>}
+ */
+export async function flush({
+  env = process.env,
+  timeoutMs = POST_TIMEOUT_MS,
+  sender,
+  metricsDir,
+  statePath,
+  queuePath,
+  now,
+  ownerConfig,
+} = {}) {
+  // Resolve owner.yaml once (injectable for hermetic tests). loadOwnerConfig reads the host's
+  // real owner.yaml — a test asserting "consent absent" MUST inject {} or a real fleet flag
+  // (telemetry.enabled: true) legitimately flips send=true.
+  const cfg = ownerConfig ?? loadOwnerConfig().config;
+
+  // OUTERMOST SEAM — the consent gate is the FIRST statement. When send !== true
+  // nothing below (no fetch, no queue write, no anon-ID mint) is reachable.
+  const consent = resolveConsent({
+    env,
+    ownerConfig: cfg,
+    state: readTelemetryState({ path: statePath }).record,
+    interactive: false,
+  });
+  if (consent.send !== true) {
+    return { sent: false, queued: false, state: consent.state, reason: 'gated' };
+  }
+
+  const nowIso = now || new Date().toISOString();
+
+  // Build the batch (this lazily mints + persists the anon-ID — only reachable
+  // here, i.e. strictly after the gate).
+  const { record, reason } = buildBatch({ metricsDir, env, ownerConfig: cfg, statePath, now: nowIso });
+  if (!record) {
+    return { sent: false, queued: false, state: consent.state, reason: reason || 'no-record' };
+  }
+
+  // Debug seam: print the exact payload, send nothing.
+  if (env?.SO_TELEMETRY_DEBUG === '1') {
+    process.stderr.write(`${JSON.stringify(record)}\n`);
+    return { sent: false, queued: false, state: consent.state, reason: 'debug' };
+  }
+
+  // Drain the existing queue together with the new record in ONE send.
+  const queuedBatches = peekAll({ path: queuePath }).map((entry) => entry.batch);
+  const batches = [...queuedBatches, record];
+
+  const send = typeof sender === 'function' ? sender : defaultSender({ env, timeoutMs });
+
+  try {
+    await send(batches);
+  } catch {
+    // Send failed → only the NEW record joins the queue (queued batches remain
+    // in place since the queue was not cleared).
+    enqueue(record, { path: queuePath, now: nowIso });
+    return { sent: false, queued: true, state: consent.state, reason: 'queued' };
+  }
+
+  // 2xx → empty the queue and stamp last_flush_at (preserving the anon-ID that
+  // buildBatch may have just persisted).
+  clear({ path: queuePath });
+  const { record: freshState } = readTelemetryState({ path: statePath });
+  writeTelemetryState({ ...freshState, last_flush_at: nowIso }, { path: statePath });
+
+  return { sent: true, queued: false, state: consent.state, reason: 'sent' };
+}
+
+// ---------------------------------------------------------------------------
+// Daily-fallback predicate
+// ---------------------------------------------------------------------------
+
+/**
+ * Whether a daily-fallback flush is due: the queue is non-empty AND more than 24h
+ * have passed since the last successful flush (a never-flushed queue with items
+ * counts as due). Cheap — a single telemetry.json read + a queue stat. Never throws.
+ *
+ * @param {object} [opts]
+ * @param {string} [opts.statePath]  telemetry.json path override.
+ * @param {string} [opts.queuePath]  queue path override.
+ * @param {number} [opts.now]        Reference time in epoch-ms (default Date.now()).
+ * @returns {boolean}
+ */
+export function shouldDailyFlush({ statePath, queuePath, now = Date.now() } = {}) {
+  try {
+    const { count } = queueStats({ path: queuePath });
+    if (count <= 0) return false;
+
+    const { record } = readTelemetryState({ path: statePath });
+    const raw = record?.last_flush_at;
+    const lastMs = typeof raw === 'string' && !Number.isNaN(Date.parse(raw)) ? Date.parse(raw) : 0;
+    return now - lastMs > DAILY_FLUSH_MS;
+  } catch {
+    return false;
+  }
+}

package/scripts/lib/vault-status/board-writer.mjs

@@ -479,7 +479,13 @@ export function writeBoard(opts) {
  * @param {object} opts
  * @param {string} opts.repoRoot — the repo whose row is being updated.
  * @param {Array<{ repoRoot: string, repoName?: string, status?: string }>} [opts.repos]
- *   Full repo list; defaults to `[{ repoRoot, status: explicitStatus }]`.
+ *   Full repo list; defaults to a single-repo descriptor
+ *   `[{ repoRoot, status: explicitStatus }]`. Whichever entry resolves to
+ *   `repoRoot` — in the caller-supplied list as well as in the default — gets
+ *   its `repoName` filled from Session Config `vault-integration.vault-name`
+ *   (#660) unless it already carries one; entries for other repos are never
+ *   touched. With no `vault-name` configured, {@link collectRows} falls back to
+ *   `path.basename(repoRoot)` as before.
  * @param {string} [opts.explicitStatus] — per-repo status override ('closed' from session-end).
  * @param {Date} [opts.now]
  * @param {boolean} [opts.dryRun]
@@ -524,10 +530,51 @@ export async function mirrorBoard({ repoRoot, repos, explicitStatus, now = new D
     return { action: 'skipped-vault-disabled' };
   }
 
-  const repoList = Array.isArray(repos) && repos.length > 0
+  // `vault-name` (#660) overrides the git-derived repo slug for per-project
+  // vault namespacing. Without it `collectRows` falls back to
+  // `path.basename(repoRoot)`, so a repo whose directory name differs from its
+  // configured vault name renders under the wrong board row.
+  //
+  // The override is applied during descriptor NORMALISATION, not descriptor
+  // CONSTRUCTION (#835). Applying it only to the fallback single-repo
+  // descriptor made it inert on the primary production path: `sweepBoard`
+  // (session-start Phase 1.7) ALWAYS passes a non-empty `repos` — see
+  // {@link buildSweepRepos}, which unconditionally appends `thisRepoRoot` and
+  // emits bare `{ repoRoot }` entries with NO `repoName`. So session-start
+  // keyed the row `foldKey(path.basename(repoRoot))` while session-end (which
+  // calls this function WITHOUT `repos`) keyed it `foldKey(vault-name)`. The
+  // merge key is `repoName`, so the close never updated the in-progress row:
+  // a duplicate row plus a permanently stale `in-progress` status.
+  //
+  // Scope: ONLY the entry whose `repoRoot` resolves to THIS repo's root is
+  // touched. Entries for FOREIGN repos are left alone — a foreign repo's
+  // `vault-name` lives in ITS own Session Config, which this function does not
+  // read; stamping our override onto it would mislabel someone else's row. An
+  // entry that already carries an explicit non-empty `repoName` also wins, so
+  // a caller can still name its own row deliberately (#832 contract).
+  const vaultName = vault['vault-name'];
+  const repoNameOverride = typeof vaultName === 'string' && vaultName.length > 0
+    ? vaultName
+    : undefined;
+
+  const baseRepoList = Array.isArray(repos) && repos.length > 0
     ? repos
     : [{ repoRoot, status: explicitStatus }];
 
+  const repoList = repoNameOverride === undefined
+    ? baseRepoList
+    : baseRepoList.map((entry) => {
+        if (!entry || typeof entry.repoRoot !== 'string' || entry.repoRoot.length === 0) return entry;
+        if (typeof entry.repoName === 'string' && entry.repoName.length > 0) return entry;
+        let isSelf;
+        try {
+          isSelf = path.resolve(entry.repoRoot) === path.resolve(repoRoot);
+        } catch {
+          isSelf = false;
+        }
+        return isSelf ? { ...entry, repoName: repoNameOverride } : entry;
+      });
+
   const outputPath = resolveBoardPath(vaultDir);
 
   // Read the EXISTING generator-owned board (if any) to:
@@ -696,9 +743,11 @@ export function buildSweepRepos(candidates, { thisRepoRoot } = {}) {
  *       {@link mirrorBoard}'s idempotent merge, never dropped.
  *   (c) The enumerate + collectRows path is synchronous fs (readdirSync /
  *       existsSync / readLock per candidate) — O(repos) small reads, single-digit
- *       ms at host scale (~31 repos observed). No timeout is applied: a sync
- *       call cannot be preempted in-process, so a timeout would only convert a
- *       slow sweep into a thrown error, not a faster one.
+ *       ms at host scale (45 repos, ~0.9-1.9ms warm measured 2026-07-19 at the
+ *       default walk depth of 2; pre-#832's depth-1 scan saw only 1 of 47). No
+ *       timeout is applied: a sync call cannot be preempted in-process, so a
+ *       timeout would only convert a slow sweep into a thrown error, not a
+ *       faster one.
  *   (d) Merge key is `repoName` (`path.basename`), case-insensitively folded via
  *       {@link foldKey} (issue #719) — two rows differing only by case (e.g.
  *       `some-repo` vs `Some-Repo`, the same physical directory on a
@@ -713,6 +762,15 @@ export function buildSweepRepos(candidates, { thisRepoRoot } = {}) {
  *       remains a known limitation, inherited from {@link collectRows}/
  *       {@link mirrorBoard}; not addressed here.
  *
+ *       This limitation got materially WORSE with the depth-2 walk (#832):
+ *       under the old depth-1 scan, `<org-a>/<name>` and `<org-b>/<name>` were
+ *       both un-enumerable, so they could not collide. Both are now enumerated
+ *       and fold to a single row. Two such basename collisions were measured on
+ *       the reference host immediately after the change (same repo name under
+ *       two different org directories). Fixing this requires re-keying rows on
+ *       something path-derived rather than `path.basename` — deliberately out
+ *       of scope for #832 and tracked as a follow-up.
+ *
  * Best-effort contract: `sweepBoard` itself never throws for an enumeration
  * failure — `enumerateCandidates` is wrapped in try/catch; on ANY failure the
  * sweep degrades to the pre-#716 single-repo write

package/scripts/lib/vault-status/narrative-mirror.mjs

@@ -489,13 +489,6 @@ export async function mirrorNarrative(opts) {
     return { action: 'skipped-vault-disabled' };
   }
 
-  // Defense-in-depth: when the caller omits (or passes an empty) `repo`, derive
-  // it from the repoRoot basename rather than mis-filing the narrative under the
-  // 'unknown' slug. A missing repo name must never silently mis-file (#675 review).
-  const repoName = (typeof repo === 'string' && repo.trim().length > 0)
-    ? repo
-    : path.basename(path.resolve(repoRoot));
-
   // Read Session Config (CLAUDE.md / AGENTS.md) and resolve vault settings.
   let config;
   try {
@@ -509,6 +502,19 @@ export async function mirrorNarrative(opts) {
   if (!vaultIntegration || vaultIntegration.enabled !== true) {
     return { action: 'skipped-vault-disabled' };
   }
+
+  // Defense-in-depth: when the caller omits (or passes an empty) `repo`, derive
+  // it from the operator-configured `vault-name` override (#660/#832) when set,
+  // else the repoRoot basename — never silently mis-file under 'unknown' (#675
+  // review). Precedence: explicit `repo` opt > `vault-name` > basename.
+  const vaultNameOverride =
+    typeof vaultIntegration['vault-name'] === 'string' && vaultIntegration['vault-name'].trim()
+      ? vaultIntegration['vault-name'].trim()
+      : null;
+  const repoName = (typeof repo === 'string' && repo.trim().length > 0)
+    ? repo
+    : vaultNameOverride ?? path.basename(path.resolve(repoRoot));
+
   const rawVaultDir = vaultIntegration['vault-dir'];
   if (!rawVaultDir || typeof rawVaultDir !== 'string') {
     return { action: 'skipped-vault-disabled' };

package/scripts/mcp-server.sh

@@ -8,7 +8,8 @@ set -euo pipefail
 #   - session_config  — reads Session Config from the project instruction file
 #                       (CLAUDE.md, or AGENTS.md alias on Codex CLI — see
 #                       skills/_shared/instruction-file-resolution.md)
-#   - session_metrics — reads last 5 session metrics entries
+#   - session_metrics — reads last 5 REAL session metrics entries (#834:
+#                       abandoned phantom stubs are filtered out first)
 
 # ---------------------------------------------------------------------------
 # Helpers
@@ -75,7 +76,7 @@ handle_tools_list() {
       },
       {
         "name": "session_metrics",
-        "description": "Reads the last 5 session metrics entries from .orchestrator/metrics/sessions.jsonl",
+        "description": "Reads the last 5 REAL session metrics entries from .orchestrator/metrics/sessions.jsonl (abandoned phantom stubs excluded)",
         "inputSchema": {"type": "object", "properties": {}, "required": []}
       }
     ]
@@ -156,8 +157,19 @@ tool_session_metrics() {
     return
   fi
 
+  # Filter out phantom `status: 'abandoned'` stubs (#834, session-close-backfill
+  # — 0 waves, seconds of runtime) BEFORE taking the tail, so a recent phantom
+  # cannot displace real session records out of the last-5 window.
+  #
+  # `-R` (raw-input) + `fromjson?` parses each line individually and SKIPS
+  # unparseable ones instead of aborting the whole stream — plain
+  # `jq -c 'select(...)'` aborts at the FIRST malformed line (jq: parse error,
+  # exit 5), which is fatal here because sessions.jsonl is append-only from
+  # multiple writers and a torn write is exactly the case that matters. This
+  # mirrors the per-line try/catch behaviour of the .mjs path
+  # (scripts/lib/session-schema/filters.mjs).
   local entries
-  entries=$(tail -n 5 "$metrics_file" 2>/dev/null) || true
+  entries=$(jq -R -c 'fromjson? | select(.status != "abandoned")' "$metrics_file" 2>/dev/null | tail -n 5) || true
 
   if [[ -z "$entries" ]]; then
     respond "$id" "$(text_content "No metrics found (file is empty)")"