session-orchestrator 3.19.0 → 3.21.0

package/scripts/emit-session.mjs

@@ -25,8 +25,15 @@
  */
 
 import { readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { appendJsonl } from './lib/common.mjs';
+import {
+  MEMORY_CLEANUP_EVENT,
+  deriveMemoryCleanupSignal,
+  stampMemoryCleanup,
+} from './lib/memory-cleanup-stamp.mjs';
+import { serializeSessionLineChecked } from './lib/session-schema/serializer.mjs';
 import {
   validateSession,
   ValidationError,
@@ -35,45 +42,7 @@ import {
   aliasLegacyEndedAt,
 } from './lib/session-schema.mjs';
 
-/**
- * Serialize `validated` to a single JSONL line and prove it round-trips:
- * the line MUST be JSON-parseable AND the parsed-back object MUST still pass
- * `validateSession`. This is the pre-write self-validation seam (#662) that
- * mirrors the `appendLearning` guard — it catches non-serializable values
- * (`undefined`, `NaN`, `Infinity`, `BigInt`, circular refs) that
- * `JSON.stringify` silently drops or that produce a line which won't parse
- * back to the same schema-valid shape.
- *
- * Throws ValidationError (consumed by the caller as a validation failure,
- * exit 1). Does not write — the caller appends only after this returns.
- *
- * @param {object} validated — already validated session entry
- * @returns {string} the verified JSONL line (newline-terminated)
- * @throws {ValidationError} when the serialized line does not round-trip
- */
-export function serializeSessionLineChecked(validated) {
-  let line;
-  try {
-    line = JSON.stringify(validated);
-  } catch (err) {
-    throw new ValidationError(`session is not JSON-serializable: ${err.message}`);
-  }
-  if (typeof line !== 'string' || line.length === 0) {
-    throw new ValidationError('session serialized to an empty line');
-  }
-  let reparsed;
-  try {
-    reparsed = JSON.parse(line);
-  } catch (err) {
-    throw new ValidationError(
-      `serialized session line does not parse back as JSON: ${err.message}`
-    );
-  }
-  // Re-validate the round-tripped shape — catches required fields that were
-  // present as `undefined`/`NaN` before stringify but vanished after.
-  validateSession(reparsed);
-  return line + '\n';
-}
+export { serializeSessionLineChecked };
 
 function parseArgs(argv) {
   const args = { file: '.orchestrator/metrics/sessions.jsonl', entry: null };
@@ -148,6 +117,42 @@ async function main() {
     );
   }
 
+  // `memory_cleanup_at` derivation (#699 follow-up — Disziplin statt Mechanik).
+  // The flag used to be a boolean the coordinator-LLM remembered to pass at
+  // session-end; it was forgotten on 2026-08-14 and the cadence marker stalled
+  // 29 days behind the operator's own notes. It is now READ from the session's
+  // own `orchestrator.memory.cleanup_completed` events, sitting in the sibling
+  // events.jsonl of the target sessions.jsonl (same `.orchestrator/metrics/`
+  // directory in production, same tmp dir under test — no env plumbing).
+  //
+  // Precedence: an EXPLICIT `memory_cleanup_at` on the incoming record WINS and
+  // is never overwritten — an explicit stamp is a caller's positive assertion,
+  // while derivation only fills the gap left by silence.
+  const alreadyStamped =
+    typeof repaired.memory_cleanup_at === 'string' && repaired.memory_cleanup_at.length > 0;
+  if (!alreadyStamped) {
+    const eventsFile = join(dirname(args.file), 'events.jsonl');
+    const signal = deriveMemoryCleanupSignal({
+      eventsFile,
+      sessionId: repaired.session_id,
+      startedAt: repaired.started_at,
+      completedAt: repaired.completed_at,
+    });
+    if (signal.ranCleanup) {
+      const before = repaired;
+      repaired = stampMemoryCleanup(repaired, {
+        ranCleanup: true,
+        completedAt: repaired.completed_at,
+      });
+      if (repaired !== before) {
+        process.stderr.write(
+          `emit-session: derived memory_cleanup_at=${repaired.memory_cleanup_at} from ` +
+            `${signal.matches} ${MEMORY_CLEANUP_EVENT} event(s) (latest ${signal.at})\n`
+        );
+      }
+    }
+  }
+
   let validated;
   try {
     validated = validateSession(repaired);
@@ -167,7 +172,7 @@ async function main() {
   // sessions.jsonl and only surface on the NEXT session's read. Treated as a
   // validation failure (exit 1); file is left untouched.
   try {
-    serializeSessionLineChecked(validated);
+    serializeSessionLineChecked(repaired);
   } catch (err) {
     if (err instanceof ValidationError) {
       process.stderr.write(

package/scripts/export-hw-learnings.mjs

@@ -22,6 +22,8 @@
  *
  * ## Anonymization pipeline (enforced; no opt-out)
  *
+ * - Mask env-derived secret VALUES first (#1025, value-based — orthogonal to the
+ *   form-based rules below; see the note above `maskSecretValues`)
  * - Strip all absolute paths (macOS, Linux system paths, Windows)
  * - Redact IPv4 addresses
  * - Redact GitHub/GitLab URLs containing org/repo paths
@@ -55,6 +57,7 @@ import {
 } from './lib/learnings.mjs';
 import { findProjectRoot, resolveInstructionFile, expandTilde } from './lib/common.mjs';
 import { parseSessionConfig } from './lib/config.mjs';
+import { createSecretValueMasker } from './lib/secret-masker.mjs';
 
 // Vault-relative default write target (Epic #774 — docs Public-Split removed
 // the prior in-repo generated telemetry doc in favor of the private Meta-Vault).
@@ -135,6 +138,58 @@ const SIGNED_OFF_RE = /Signed-off-by:[^\n]+/g;
 // those never appear with a .local/.lan/.home suffix.
 const HOSTNAME_RE = /\b[A-Za-z0-9]+(?:[-.][A-Za-z0-9]+)*\.(?:local|lan|home|internal|corp)\b/g;
 
+// ---------------------------------------------------------------------------
+// Value-based secret masking (#1025) — the SECOND half of the pipeline
+// ---------------------------------------------------------------------------
+//
+// `anonymizeString` below is FORM-based: eight regexes over the SHAPE of a
+// string. `createSecretValueMasker` is VALUE-based: it looks for the literal
+// values of secret-NAMED env vars. The two are orthogonal and BOTH are needed —
+// measured on four probe secrets:
+//
+//   AWS secret key (with slashes)   FORM: secret survives   VALUE: [REDACTED]
+//   DB password (short, symbols)    FORM: secret survives   VALUE: [REDACTED]
+//   GitLab PAT (token shape)        FORM: <redacted-token>  VALUE: [REDACTED]
+//   all-letter passphrase           FORM: secret survives   VALUE: [REDACTED]
+//
+// The AWS key is the sharpest case: `/` is outside TOKEN_RE's character class,
+// so the key breaks into runs of 13/7/18 characters — every one of them BELOW
+// the 20-character floor, hence not a single match. Conversely the form catches
+// tokens of FOREIGN hosts that were never in `process.env`, which the value
+// masker cannot see. Neither subsumes the other.
+//
+// ORDER IS LOAD-BEARING: mask FIRST, anonymize SECOND. `[REDACTED]` is 8
+// characters with no digit, so it passes through `anonymizeString` untouched
+// (TOKEN_RE requires ≥20 chars AND a digit). The reverse order blinds the
+// masker: a value that `anonymizeString` has already rewritten to
+// `<redacted-token>` is no longer findable as a literal, so a secret that only
+// the VALUE filter would catch — because the form leaves a residue rather than
+// the whole span — can no longer be matched.
+
+/**
+ * Lazily-built, process-wide masker. `createSecretValueMasker` scans the whole
+ * env and compiles one RegExp per needle, so it is built ONCE per process and
+ * reused for every entry — never rebuilt per record.
+ *
+ * @type {{ mask: (text: string) => string, needleCount: number } | null}
+ */
+let _secretMasker = null;
+
+/**
+ * Mask env-derived secret VALUES in a free-form string.
+ *
+ * Fail-soft: non-strings pass through by reference, and a zero-needle env makes
+ * `mask` the identity — the masker must never be the reason an export dies.
+ *
+ * @param {string} s
+ * @returns {string}
+ */
+function maskSecretValues(s) {
+  if (typeof s !== 'string') return s;
+  if (_secretMasker === null) _secretMasker = createSecretValueMasker(process.env);
+  return _secretMasker.mask(s);
+}
+
 /**
  * Scrub a free-form string of PII / host-identifying content.
  * Order matters: author/signoff patterns first (they contain emails), then
@@ -185,8 +240,12 @@ export function anonymizeLearning(entry) {
   const e = normalizeLearning(entry);
   const out = {
     ...e,
-    insight: anonymizeString(e.insight),
-    evidence: anonymizeString(e.evidence),
+    // #1025: value-mask BEFORE form-anonymize — see the ORDER IS LOAD-BEARING
+    // note above `maskSecretValues`. This is the choke-point for BOTH write
+    // paths (promoteHwLearnings and exportHwLearnings both route through here);
+    // wiring at exportHwLearnings alone would leave the promote path unhardened.
+    insight: anonymizeString(maskSecretValues(e.insight)),
+    evidence: anonymizeString(maskSecretValues(e.evidence)),
     // Stamp after redaction so callers that write back through validateLearning
     // do not hit the scope=public contract check.
     anonymized: true,

package/scripts/lib/autopilot/worktree-pipeline.mjs

@@ -315,14 +315,14 @@ export async function teardownWorktree(context, result, opts = {}) {
     try {
       // #987 defense-in-depth: pass the in-memory genesis proof (captured by
       // runStoryPipeline right after acquire — same process, no file I/O)
-      // so the delete is double-gated at the fs layer. Spread-guarded:
-      // release()'s proof gate triggers on `proof !== undefined`, so a
-      // null/absent proof MUST be omitted entirely or EVERY release would
-      // be refused with 'proof-mismatch'.
+      // so the delete is double-gated at the fs layer. `_lockOwnerProof` is
+      // null/undefined when the lock could not yield a full proof; release()
+      // gates on `proof != null` (#989) and degrades to the session_id-only
+      // path for that case, so no call-site guard is needed.
       releaseResult = release({
         sessionId: result._lockSessionId,
         repoRoot: result.worktreePath,
-        ...(result._lockOwnerProof ? { proof: result._lockOwnerProof } : {}),
+        proof: result._lockOwnerProof,
       });
     } catch (lockErr) {
       console.error(

package/scripts/lib/backlog-scan.mjs

@@ -10,39 +10,96 @@
  *  - Module-level cache keyed on (vcs, limit) — one VCS round-trip per session.
  *  - Graceful degradation: missing CLI / non-zero exit / parse failure → null.
  *    Never throws to the caller.
+ *  - No silent caps: the scan reads at most `limit` issues, so every count is a
+ *    LOWER BOUND when the window fills. `truncated` says so in the return value
+ *    and a WARNING says so on stderr — a bound that is applied is announced.
  *
  * Stale threshold: 30 days since `updated_at`.
  *
  * Dependencies:
- *  - VCS detection follows `skills/gitlab-ops/SKILL.md` (origin URL contains "github.com" → gh, else glab).
+ *  - VCS detection delegates to `vcs-repo-spec.mjs::detectVcsFamily` (#1039):
+ *    remote-host family classification over ALL remotes, not a hard-coded
+ *    `origin` lookup with a `url.includes('github.com')` test.
  */
 
 import { spawnSync } from 'node:child_process';
 
+import { warn } from './common.mjs';
 import { normalizeLabel } from './label-scope.mjs';
-import { resolveRepoSpec } from './vcs-repo-spec.mjs';
+import { detectVcsFamily, isQueryFailure, redactUrlCredentials, resolveRepoSpec } from './vcs-repo-spec.mjs';
 
 export const STALE_THRESHOLD_DAYS = 30;
 
+/**
+ * Default scan window, and the SINGLE source for that number — every caller
+ * either omits `limit` or imports this constant. A second hand-written copy is
+ * what made the old default wrong in three places at once.
+ *
+ * Ceiling: 100 is the largest single-request page BOTH CLIs serve reliably
+ * (the GitLab API clamps `per_page` at 100), so it is the widest exact window
+ * available without paginating. Above it the scan truncates — `truncated: true`
+ * plus a stderr WARNING announce that, and every count becomes a lower bound.
+ * Revisit with a `--page` loop if a repo's OPEN backlog routinely exceeds 100.
+ */
+export const DEFAULT_BACKLOG_LIMIT = 100;
+
 /** Module-level cache. Keyed by JSON.stringify({vcs, limit}). */
 const _cache = new Map();
 
 /**
- * Detect the VCS for the current working directory by inspecting the origin URL.
+ * Detect the VCS family of a repo from its git remotes.
  * Returns 'github' | 'gitlab' | null. Never throws.
  *
+ * This probe cannot take `vcs` as a parameter and cannot use
+ * `resolveRepoSpec({vcs})`: it runs in order to DETERMINE `vcs`. So it uses the
+ * vcs-less projection {@link detectVcsFamily}, which classifies every remote by
+ * URL host (then by remote name) instead of pinning `origin`.
+ *
+ * Two #1039 defects this replaces:
+ *  1. `git remote get-url origin` returned non-zero on a repo whose remotes are
+ *     named `gitlab`/`github` (no `origin`) — a perfectly scannable backlog
+ *     read as "no VCS".
+ *  2. `url.includes('github.com')` classified GitHub Enterprise
+ *     (`git@github.example.com:o/r.git`) as gitlab, pointing `glab` at a GitHub
+ *     instance. `detectVcsFamily`'s host rule (`github.*`) covers it.
+ *
+ * `null` still means "no backlog signal" to the caller, but the two states it
+ * used to fold are now distinguishable on stderr: a QUERY FAILURE (git missing,
+ * not a git repo) emits exactly one WARNING, because a 40-issue backlog reading
+ * as empty is a degraded measurement the operator must see. An ABSENCE
+ * (`no-remotes`, `no-matching-remote`) stays SILENT — it is a legitimate repo
+ * state, and warning on it would train operators to ignore the warning that
+ * matters.
+ *
+ * @param {{ repoRoot?: string, gitRun?: (args: string[]) => { ok: boolean, stdout?: string, stderr?: string, status?: number, code?: string } }} [opts]
+ *   `gitRun` is the injectable git seam of `detectVcsFamily` (tests stub it
+ *   instead of shelling out); `repoRoot` defaults to `process.cwd()` there.
  * @returns {'github'|'gitlab'|null}
  */
-export function detectVcs() {
+export function detectVcs({ repoRoot, gitRun } = {}) {
+  let detected;
   try {
-    const r = spawnSync('git', ['remote', 'get-url', 'origin'], { encoding: 'utf8' });
-    if (r.status !== 0) return null;
-    const url = String(r.stdout || '').trim();
-    if (!url) return null;
-    return url.includes('github.com') ? 'github' : 'gitlab';
+    detected = detectVcsFamily({ repoRoot, gitRun });
   } catch {
     return null;
   }
+
+  if (detected.ok) return detected.vcs;
+
+  if (isQueryFailure(detected.reason)) {
+    // Redact before logging: git stderr can echo a remote URL carrying
+    // userinfo credentials (#907, CWE-214). First line only — a git fatal is
+    // one line, and the rest is noise in a session banner.
+    const detail = redactUrlCredentials(String(detected.stderr || ''))
+      .split('\n')[0]
+      .trim();
+    warn(
+      `backlog scan could not determine the VCS family (${detected.reason})` +
+        `${detail ? `: ${detail}` : ''} — backlog signal degraded to null (contributes 0 delta to mode selection).`
+    );
+  }
+
+  return null;
 }
 
 /**
@@ -91,6 +148,10 @@ function ageDays(iso, nowMs) {
  *  - `labels`: array of strings OR array of {name: string} objects
  *  - `updated_at` (glab) or `updatedAt` (gh): ISO-8601 timestamp
  *
+ * `total` is the number of records AGGREGATED, never the number of records that
+ * exist in the tracker — the caller decides the window, so only the caller
+ * (or `scanBacklog`'s `truncated` flag) can tell the two apart.
+ *
  * @param {Array<object>} issues
  * @param {number} nowMs — injected for tests
  * @returns {{criticalCount: number, highCount: number, staleCount: number, byLabel: Record<string, number>, total: number}}
@@ -140,12 +201,21 @@ export function summarizeIssues(issues, nowMs = Date.now()) {
  * (vcs, limit) pair never collide on a shared cache entry.
  *
  * Returns null on any of:
- *  - VCS cannot be detected (no git origin)
+ *  - VCS family cannot be detected ({@link detectVcs} — no remotes, or git
+ *    itself unavailable; the latter also emits one WARNING)
  *  - CLI binary missing (`glab` for gitlab, `gh` for github)
  *  - CLI exits non-zero or produces unparsable output
  *
  * Never throws.
  *
+ * The scan reads at most `limit` records (default `DEFAULT_BACKLOG_LIMIT`), so
+ * every count is a LOWER BOUND once the window fills. `truncated` reports that:
+ * `true` means the CLI returned a full window, so records — and the critical /
+ * high / stale issues among them — may lie beyond it. It is deliberately
+ * conservative: a backlog of exactly `limit` issues reports `truncated: true`
+ * even though nothing was missed. Over-reporting "you may have missed some" is
+ * safe; under-reporting it is the bug this flag exists to prevent.
+ *
  * @param {{
  *   limit?: number,
  *   vcs?: 'github'|'gitlab'|null,
@@ -160,19 +230,23 @@ export function summarizeIssues(issues, nowMs = Date.now()) {
  *   get-url`); tests inject a stub instead of shelling out. `runJsonFn` is
  *   the injectable seam for the CLI runner — defaults to the real `runJson`
  *   (shells out to `glab`/`gh`).
- * @returns {Promise<null | {criticalCount: number, highCount: number, staleCount: number, byLabel: Record<string, number>, total: number, vcs: string, limit: number}>}
+ * @returns {Promise<null | {criticalCount: number, highCount: number, staleCount: number, byLabel: Record<string, number>, total: number, vcs: string, limit: number, truncated: boolean}>}
  */
 export async function scanBacklog(opts = {}) {
-  const limit = Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : 50;
+  const limit =
+    Number.isInteger(opts.limit) && opts.limit > 0 ? opts.limit : DEFAULT_BACKLOG_LIMIT;
+  const repoRoot = typeof opts.repoRoot === 'string' ? opts.repoRoot : process.cwd();
   // Distinguish "user did not pass vcs" (auto-detect) from "user explicitly passed
   // null" (degrade). 'vcs' in opts catches the explicit-null path so callers can
   // force the no-vcs branch in tests without monkey-patching detectVcs.
-  const vcs = 'vcs' in opts ? opts.vcs : detectVcs();
+  // `repoRoot` is passed through so detection and `-R` spec resolution below
+  // answer about the SAME repo — they used to disagree whenever a caller passed
+  // `repoRoot` (detection silently read `process.cwd()` instead).
+  const vcs = 'vcs' in opts ? opts.vcs : detectVcs({ repoRoot });
   const nowMs = typeof opts.nowMs === 'number' ? opts.nowMs : Date.now();
 
   if (vcs !== 'github' && vcs !== 'gitlab') return null;
 
-  const repoRoot = typeof opts.repoRoot === 'string' ? opts.repoRoot : process.cwd();
   const resolveRepoSpecFn =
     typeof opts.resolveRepoSpecFn === 'function' ? opts.resolveRepoSpecFn : resolveRepoSpec;
   const runJsonFn = typeof opts.runJsonFn === 'function' ? opts.runJsonFn : runJson;
@@ -197,8 +271,25 @@ export async function scanBacklog(opts = {}) {
     return null;
   }
 
+  // A full window means records may lie beyond it, so every count is a lower
+  // bound. `>=` (not `> `) because the CLIs cap silently: the GitLab API clamps
+  // `per_page` at 100, so an over-fetch of `limit + 1` would come back capped
+  // and read as "not truncated" — under-approximating exactly the way the
+  // window itself did. A full window is the only signal that survives clamping.
+  const truncated = issues.length >= limit;
+
   const summary = summarizeIssues(issues, nowMs);
-  const result = { ...summary, vcs, limit };
+  const result = { ...summary, vcs, limit, truncated };
+
+  // Announce the bound (never a silent cap). Emitted once per cache key — a
+  // cache hit returns before this point, so a per-session scan warns once.
+  if (truncated) {
+    warn(
+      `backlog scan filled its ${limit}-issue window (${bin}): criticalCount/highCount/staleCount are LOWER BOUNDS. ` +
+        `Pass a larger limit for exact counts.`
+    );
+  }
+
   _cache.set(cacheKey, result);
   return result;
 }

package/scripts/lib/build-live-signals.mjs

@@ -18,7 +18,7 @@ import { resolve } from 'node:path';
 import { parseStateMd, parseRecommendations } from './state-md.mjs';
 import { normalizeSession, tailRealSessions } from './session-schema.mjs';
 import { parseBootstrapLock } from './bootstrap-lock-freshness.mjs';
-import { scanBacklog } from './backlog-scan.mjs';
+import { scanBacklog, DEFAULT_BACKLOG_LIMIT } from './backlog-scan.mjs';
 
 // ---------------------------------------------------------------------------
 // Public API
@@ -36,7 +36,11 @@ import { scanBacklog } from './backlog-scan.mjs';
  * @param {string} [opts.sessionsPath]   — defaults to '.orchestrator/metrics/sessions.jsonl'
  * @param {string} [opts.lockPath]       — defaults to '.orchestrator/bootstrap.lock'
  * @param {Array}  [opts.learnings]      — pre-surfaced top-N learnings; defaults to []
- * @param {number} [opts.backlogLimit]   — passed to scanBacklog; defaults to 50
+ * @param {number} [opts.backlogLimit]   — passed to scanBacklog; defaults to
+ *   `DEFAULT_BACKLOG_LIMIT` from backlog-scan.mjs (never a local copy of that
+ *   number). A window smaller than the repo's open backlog makes
+ *   `backlog.criticalCount`/`highCount`/`staleCount` lower bounds —
+ *   `backlog.truncated` is the flag that says so.
  * @param {number} [opts.sessionTailN]   — defaults to 10 (last N sessions)
  * @param {Function} [opts._scanBacklog] — injectable seam for tests (defaults to scanBacklog)
  * @returns {Promise<import('./mode-selector.mjs').Signals>}
@@ -56,7 +60,7 @@ export async function buildLiveSignals(opts = {}) {
   const learnings = Array.isArray(opts.learnings) ? opts.learnings : [];
   const backlogLimit = typeof opts.backlogLimit === 'number' && opts.backlogLimit > 0
     ? opts.backlogLimit
-    : 50;
+    : DEFAULT_BACKLOG_LIMIT;
   const sessionTailN = typeof opts.sessionTailN === 'number' && opts.sessionTailN > 0
     ? opts.sessionTailN
     : 10;