slice-tournament-zoo 0.5.6

package/src/hack-detector.ts

@@ -0,0 +1,124 @@
+/**
+ * Layer L3 of the anti-reward-hacking defense (F10): the trace/diff inspector.
+ *
+ * Scans specimen source for known reward-hacking signatures. Any finding is a
+ * disqualification at the eval gate (F7 stage 1). Each finding carries a
+ * remediation string that is re-injected into the specimen's next prompt if the
+ * failure-replan loop activates (F14, Codex custom-linter pattern).
+ *
+ * This is a pragmatic regex/line-scan detector (the AST upgrade is a v1.1 task
+ * in the roadmap). It is intentionally conservative: it flags the documented
+ * patterns and explains why, rather than guessing at novel ones.
+ */
+import type { HackFinding, HackPattern, SpecimenId } from "./types.js";
+
+interface Rule {
+  pattern: HackPattern;
+  /** Matches a line that exhibits the pattern. */
+  test: (line: string, ctx: ScanContext) => boolean;
+  remediation: string;
+}
+
+interface ScanContext {
+  /** Names of sealed test fixtures the specimen must not key behaviour on. */
+  fixtureNames: string[];
+}
+
+const RULES: Rule[] = [
+  {
+    pattern: "test-skip",
+    test: (l) =>
+      /\b(it|test|describe)\.(skip|todo)\b/.test(l) ||
+      /\bx(it|describe)\b/.test(l) ||
+      /@pytest\.mark\.skip/.test(l) ||
+      /\bpytest\.skip\(/.test(l),
+    remediation:
+      "Do not skip or mark tests todo. Implement the contract so every sealed test runs and passes.",
+  },
+  {
+    pattern: "assertion-mutation",
+    test: (l) =>
+      // Weakening assertions to always-true forms.
+      /\bassert\s+true\b/i.test(l) ||
+      /\bexpect\([^)]*\)\.toBe(Truthy|Defined)\(\)\s*;?\s*\/\/\s*always/i.test(l) ||
+      /\bassert\s+1\s*==\s*1\b/.test(l) ||
+      /\bexpect\(true\)\.toBe\(true\)/.test(l),
+    remediation:
+      "Do not replace meaningful assertions with tautologies. Assert the real contract behaviour.",
+  },
+  {
+    pattern: "network-bypass",
+    test: (l) =>
+      // Real network egress in code that should use fixtures.
+      /\b(fetch|axios|requests\.(get|post)|urllib|http\.request)\s*\(/.test(l) &&
+      !/mock|fixture|localhost|127\.0\.0\.1/.test(l),
+    remediation:
+      "Do not make live network calls that bypass fixtures. Use the provided test doubles.",
+  },
+  {
+    pattern: "fixture-keyed-branch",
+    test: (l, ctx) =>
+      ctx.fixtureNames.length > 0 &&
+      ctx.fixtureNames.some(
+        (f) => l.includes(f) && /\b(if|switch|case|when|elif)\b/.test(l),
+      ),
+    remediation:
+      "Do not special-case behaviour on a specific test fixture name/value. Implement the general contract.",
+  },
+  {
+    pattern: "hardcoded-test-input",
+    test: (l) =>
+      // return of a literal keyed on an obvious test sentinel.
+      /\bif\b.*==.*["'](test|expected|__TEST__|fixture)["']/.test(l) ||
+      /return\s+["'](EXPECTED_OUTPUT|__GOLDEN__)["']/.test(l),
+    remediation:
+      "Do not hardcode expected outputs keyed on test sentinels. Compute the result from inputs.",
+  },
+];
+
+/** Scan one specimen's source files. `files` maps relative path → contents. */
+export function detectHacks(
+  specimen: SpecimenId,
+  files: Record<string, string>,
+  ctx: ScanContext = { fixtureNames: [] },
+): HackFinding[] {
+  const findings: HackFinding[] = [];
+  for (const [path, content] of Object.entries(files)) {
+    // Only scan implementation files; specimens never author the sealed tests,
+    // but a specimen may add its own helper tests — those are still in scope
+    // for skip/assertion-mutation detection.
+    const lines = content.split("\n");
+    lines.forEach((line, i) => {
+      const stripped = stripComments(line);
+      for (const rule of RULES) {
+        if (rule.test(stripped, ctx)) {
+          findings.push({
+            specimen,
+            pattern: rule.pattern,
+            location: `${path}:${i + 1}`,
+            remediation: rule.remediation,
+          });
+        }
+      }
+    });
+  }
+  return findings;
+}
+
+/** Strip trailing line comments so commented-out code does not false-positive. */
+function stripComments(line: string): string {
+  // Keep `// always` style markers used by assertion-mutation rule, so only
+  // strip a comment if it is clearly a full-line comment.
+  const trimmed = line.trimStart();
+  if (trimmed.startsWith("//") || trimmed.startsWith("#")) return "";
+  return line;
+}
+
+/** Aggregate all specimens' remediations for a replan prompt (F14). */
+export function remediationContext(findings: HackFinding[]): string {
+  const unique = new Map<HackPattern, string>();
+  for (const f of findings) unique.set(f.pattern, f.remediation);
+  return [...unique.entries()]
+    .map(([p, r]) => `- [${p}] ${r}`)
+    .join("\n");
+}

package/src/index.ts

@@ -0,0 +1,17 @@
+/** STZ public surface. */
+export * from "./types.js";
+export * from "./grpo.js";
+export * from "./selection.js";
+export * from "./hack-detector.js";
+export * from "./escalation.js";
+export * from "./budget.js";
+export * from "./cost-tracker.js";
+export * from "./taxonomy.js";
+export * from "./state.js";
+export * from "./project.js";
+export * from "./pressure.js";
+export * from "./specdiff.js";
+export * from "./seal.js";
+export * from "./mock/orchestrator.js";
+export * as llm from "./mock/interfaces.js";
+export { MockModelLayer, defaultMockConfig, alwaysFailConfig } from "./mock/mock.js";

package/src/merge.ts

@@ -0,0 +1,245 @@
+/**
+ * Cross-slice merge integrity — sealed-invariant supersession (F13-adjacent).
+ *
+ * When several slice winners are assembled into one integrated crate, an EARLIER
+ * slice's sealed suite can legitimately fail: it encodes an invariant that was
+ * correct in isolation but is obsolete under a LATER slice's composition. The
+ * canonical case from the dogfood run: slice-03's suite asserts "aliens never
+ * respawn", which slice-05's wave-clear deliberately supersedes. The assembled
+ * crate fails slice-03's suite — but that is not a merge defect.
+ *
+ * The danger is the orchestrator hand-waving that distinction ("looks like the
+ * expected interaction, moving on") — exactly the unaudited, gameable judgment
+ * STZ exists to eliminate. This module makes the call deterministic and audited
+ * instead. A failing sealed suite is only sanctioned when ALL of:
+ *
+ *   1. a **signature-pinned** compat entry matches the actual failure text (not
+ *      the test name alone — the exact panic/assert substring),
+ *   2. the **superseding invariant also passes** on the same assembled crate
+ *      (you cannot claim supersession when the replacement isn't even proven),
+ *   3. the entry is **approved** (the merge agent may propose but not self-bless).
+ *
+ * Trust boundary (be honest, same split as `eval` vs `record-eval`): this module
+ * does NOT run the suites — it can't, the assembled crate may be Rust. It
+ * consumes the *reported* per-suite results and deterministically ADJUDICATES
+ * those failures against the audited compat rules. A dishonest results file
+ * defeats it; the orchestration contract (run the suites in an ephemeral scratch
+ * copy, never the canonical crate) lives in the merge command doc.
+ *
+ * Rules 3 (ephemeral scratch) and the "agent can't self-approve" half of rule 1
+ * are conventions backed by AUDIT, not hard barriers: the append-only `history`
+ * makes a self-approval a visible anomaly rather than a silent one — STZ's
+ * auditability-over-prevention posture (N1). A compat entry is transitional debt:
+ * it points at a pending wave-aware amendment and is retired once that
+ * `seal-amend` lands and the amended (wave-aware) suite replaces the old one.
+ */
+import { readFileSync, existsSync, mkdirSync } from "node:fs";
+import { writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { stzPath } from "./taxonomy.js";
+
+/** The replacement invariant whose passing proves a supersession is legitimate. */
+export interface ReplacementInvariant {
+  /** Sealed suite (slice id) whose passing proves the superseding invariant. */
+  slice: string;
+  /** Optional specific test within that suite (informational; suite-level enforced). */
+  test?: string;
+}
+
+/** One audited "this old invariant is superseded by a later slice" entry. */
+export interface MergeCompatEntry {
+  id: string;
+  /** The sealed suite that now fails on the assembled crate. */
+  supersededSlice: string;
+  /** Optional test target/name within the superseded suite. */
+  supersededTest?: string;
+  /** Signature-pinned: the exact panic/assert substring that must appear in the
+   *  reported failure. Matching the test name alone is forbidden — that is how a
+   *  genuinely new bug in the same test would be laundered as "expected". */
+  panicSubstring: string;
+  /** The slice whose newer invariant legitimately supersedes the old one. */
+  supersededBy: string;
+  /** The replacement invariant that MUST simultaneously pass (rule 2). */
+  replacement: ReplacementInvariant;
+  reason: string;
+  /** Pointer to the pending wave-aware amendment that retires this entry (debt). */
+  pendingAmendment: string;
+  /** Approval gate (rule 3): the merge agent proposes false; an approver flips true. */
+  approved: boolean;
+  /** Who/why, recorded on approval — a self-approval is then an auditable anomaly. */
+  approvedBy?: string;
+}
+
+export interface MergeCompatHistoryEvent {
+  seq: number;
+  action: "propose" | "approve" | "retire";
+  id: string;
+  detail: string;
+}
+
+export interface MergeCompatManifest {
+  schemaVersion: 1;
+  entries: MergeCompatEntry[];
+  /** Append-only audit of propose/approve/retire — the protection for rules 1/3. */
+  history: MergeCompatHistoryEvent[];
+}
+
+/** One reported sealed-suite result on the assembled crate (caller-supplied). */
+export interface SealedSuiteResult {
+  slice: string;
+  passed: boolean;
+  /** The failure/panic text (signature) when passed === false. */
+  failure?: string;
+}
+
+export interface MergeVerdict {
+  ok: boolean;
+  /** Failures sanctioned by a matched + superseding-proven + approved entry. */
+  sanctioned: { slice: string; entryId: string; supersededBy: string }[];
+  /** Matched + superseding passes but NOT approved → blocks, awaits approval. */
+  pendingApproval: { slice: string; entryId: string }[];
+  /** Matched but the replacement invariant did NOT pass → blocks even if approved. */
+  invalid: { slice: string; entryId: string; reason: string }[];
+  /** No entry matches the signature → blocks; suspect a real merge defect. */
+  unsanctioned: { slice: string; reason: string }[];
+  /** Approved entries that sanctioned nothing this run — informational retire candidates. */
+  unused: string[];
+}
+
+// ── paths + persistence ──────────────────────────────────────────────────────
+
+const COMPAT_REL = join("90-audit", "merge-compat.json");
+
+export function mergeCompatPath(root: string): string {
+  return stzPath(root, COMPAT_REL);
+}
+
+export function freshCompatManifest(): MergeCompatManifest {
+  return { schemaVersion: 1, entries: [], history: [] };
+}
+
+export function loadCompat(root: string): MergeCompatManifest {
+  const p = mergeCompatPath(root);
+  if (!existsSync(p)) return freshCompatManifest();
+  return JSON.parse(readFileSync(p, "utf8")) as MergeCompatManifest;
+}
+
+export function saveCompat(root: string, manifest: MergeCompatManifest): void {
+  const p = mergeCompatPath(root);
+  mkdirSync(join(p, ".."), { recursive: true });
+  writeFileSync(p, JSON.stringify(manifest, null, 2) + "\n", "utf8");
+}
+
+function appendHistory(m: MergeCompatManifest, action: MergeCompatHistoryEvent["action"], id: string, detail: string): void {
+  m.history.push({ seq: m.history.length, action, id, detail });
+}
+
+// ── mutations (propose / approve / retire) ───────────────────────────────────
+
+export type ProposeInput = Omit<MergeCompatEntry, "approved" | "approvedBy">;
+
+/**
+ * Propose a compat entry. Always lands `approved:false` regardless of the input
+ * (the merge agent cannot self-approve — rule 3). Rejects an empty
+ * `panicSubstring` (it would match every failure) and a duplicate id.
+ */
+export function proposeCompat(m: MergeCompatManifest, entry: ProposeInput): { ok: true } | { ok: false; error: string } {
+  if (!entry.id) return { ok: false, error: "entry id is required" };
+  if (!entry.panicSubstring || entry.panicSubstring.trim() === "") {
+    return { ok: false, error: "panicSubstring must be non-empty (an empty signature would match every failure)" };
+  }
+  if (!entry.pendingAmendment || entry.pendingAmendment.trim() === "") {
+    return { ok: false, error: "pendingAmendment is required — a compat entry is transitional debt and must name the amendment that retires it" };
+  }
+  if (m.entries.some((e) => e.id === entry.id)) return { ok: false, error: `duplicate entry id: ${entry.id}` };
+  m.entries.push({ ...entry, approved: false });
+  appendHistory(m, "propose", entry.id, `${entry.supersededSlice} superseded by ${entry.supersededBy}; sig="${entry.panicSubstring}"`);
+  return { ok: true };
+}
+
+/** Approve a proposed entry. Records who/why so a self-approval is auditable. */
+export function approveCompat(m: MergeCompatManifest, id: string, by: string): { ok: true } | { ok: false; error: string } {
+  const e = m.entries.find((x) => x.id === id);
+  if (!e) return { ok: false, error: `no such compat entry: ${id}` };
+  if (e.approved) return { ok: false, error: `already approved: ${id}` };
+  e.approved = true;
+  e.approvedBy = by;
+  appendHistory(m, "approve", id, `approved by ${by}`);
+  return { ok: true };
+}
+
+/**
+ * Retire an entry — the transitional-debt end state (rule 5). Should correspond
+ * to a `seal-amend` of the superseded suite (now wave-aware); the amendment
+ * reference is recorded so the audit links the two.
+ */
+export function retireCompat(m: MergeCompatManifest, id: string, amendmentRef: string): { ok: true } | { ok: false; error: string } {
+  const idx = m.entries.findIndex((x) => x.id === id);
+  if (idx < 0) return { ok: false, error: `no such compat entry: ${id}` };
+  m.entries.splice(idx, 1);
+  appendHistory(m, "retire", id, `retired; superseded suite amended via ${amendmentRef}`);
+  return { ok: true };
+}
+
+// ── the deterministic verdict (the heart) ────────────────────────────────────
+
+/**
+ * Adjudicate reported sealed-suite results against the compat manifest. Pure and
+ * total: same inputs → same verdict (N6). A failing suite is sanctioned only when
+ * a signature-matched entry is approved AND its replacement invariant passes;
+ * matched-but-unapproved is `pendingApproval`, matched-but-replacement-fails is
+ * `invalid`, and no-match is `unsanctioned`. Any non-empty pendingApproval /
+ * invalid / unsanctioned blocks the merge.
+ */
+export function validateMerge(results: SealedSuiteResult[], manifest: MergeCompatManifest): MergeVerdict {
+  const bySlice = new Map(results.map((r) => [r.slice, r]));
+  const passing = (slice: string): boolean => bySlice.get(slice)?.passed === true;
+
+  const sanctioned: MergeVerdict["sanctioned"] = [];
+  const pendingApproval: MergeVerdict["pendingApproval"] = [];
+  const invalid: MergeVerdict["invalid"] = [];
+  const unsanctioned: MergeVerdict["unsanctioned"] = [];
+  const used = new Set<string>();
+
+  for (const r of results) {
+    if (r.passed) continue;
+    const failure = r.failure ?? "";
+    // Signature-pinned: the entry's panicSubstring must appear in the actual text.
+    const matches = manifest.entries.filter(
+      (e) => e.supersededSlice === r.slice && e.panicSubstring.length > 0 && failure.includes(e.panicSubstring),
+    );
+    if (matches.length === 0) {
+      unsanctioned.push({ slice: r.slice, reason: "no compat entry matches the failure signature — suspect a real merge defect" });
+      continue;
+    }
+    // Best outcome first: approved + replacement-proven → sanctioned.
+    const sanction = matches.find((e) => e.approved && passing(e.replacement.slice));
+    if (sanction) {
+      sanctioned.push({ slice: r.slice, entryId: sanction.id, supersededBy: sanction.supersededBy });
+      used.add(sanction.id);
+      continue;
+    }
+    // Replacement proven but not yet approved → pending (rule 3).
+    const pend = matches.find((e) => !e.approved && passing(e.replacement.slice));
+    if (pend) {
+      pendingApproval.push({ slice: r.slice, entryId: pend.id });
+      used.add(pend.id);
+      continue;
+    }
+    // Matched but the replacement invariant is not proven → invalid, blocks even
+    // if approved (rule 2: no supersession claim without a proven replacement).
+    // Distinguish "ran and failed" from "never reported" — saying a suite "did
+    // not pass" when it simply wasn't run is the same misleading verdict this
+    // whole feature replaces.
+    const inv = matches[0]!;
+    const reason = bySlice.has(inv.replacement.slice)
+      ? `replacement invariant ${inv.replacement.slice} did not pass — supersession unproven`
+      : `replacement suite ${inv.replacement.slice} was not in the reported results — cannot prove supersession; run and report it`;
+    invalid.push({ slice: r.slice, entryId: inv.id, reason });
+    used.add(inv.id);
+  }
+
+  const unused = manifest.entries.filter((e) => e.approved && !used.has(e.id)).map((e) => e.id);
+  const ok = unsanctioned.length === 0 && invalid.length === 0 && pendingApproval.length === 0;
+  return { ok, sanctioned, pendingApproval, invalid, unsanctioned, unused };
+}

package/src/mock/README.md

@@ -0,0 +1,40 @@
+# Mock testing harness
+
+This folder holds the **deterministic mock run** — a self-contained, no-network
+demo of the STZ pipeline. It is a testing and illustration aid, not part of the
+production path. The production harness runs in-session through the `/stz:*`
+commands and the `stz bridge` CLI (`src/bridge.ts`, `src/project.ts`); none of
+those depend on anything in this folder.
+
+## What is here
+
+- `orchestrator.ts` — a single-process pipeline that drives one slice through all
+  eight phases against a fake model layer. Used only by `stz run` and by
+  `test/orchestrator.test.ts`.
+- `interfaces.ts` — the model-layer seam (`Specimen`, `Judge`, `TestAuthor`,
+  `Documenter`, `ModelLayer`). A live TypeScript model implementation would
+  implement these; today only the mock does.
+- `mock.ts` — the deterministic `MockModelLayer`. Specimen quality is configured,
+  not sampled, so every run is reproducible (N6). The mock eval runner still runs
+  the **real** hack-detector, so the anti-reward-hacking layer is exercised for
+  real even though the model is fake.
+
+## Run it
+
+```bash
+stz run <dir>     # or: node bin/stz.mjs run <dir>
+```
+
+This drives the demo slice end to end: four specimens compete, a test-skipping
+specimen is disqualified by the hack-detector, a GRPO-weighted winner is chosen,
+and the full `.stz/` audit tree is written under `<dir>`. No API keys, no
+network, no subagents.
+
+## Why it exists
+
+The mock proves the deterministic spine (taxonomy, state, selection, GRPO,
+escalation, budget, pressure log, spec-diff, audit) end to end without spending
+tokens. The same spine powers the real in-session tournament; the only thing the
+mock replaces is the model layer. It is covered by `test/orchestrator.test.ts`
+(success path, bounded-escalation failure path, budget kill-switch, determinism,
+anti-hacking integration).

package/src/mock/interfaces.ts

@@ -0,0 +1,114 @@
+/**
+ * The LLM / subagent boundary (§3 Generative AI Layer).
+ *
+ * These interfaces are the seam between the deterministic STZ spine and the
+ * non-deterministic model layer. A live implementation invokes Claude Code /
+ * Codex subagents; the deterministic mock (./mock.ts) lets the whole pipeline
+ * run end-to-end in tests without any network call. Keeping the seam this thin
+ * is what makes "drop in a live impl" real rather than aspirational.
+ */
+import type {
+  DonePredicate,
+  EvalResult,
+  PairwiseVote,
+  SliceManifest,
+  SpecimenId,
+} from "../types.js";
+import type { Spec } from "../specdiff.js";
+
+/** What a specimen produces when it implements a slice (F6). */
+export interface SpecimenOutput {
+  specimen: SpecimenId;
+  /** relative path → file contents (the implementation diff, materialized). */
+  files: Record<string, string>;
+  /** Strategy label assigned by the diversification subagent (R5). */
+  strategy: string;
+}
+
+/** Elicitation subagent (F2): questionnaire → done predicates + complexity. */
+export interface Elicitor {
+  elicit(request: string): Promise<{
+    questionnaire: Record<string, string>;
+    donePredicates: DonePredicate[];
+    complexity: number;
+  }>;
+}
+
+/** Frozen test-author subagent (F10/L1): authors the sealed held-out suite. */
+export interface TestAuthor {
+  authorTests(manifest: SliceManifest): Promise<{
+    /** relative path → sealed test contents (judge-only, read-only). */
+    sealed: Record<string, string>;
+    rubric: string;
+  }>;
+}
+
+/** Strategy-diversification subagent (R5): N distinct implementation strategies. */
+export interface Strategist {
+  strategies(manifest: SliceManifest, n: number): Promise<string[]>;
+}
+
+/** An implementer specimen (F6). */
+export interface Specimen {
+  implement(
+    manifest: SliceManifest,
+    strategy: string,
+    /** Refinement context from the pressure log on a retry round (F9/F14). */
+    refinement: string | null,
+  ): Promise<SpecimenOutput>;
+}
+
+/** Eval runner (§3 Compute): runs sealed suite + coverage + mutation (F7/F11). */
+export interface EvalRunner {
+  evaluate(
+    output: SpecimenOutput,
+    sealed: Record<string, string>,
+  ): Promise<EvalResult>;
+}
+
+/** Judge subagent (F7 stage 2): one pairwise vote, frozen separate context. */
+export interface Judge {
+  vote(
+    a: SpecimenOutput,
+    b: SpecimenOutput,
+    sealed: Record<string, string>,
+  ): Promise<SpecimenId>;
+}
+
+/** Documenter subagent (F13): as-built spec from winning merged code. */
+export interface Documenter {
+  asBuilt(winner: SpecimenOutput): Promise<Spec>;
+}
+
+/** Planner subagent (F5): intent spec from the manifest. */
+export interface Planner {
+  intentSpec(manifest: SliceManifest): Promise<Spec>;
+}
+
+/** The full set of model-layer collaborators the orchestrator needs. */
+export interface ModelLayer {
+  elicitor: Elicitor;
+  testAuthor: TestAuthor;
+  strategist: Strategist;
+  specimen: Specimen;
+  evalRunner: EvalRunner;
+  judge: Judge;
+  documenter: Documenter;
+  planner: Planner;
+}
+
+/** Convenience: aggregate V votes for a pair into the list form (F7). */
+export async function votePair(
+  judge: Judge,
+  a: SpecimenOutput,
+  b: SpecimenOutput,
+  sealed: Record<string, string>,
+  votes: number,
+): Promise<PairwiseVote[]> {
+  const out: PairwiseVote[] = [];
+  for (let v = 0; v < votes; v++) {
+    const winner = await judge.vote(a, b, sealed);
+    out.push({ a: a.specimen, b: b.specimen, winner });
+  }
+  return out;
+}