testguard-cli 0.4.0__tar.gz → 0.6.0__tar.gz

{testguard_cli-0.4.0 → testguard_cli-0.6.0}/.github/scripts/sync-release-version.mjs

@@ -5,6 +5,7 @@
  *   - pyproject.toml         `version = "x.y.z"`
  *   - action.yml             default of the `version` input
  *   - packaging/homebrew/*.rb the tarball URL (sha256 is set after publish)
+ *   - packaging/gitlab/*.yml  the include URL tag and the `version` input default
  *
  * `--check` verifies instead of writing (used by release.yml).
  */
@@ -21,6 +22,8 @@ const surfaces = [
   ['action.yml', /(\n  version:\n    description: [^\n]*\n    required: false\n    default: ')[^']*(')/, `$1${version}$2`],
   ['packaging/homebrew/testguard.rb', /testguard-cli-\d+\.\d+\.\d+\.tgz/g, `testguard-cli-${version}.tgz`],
   ['README.md', /raccioly\/testguard@v\d+\.\d+\.\d+/g, `raccioly/testguard@v${version}`],
+  ['packaging/gitlab/testguard.gitlab-ci.yml', /testguard\/v\d+\.\d+\.\d+\/packaging/g, `testguard/v${version}/packaging`],
+  ['packaging/gitlab/testguard.gitlab-ci.yml', /(\n    version:\n      description: [^\n]*\n      default: ")\d+\.\d+\.\d+(")/, `$1${version}$2`],
 ];
 
 let drift = 0;

testguard_cli-0.6.0/.github/workflows/ci.yml

@@ -0,0 +1,260 @@
+name: TestGuard CI
+
+on:
+  workflow_dispatch: # manual recovery for pushes that don't auto-trigger (e.g. release PRs opened by GITHUB_TOKEN)
+  push:
+    branches: [main]
+    tags: ['v*'] # publish-check below runs on tags only
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # 20 is the floor (node:util.parseArgs). 24 ships npm 11 — keep it so
+        # `npm ci` exercises the npm that treats some config as fatal where
+        # npm 10 is silent. Do not drop 24.
+        node-version: [20, 22, 24]
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0 # probe builds a git worktree from HEAD; the self-probe below needs history
+
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install
+        run: npm ci
+
+      # ── Syntax-check EVERY source, not just what the entry point eagerly
+      #    imports; a lazily loaded module's parse error would otherwise slip. ──
+      - name: Syntax-check all sources
+        run: find cli src spec -name '*.mjs' -print0 | xargs -0 -n1 node --check
+
+      # ── Install-from-tarball smoke: the package must run as INSTALLED, with
+      #    production dependencies only. 0.1.0 shipped unrunnable because the
+      #    schema validator's dependency was a devDependency; the suite, which
+      #    runs from the checkout, could not see it. ──
+      - name: Install smoke (pack → install --omit=dev → run)
+        run: npm run test:install
+
+      - name: Python wrapper imports (PyPI surface)
+        run: python3 -c "import testguard_cli.wrapper; print('testguard_cli.wrapper OK')"
+
+      # ── Test suite with a RUNTIME BUDGET. A test that walks the wrong tree
+      #    or probes far more than it should still passes — it just takes
+      #    forever. Only a duration gate catches that class. Normal: ~15s
+      #    locally, ~30–60s here. Budget is loose on purpose. ──
+      - name: Run tests (with runtime budget)
+        shell: bash
+        env:
+          TEST_BUDGET_MS: 240000
+        run: |
+          START=$(date +%s%3N)
+          npm test
+          DURATION=$(( $(date +%s%3N) - START ))
+          echo "Test suite duration: ${DURATION}ms (budget: ${TEST_BUDGET_MS}ms)"
+          if [ "$DURATION" -gt "$TEST_BUDGET_MS" ]; then
+            echo "::error::Test suite took ${DURATION}ms, over the ${TEST_BUDGET_MS}ms budget — something is probing far more than it should."
+            exit 1
+          fi
+
+      # ── Dogfood: the real binary against the known-answer fixture, as a
+      #    user would run it (the suite drives main() in-process). ──
+      - name: CLI version
+        run: node cli/testguard.mjs --version
+
+      - name: claims — fixture lists cleanly, no drift
+        run: node cli/testguard.mjs claims fixtures/known-answer
+
+      - name: probe → baseline → probe → brief on the fixture (through the bin)
+        shell: bash
+        run: |
+          set -o pipefail
+          S=$(mktemp -d)
+          cp -R fixtures/known-answer/. "$S/"
+          rm -rf "$S/node_modules" "$S/.flake-counter" "$S/.testguard"
+          ln -s "$GITHUB_WORKSPACE/node_modules" "$S/node_modules"
+          git -C "$S" init -q
+          git -C "$S" -c user.email=ci@example.invalid -c user.name=ci add -A
+          git -C "$S" -c user.email=ci@example.invalid -c user.name=ci commit -qm fixture
+          B="$GITHUB_WORKSPACE/cli/testguard.mjs"
+          status=0; node "$B" probe "$S" --budget 60000 --quiet || status=$?
+          [ "$status" -eq 1 ] || { echo "::error::expected exit 1 (unproven claims), got $status"; exit 1; }
+          node "$B" baseline "$S"
+          status=0; node "$B" probe "$S" --budget 60000 --quiet || status=$?
+          [ "$status" -eq 0 ] || { echo "::error::expected exit 0 after baseline, got $status"; exit 1; }
+          node "$B" brief "$S" --text | head -12
+          # every verdict must appear exactly once per expected.json
+          node --input-type=module -e "
+            import { readFileSync } from 'node:fs';
+            const ev = JSON.parse(readFileSync('$S/.testguard/evidence.json','utf8'));
+            const exp = JSON.parse(readFileSync('fixtures/known-answer/expected.json','utf8')).expected;
+            const bad = ev.records.filter(r => exp[r.claim.id+'/'+r.subject.id].verdict !== r.verdict);
+            if (bad.length) { console.error('MISMATCH', bad.map(r => r.claim.id+'/'+r.subject.id+':'+r.verdict)); process.exit(1); }
+            console.log('fixture verdicts match expected.json:', ev.records.length);
+          "
+
+      - name: probe the jest fixture (through the bin, --runner jest)
+        shell: bash
+        run: |
+          S=$(mktemp -d)
+          cp -R fixtures/known-answer-jest/. "$S/"
+          rm -rf "$S/node_modules" "$S/.flake-counter" "$S/.testguard"
+          ln -s "$GITHUB_WORKSPACE/node_modules" "$S/node_modules"
+          git -C "$S" init -q && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci add -A && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci commit -qm fixture
+          status=0; node cli/testguard.mjs probe "$S" --runner jest --budget 60000 --quiet --no-escalate || status=$?
+          [ "$status" -eq 1 ] || { echo "::error::expected exit 1, got $status"; exit 1; }
+          node --input-type=module -e "
+            import { readFileSync } from 'node:fs';
+            const ev = JSON.parse(readFileSync('$S/.testguard/evidence.json','utf8'));
+            const exp = JSON.parse(readFileSync('fixtures/known-answer-jest/expected.json','utf8')).expected;
+            if (ev.run.runner.name !== 'jest') { console.error('runner was', ev.run.runner.name); process.exit(1); }
+            const bad = ev.records.filter(r => exp[r.claim.id+'/'+r.subject.id].verdict !== r.verdict);
+            if (bad.length) { console.error('MISMATCH', bad.map(r => r.claim.id+'/'+r.subject.id+':'+r.verdict)); process.exit(1); }
+            console.log('jest fixture verdicts match expected.json:', ev.records.length);
+          "
+
+      # ── The MCP server, driven as a client would: handshake, list, call.
+      #    A harness-independent surface is only worth having if it actually
+      #    speaks the protocol from a cold start. ──
+      - name: mcp — handshake, tools/list, tools/call over stdio
+        shell: bash
+        run: |
+          printf '%s\n' \
+            '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
+            '{"jsonrpc":"2.0","method":"notifications/initialized"}' \
+            '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \
+            '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"testguard_status","arguments":{"dir":"."}}}' \
+            | node cli/testguard.mjs mcp > "$RUNNER_TEMP/mcp.jsonl"
+          node --input-type=module -e "
+            import { readFileSync } from 'node:fs';
+            const msgs = readFileSync(process.env.RUNNER_TEMP + '/mcp.jsonl', 'utf8').trim().split('\n').map((l) => JSON.parse(l));
+            if (msgs.length !== 3) { console.error('expected 3 replies (the notification is never answered), got', msgs.length); process.exit(1); }
+            const [init, list, call] = msgs;
+            if (!init.result.protocolVersion) { console.error('no protocolVersion'); process.exit(1); }
+            if (list.result.tools.length !== 5) { console.error('expected 5 tools, got', list.result.tools.length); process.exit(1); }
+            if (!list.result.tools.every((t) => t.annotations.readOnlyHint)) { console.error('a tool is not marked read-only'); process.exit(1); }
+            if (call.result.isError) { console.error('status call failed:', call.result.content[0].text); process.exit(1); }
+            if (!call.result.structuredContent.next.action) { console.error('status carried no next action'); process.exit(1); }
+            console.log('mcp:', list.result.tools.map((t) => t.name).join(', '), '→ state', call.result.structuredContent.state);
+          "
+
+      # ── Every change needs a claim: the gate runs on this repository's own
+      #    pull requests. A new source file without a claim (or an excusing
+      #    ignore entry with a reason) turns the PR red. ──
+      # A claim that disappeared is invisible to every other check: `probe`
+      # verifies what is there, `gate` sees the file covered by another claim.
+      # Removal is allowed; it is never silent.
+      - name: claims — no claim or fault removed without a reason
+        if: github.event_name == 'pull_request'
+        run: node cli/testguard.mjs claims . --since "origin/${{ github.base_ref }}"
+
+      - name: gate — every changed source file carries a claim
+        if: github.event_name == 'pull_request'
+        run: node cli/testguard.mjs gate . --changed "origin/${{ github.base_ref }}"
+
+      # ── Self-verification: TestGuard is governed by the standard it defines.
+      #    testguard.claims.json states invariants of the tool itself; every
+      #    fault must be killed by the tool's own tests. Exit 0 = all defended. ──
+      #    Runs on ONE Node leg: the verdicts do not depend on the Node minor,
+      #    and each from-scratch probe is ~14 minutes. The previous run's
+      #    evidence is restored from the cache so claims whose target and
+      #    defenders did not change reuse their verdict (the tool's own
+      #    reuse rule); only what changed is re-probed. No escalation in CI:
+      #    a survivor here is a failure either way. ──
+      - name: Restore previous self-probe evidence (verdict reuse)
+        if: matrix.node-version == 22
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: .testguard/ci-self-evidence.json
+          key: self-evidence-${{ hashFiles('testguard.claims.json') }}-${{ github.sha }}
+          restore-keys: |
+            self-evidence-${{ hashFiles('testguard.claims.json') }}-
+            self-evidence-
+
+      - name: probe TestGuard's own claims
+        if: matrix.node-version == 22
+        run: node cli/testguard.mjs probe . --budget 180000 --no-escalate --out .testguard/ci-self-evidence.json
+
+      # A stable, branch-named artifact so a developer can fetch the evidence
+      # CI produced without knowing a run id (README: read CI's evidence
+      # locally). One Node leg is enough; the verdicts are identical.
+      - name: Upload evidence under a stable, branch-named artifact
+        if: always() && matrix.node-version == 22 && github.event_name == 'push'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: testguard-evidence-${{ github.ref_name }}
+          path: .testguard/ci-self-evidence.json
+          if-no-files-found: warn
+
+      - name: Upload self-probe evidence
+        if: always() && matrix.node-version == 22
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: testguard-self-evidence
+          path: .testguard/ci-self-evidence.json
+          if-no-files-found: warn
+
+  # ── Playwright: the per-file runner is proven against its own browserless
+  #    fixture, whose dependencies live in the fixture (never in the tool's
+  #    package.json). Installed and run here only. ──
+  playwright:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: |
+            package-lock.json
+            fixtures/known-answer-playwright/package-lock.json
+      - name: Install (tool + fixture)
+        run: |
+          npm ci
+          npm ci --prefix fixtures/known-answer-playwright
+      - name: Playwright adapter units + mixed-defender acceptance
+        run: npx vitest run test/runner-playwright.test.mjs test/probe.playwright.test.mjs
+      - name: probe the playwright fixture through the bin (mixed defenders, auto runner)
+        shell: bash
+        run: |
+          set -o pipefail
+          S=$(mktemp -d)
+          cp -R fixtures/known-answer-playwright/. "$S/"
+          rm -rf "$S/node_modules" "$S/.flake-counter" "$S/.testguard" "$S/test-results"
+          ln -s "$GITHUB_WORKSPACE/fixtures/known-answer-playwright/node_modules" "$S/node_modules"
+          git -C "$S" init -q && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci add -A && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci commit -qm fixture
+          status=0; node cli/testguard.mjs probe "$S" --budget 60000 --quiet --no-escalate || status=$?
+          [ "$status" -eq 1 ] || { echo "::error::expected exit 1, got $status"; exit 1; }
+          node --input-type=module -e "
+            import { readFileSync } from 'node:fs';
+            const ev = JSON.parse(readFileSync('$S/.testguard/evidence.json','utf8'));
+            const exp = JSON.parse(readFileSync('fixtures/known-answer-playwright/expected.json','utf8')).expected;
+            const names = (ev.run.runners ?? []).map(r => r.name).sort().join(',');
+            if (names !== 'playwright,vitest') { console.error('runners were', names); process.exit(1); }
+            const bad = ev.records.filter(r => exp[r.claim.id+'/'+r.subject.id].verdict !== r.verdict);
+            if (bad.length) { console.error('MISMATCH', bad.map(r => r.claim.id+'/'+r.subject.id+':'+r.verdict)); process.exit(1); }
+            console.log('playwright fixture verdicts match expected.json:', ev.records.length);
+          "
+
+  # ── npm publish dry-run on tags ──
+  publish-check:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs: test
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 24
+      - run: npm publish --dry-run

{testguard_cli-0.4.0 → testguard_cli-0.6.0}/.gitignore

@@ -8,6 +8,8 @@ coverage/
 .testguard/evidence-provisional.json
 .testguard/evidence-partial.json
 .testguard/brief.json
+.testguard/gate.json
+.testguard/ci-self-evidence.json
 .testguard/scaffold-*.json
 # Local agent tooling — not part of the public project
 .wolf/

{testguard_cli-0.4.0 → testguard_cli-0.6.0}/.pre-commit-hooks.yaml

@@ -10,6 +10,7 @@
 #         rev: v0.1.0
 #         hooks:
 #           - id: testguard-claims        # fast: validates the claims file + drift
+#           - id: testguard-gate          # fast: every changed source file must carry a claim
 #           - id: testguard-probe         # slow: pre-push stage, runs the faults
 #
 #     # install the push-stage hook once per clone:
@@ -23,6 +24,14 @@
   pass_filenames: false
   always_run: true
 
+- id: testguard-gate
+  name: TestGuard gate (every change needs a claim)
+  description: Fails when a source file in this commit carries no claim, does not defend one, and is not excused by testguard.ignore.json. Fast; runs no tests.
+  entry: testguard gate --changed HEAD --include-dirty --quiet
+  language: node
+  pass_filenames: false
+  always_run: true
+
 - id: testguard-probe
   name: TestGuard probe (pre-push)
   description: Injects every claim's faults and fails on any the tests miss. Gated by the committed baseline.

{testguard_cli-0.4.0 → testguard_cli-0.6.0}/AGENTS.md

@@ -27,11 +27,13 @@ node cli/testguard.mjs scaffold src/probe/classify.mjs --json   # what the produ
 - `spec/schemas/common.schema.json` — verdicts, fault classes, provenance. The closed sets.
 - `spec/lib/validate.mjs` — semantic rules (green baseline, N/N agreement, assertion-only kills, fingerprint derivation).
 - `src/probe/classify.mjs` — the verdict function. Pure. Its check order *is* `GATE-SEMANTICS.md`.
+- `src/probe/mocks.mjs` — mock-awareness: which candidate tests mock the target (never defenders), and the `mocked-never-asserted` signal. `src/probe/rank.mjs` resolves aliases (tsconfig `paths` through `extends`/`references`, vite/vitest `resolve.alias`, package.json `imports`).
 - `src/probe/probe.mjs` — orchestrator: isolation → baseline (cached per defender set) → apply → probe → escalate → restore → classify → rank.
 - `fixtures/known-answer/expected.json` — the oracle. Never edit it to match output.
 - `testguard.claims.json` — claims about this codebase, probed in CI.
-- `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
-- `src/probe/runners/` — one module per runner (`name`, `testGlobs`, `check`, `tests`, `run`) over `shared.mjs` (jest-compatible report parsing, budgeted process runner). A new runner needs its own fixture copy with the same `expected.json`.
+- `src/scaffold/producers.mjs` — the nine fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
+- `src/probe/runners/` — one module per runner (`name`, `testGlobs`, `check`, `tests`, `run`, optionally `owns` for a per-file runner) over `shared.mjs` (jest-compatible report parsing, budgeted process runner, pluggable `parse`/`env`). `playwright.mjs` is a per-file runner: it owns the files under its `testDir`; `index.mjs` partitions defenders by runner and merges their runs pessimistically. A new runner needs its own fixture copy with its own `expected.json`, verified by hand.
+- `src/gate/changed.mjs` — claim coverage of a change (`gate --changed`). File-level, delta-only; an uncovered file exits 1; every ignore reliance is reported. Its rules are a section of `GATE-SEMANTICS.md`.
 - `src/status/status.mjs` — the state machine every rendering derives from. A new state or action is a spec change (`status.schema.json`) and a skill-template change (`src/init/templates/SKILL.md`) in the same PR.
 
 ## Rules