testguard-cli 0.5.0__tar.gz → 0.6.0__tar.gz

{testguard_cli-0.5.0 → testguard_cli-0.6.0}/.github/scripts/sync-release-version.mjs

@@ -5,7 +5,7 @@
  *   - pyproject.toml         `version = "x.y.z"`
  *   - action.yml             default of the `version` input
  *   - packaging/homebrew/*.rb the tarball URL (sha256 is set after publish)
- *   - packaging/gitlab/*.yml  the include URL tag and TESTGUARD_VERSION
+ *   - packaging/gitlab/*.yml  the include URL tag and the `version` input default
  *
  * `--check` verifies instead of writing (used by release.yml).
  */
@@ -23,7 +23,7 @@ const surfaces = [
   ['packaging/homebrew/testguard.rb', /testguard-cli-\d+\.\d+\.\d+\.tgz/g, `testguard-cli-${version}.tgz`],
   ['README.md', /raccioly\/testguard@v\d+\.\d+\.\d+/g, `raccioly/testguard@v${version}`],
   ['packaging/gitlab/testguard.gitlab-ci.yml', /testguard\/v\d+\.\d+\.\d+\/packaging/g, `testguard/v${version}/packaging`],
-  ['packaging/gitlab/testguard.gitlab-ci.yml', /TESTGUARD_VERSION: "\d+\.\d+\.\d+"/, `TESTGUARD_VERSION: "${version}"`],
+  ['packaging/gitlab/testguard.gitlab-ci.yml', /(\n    version:\n      description: [^\n]*\n      default: ")\d+\.\d+\.\d+(")/, `$1${version}$2`],
 ];
 
 let drift = 0;

{testguard_cli-0.5.0 → testguard_cli-0.6.0}/.github/workflows/ci.yml

@@ -122,9 +122,41 @@ jobs:
             console.log('jest fixture verdicts match expected.json:', ev.records.length);
           "
 
+      # ── The MCP server, driven as a client would: handshake, list, call.
+      #    A harness-independent surface is only worth having if it actually
+      #    speaks the protocol from a cold start. ──
+      - name: mcp — handshake, tools/list, tools/call over stdio
+        shell: bash
+        run: |
+          printf '%s\n' \
+            '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}' \
+            '{"jsonrpc":"2.0","method":"notifications/initialized"}' \
+            '{"jsonrpc":"2.0","id":2,"method":"tools/list"}' \
+            '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"testguard_status","arguments":{"dir":"."}}}' \
+            | node cli/testguard.mjs mcp > "$RUNNER_TEMP/mcp.jsonl"
+          node --input-type=module -e "
+            import { readFileSync } from 'node:fs';
+            const msgs = readFileSync(process.env.RUNNER_TEMP + '/mcp.jsonl', 'utf8').trim().split('\n').map((l) => JSON.parse(l));
+            if (msgs.length !== 3) { console.error('expected 3 replies (the notification is never answered), got', msgs.length); process.exit(1); }
+            const [init, list, call] = msgs;
+            if (!init.result.protocolVersion) { console.error('no protocolVersion'); process.exit(1); }
+            if (list.result.tools.length !== 5) { console.error('expected 5 tools, got', list.result.tools.length); process.exit(1); }
+            if (!list.result.tools.every((t) => t.annotations.readOnlyHint)) { console.error('a tool is not marked read-only'); process.exit(1); }
+            if (call.result.isError) { console.error('status call failed:', call.result.content[0].text); process.exit(1); }
+            if (!call.result.structuredContent.next.action) { console.error('status carried no next action'); process.exit(1); }
+            console.log('mcp:', list.result.tools.map((t) => t.name).join(', '), '→ state', call.result.structuredContent.state);
+          "
+
       # ── Every change needs a claim: the gate runs on this repository's own
       #    pull requests. A new source file without a claim (or an excusing
       #    ignore entry with a reason) turns the PR red. ──
+      # A claim that disappeared is invisible to every other check: `probe`
+      # verifies what is there, `gate` sees the file covered by another claim.
+      # Removal is allowed; it is never silent.
+      - name: claims — no claim or fault removed without a reason
+        if: github.event_name == 'pull_request'
+        run: node cli/testguard.mjs claims . --since "origin/${{ github.base_ref }}"
+
       - name: gate — every changed source file carries a claim
         if: github.event_name == 'pull_request'
         run: node cli/testguard.mjs gate . --changed "origin/${{ github.base_ref }}"
@@ -132,17 +164,89 @@ jobs:
       # ── Self-verification: TestGuard is governed by the standard it defines.
       #    testguard.claims.json states invariants of the tool itself; every
       #    fault must be killed by the tool's own tests. Exit 0 = all defended. ──
+      #    Runs on ONE Node leg: the verdicts do not depend on the Node minor,
+      #    and each from-scratch probe is ~14 minutes. The previous run's
+      #    evidence is restored from the cache so claims whose target and
+      #    defenders did not change reuse their verdict (the tool's own
+      #    reuse rule); only what changed is re-probed. No escalation in CI:
+      #    a survivor here is a failure either way. ──
+      - name: Restore previous self-probe evidence (verdict reuse)
+        if: matrix.node-version == 22
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: .testguard/ci-self-evidence.json
+          key: self-evidence-${{ hashFiles('testguard.claims.json') }}-${{ github.sha }}
+          restore-keys: |
+            self-evidence-${{ hashFiles('testguard.claims.json') }}-
+            self-evidence-
+
       - name: probe TestGuard's own claims
-        run: node cli/testguard.mjs probe . --budget 180000 --out "$RUNNER_TEMP/self-evidence.json"
+        if: matrix.node-version == 22
+        run: node cli/testguard.mjs probe . --budget 180000 --no-escalate --out .testguard/ci-self-evidence.json
+
+      # A stable, branch-named artifact so a developer can fetch the evidence
+      # CI produced without knowing a run id (README: read CI's evidence
+      # locally). One Node leg is enough; the verdicts are identical.
+      - name: Upload evidence under a stable, branch-named artifact
+        if: always() && matrix.node-version == 22 && github.event_name == 'push'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: testguard-evidence-${{ github.ref_name }}
+          path: .testguard/ci-self-evidence.json
+          if-no-files-found: warn
 
       - name: Upload self-probe evidence
-        if: always()
+        if: always() && matrix.node-version == 22
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: testguard-self-evidence-node-${{ matrix.node-version }}
-          path: ${{ runner.temp }}/self-evidence.json
+          name: testguard-self-evidence
+          path: .testguard/ci-self-evidence.json
           if-no-files-found: warn
 
+  # ── Playwright: the per-file runner is proven against its own browserless
+  #    fixture, whose dependencies live in the fixture (never in the tool's
+  #    package.json). Installed and run here only. ──
+  playwright:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: |
+            package-lock.json
+            fixtures/known-answer-playwright/package-lock.json
+      - name: Install (tool + fixture)
+        run: |
+          npm ci
+          npm ci --prefix fixtures/known-answer-playwright
+      - name: Playwright adapter units + mixed-defender acceptance
+        run: npx vitest run test/runner-playwright.test.mjs test/probe.playwright.test.mjs
+      - name: probe the playwright fixture through the bin (mixed defenders, auto runner)
+        shell: bash
+        run: |
+          set -o pipefail
+          S=$(mktemp -d)
+          cp -R fixtures/known-answer-playwright/. "$S/"
+          rm -rf "$S/node_modules" "$S/.flake-counter" "$S/.testguard" "$S/test-results"
+          ln -s "$GITHUB_WORKSPACE/fixtures/known-answer-playwright/node_modules" "$S/node_modules"
+          git -C "$S" init -q && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci add -A && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci commit -qm fixture
+          status=0; node cli/testguard.mjs probe "$S" --budget 60000 --quiet --no-escalate || status=$?
+          [ "$status" -eq 1 ] || { echo "::error::expected exit 1, got $status"; exit 1; }
+          node --input-type=module -e "
+            import { readFileSync } from 'node:fs';
+            const ev = JSON.parse(readFileSync('$S/.testguard/evidence.json','utf8'));
+            const exp = JSON.parse(readFileSync('fixtures/known-answer-playwright/expected.json','utf8')).expected;
+            const names = (ev.run.runners ?? []).map(r => r.name).sort().join(',');
+            if (names !== 'playwright,vitest') { console.error('runners were', names); process.exit(1); }
+            const bad = ev.records.filter(r => exp[r.claim.id+'/'+r.subject.id].verdict !== r.verdict);
+            if (bad.length) { console.error('MISMATCH', bad.map(r => r.claim.id+'/'+r.subject.id+':'+r.verdict)); process.exit(1); }
+            console.log('playwright fixture verdicts match expected.json:', ev.records.length);
+          "
+
   # ── npm publish dry-run on tags ──
   publish-check:
     runs-on: ubuntu-latest

{testguard_cli-0.5.0 → testguard_cli-0.6.0}/.gitignore

@@ -9,6 +9,7 @@ coverage/
 .testguard/evidence-partial.json
 .testguard/brief.json
 .testguard/gate.json
+.testguard/ci-self-evidence.json
 .testguard/scaffold-*.json
 # Local agent tooling — not part of the public project
 .wolf/

{testguard_cli-0.5.0 → testguard_cli-0.6.0}/AGENTS.md

@@ -27,11 +27,12 @@ node cli/testguard.mjs scaffold src/probe/classify.mjs --json   # what the produ
 - `spec/schemas/common.schema.json` — verdicts, fault classes, provenance. The closed sets.
 - `spec/lib/validate.mjs` — semantic rules (green baseline, N/N agreement, assertion-only kills, fingerprint derivation).
 - `src/probe/classify.mjs` — the verdict function. Pure. Its check order *is* `GATE-SEMANTICS.md`.
+- `src/probe/mocks.mjs` — mock-awareness: which candidate tests mock the target (never defenders), and the `mocked-never-asserted` signal. `src/probe/rank.mjs` resolves aliases (tsconfig `paths` through `extends`/`references`, vite/vitest `resolve.alias`, package.json `imports`).
 - `src/probe/probe.mjs` — orchestrator: isolation → baseline (cached per defender set) → apply → probe → escalate → restore → classify → rank.
 - `fixtures/known-answer/expected.json` — the oracle. Never edit it to match output.
 - `testguard.claims.json` — claims about this codebase, probed in CI.
-- `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
-- `src/probe/runners/` — one module per runner (`name`, `testGlobs`, `check`, `tests`, `run`) over `shared.mjs` (jest-compatible report parsing, budgeted process runner). A new runner needs its own fixture copy with the same `expected.json`.
+- `src/scaffold/producers.mjs` — the nine fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
+- `src/probe/runners/` — one module per runner (`name`, `testGlobs`, `check`, `tests`, `run`, optionally `owns` for a per-file runner) over `shared.mjs` (jest-compatible report parsing, budgeted process runner, pluggable `parse`/`env`). `playwright.mjs` is a per-file runner: it owns the files under its `testDir`; `index.mjs` partitions defenders by runner and merges their runs pessimistically. A new runner needs its own fixture copy with its own `expected.json`, verified by hand.
 - `src/gate/changed.mjs` — claim coverage of a change (`gate --changed`). File-level, delta-only; an uncovered file exits 1; every ignore reliance is reported. Its rules are a section of `GATE-SEMANTICS.md`.
 - `src/status/status.mjs` — the state machine every rendering derives from. A new state or action is a spec change (`status.schema.json`) and a skill-template change (`src/init/templates/SKILL.md`) in the same PR.