testguard-cli 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/scripts/sync-release-version.mjs +3 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/workflows/ci.yml +7 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.gitignore +1 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.pre-commit-hooks.yaml +9 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/AGENTS.md +1 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/CHANGELOG.md +36 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/CONTRIBUTING.md +1 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/PKG-INFO +55 -5
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/README.md +54 -4
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/action.yml +19 -2
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/bench/README.md +19 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/package-lock.json +2 -2
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/package.json +1 -1
- testguard_cli-0.5.0/packaging/gitlab/testguard.gitlab-ci.yml +47 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/packaging/homebrew/testguard.rb +1 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/pyproject.toml +1 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/GATE-SEMANTICS.md +51 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/README.md +3 -2
- testguard_cli-0.5.0/spec/conformance/examples/gate.json +38 -0
- testguard_cli-0.5.0/spec/conformance/invalid/gate.counts-disagree.json +20 -0
- testguard_cli-0.5.0/spec/conformance/invalid/gate.exit-zero-with-uncovered.json +20 -0
- testguard_cli-0.5.0/spec/conformance/invalid/gate.ignore-cover-without-pattern.json +20 -0
- testguard_cli-0.5.0/spec/conformance/invalid/status.claim-without-file.json +29 -0
- testguard_cli-0.5.0/spec/conformance/invalid/status.unclaimed-hidden-behind-unproven.json +40 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/lib/validate.mjs +19 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/brief.schema.json +11 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/common.schema.json +27 -0
- testguard_cli-0.5.0/spec/schemas/gate.schema.json +52 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/status.schema.json +26 -4
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/brief/brief.mjs +26 -2
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/cli.mjs +17 -3
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/brief.mjs +18 -10
- testguard_cli-0.5.0/src/commands/gate.mjs +47 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/probe.mjs +2 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/status.mjs +2 -1
- testguard_cli-0.5.0/src/gate/changed.mjs +222 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/init/init.mjs +1 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/init/templates/SKILL.md +19 -1
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/status/status.mjs +42 -2
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/brief.test.mjs +20 -1
- testguard_cli-0.5.0/test/gate.test.mjs +227 -0
- testguard_cli-0.5.0/test/release-sync.test.mjs +57 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/status.test.mjs +29 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/testguard.claims.json +288 -0
- testguard_cli-0.5.0/testguard.ignore.json +34 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.codex/hooks.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.gitattributes +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/dependabot.yml +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/scripts/install-smoke.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/workflows/auto-merge.yml +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/workflows/release.yml +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/workflows/scheduled-release.yml +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.github/workflows/supply-chain.yml +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.npmignore +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.npmrc +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.testguard/baseline.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/.testguard/status.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/CLAUDE.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/LICENSE +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/PRIVACY.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/SECURITY.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/SUPPORT.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/cli/testguard.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/README.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/expected.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/testguard.claims.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/.gitignore +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/README.md +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/expected.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/jest.config.js +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/package.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/src/export.js +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/src/redact.js +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/test/flaky.test.js +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/test/redact.test.js +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/fixtures/known-answer-jest/testguard.claims.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/baseline.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/evidence-provisional.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/evidence.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/examples/status.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.provisional-flag-on-confirmed-run.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.single-run-without-provisional.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/status.clean-with-new-findings.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/invalid/status.write-test-without-target.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/conformance/schemas.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/baseline.schema.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/claims.schema.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/evidence.schema.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/baseline/baseline.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/claims/load.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/baseline.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/claims.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/init.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/commands/scaffold.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/git.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/classify.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/discover.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/probe.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/rank.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/runners/index.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/runners/jest.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/runners/shared.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/runners/vitest.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/probe/worktree.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/render.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/scaffold/producers.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/scaffold/scaffold.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/util/glob.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/src/util/hash.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/classify.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/discover.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/glob.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/init.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/inject.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/probe-preconditions.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/probe.fixture.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/probe.jest.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/rank-aliases.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/runner-command.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/runner-vitest.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/scaffold.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/snapshot.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/worktree.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/test/writer.test.mjs +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.4.0 → testguard_cli-0.5.0}/vitest.config.mjs +0 -0
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* - pyproject.toml `version = "x.y.z"`
|
|
6
6
|
* - action.yml default of the `version` input
|
|
7
7
|
* - packaging/homebrew/*.rb the tarball URL (sha256 is set after publish)
|
|
8
|
+
* - packaging/gitlab/*.yml the include URL tag and TESTGUARD_VERSION
|
|
8
9
|
*
|
|
9
10
|
* `--check` verifies instead of writing (used by release.yml).
|
|
10
11
|
*/
|
|
@@ -21,6 +22,8 @@ const surfaces = [
|
|
|
21
22
|
['action.yml', /(\n version:\n description: [^\n]*\n required: false\n default: ')[^']*(')/, `$1${version}$2`],
|
|
22
23
|
['packaging/homebrew/testguard.rb', /testguard-cli-\d+\.\d+\.\d+\.tgz/g, `testguard-cli-${version}.tgz`],
|
|
23
24
|
['README.md', /raccioly\/testguard@v\d+\.\d+\.\d+/g, `raccioly/testguard@v${version}`],
|
|
25
|
+
['packaging/gitlab/testguard.gitlab-ci.yml', /testguard\/v\d+\.\d+\.\d+\/packaging/g, `testguard/v${version}/packaging`],
|
|
26
|
+
['packaging/gitlab/testguard.gitlab-ci.yml', /TESTGUARD_VERSION: "\d+\.\d+\.\d+"/, `TESTGUARD_VERSION: "${version}"`],
|
|
24
27
|
];
|
|
25
28
|
|
|
26
29
|
let drift = 0;
|
|
@@ -122,6 +122,13 @@ jobs:
|
|
|
122
122
|
console.log('jest fixture verdicts match expected.json:', ev.records.length);
|
|
123
123
|
"
|
|
124
124
|
|
|
125
|
+
# ── Every change needs a claim: the gate runs on this repository's own
|
|
126
|
+
# pull requests. A new source file without a claim (or an excusing
|
|
127
|
+
# ignore entry with a reason) turns the PR red. ──
|
|
128
|
+
- name: gate — every changed source file carries a claim
|
|
129
|
+
if: github.event_name == 'pull_request'
|
|
130
|
+
run: node cli/testguard.mjs gate . --changed "origin/${{ github.base_ref }}"
|
|
131
|
+
|
|
125
132
|
# ── Self-verification: TestGuard is governed by the standard it defines.
|
|
126
133
|
# testguard.claims.json states invariants of the tool itself; every
|
|
127
134
|
# fault must be killed by the tool's own tests. Exit 0 = all defended. ──
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
# rev: v0.1.0
|
|
11
11
|
# hooks:
|
|
12
12
|
# - id: testguard-claims # fast: validates the claims file + drift
|
|
13
|
+
# - id: testguard-gate # fast: every changed source file must carry a claim
|
|
13
14
|
# - id: testguard-probe # slow: pre-push stage, runs the faults
|
|
14
15
|
#
|
|
15
16
|
# # install the push-stage hook once per clone:
|
|
@@ -23,6 +24,14 @@
|
|
|
23
24
|
pass_filenames: false
|
|
24
25
|
always_run: true
|
|
25
26
|
|
|
27
|
+
- id: testguard-gate
|
|
28
|
+
name: TestGuard gate (every change needs a claim)
|
|
29
|
+
description: Fails when a source file in this commit carries no claim, does not defend one, and is not excused by testguard.ignore.json. Fast; runs no tests.
|
|
30
|
+
entry: testguard gate --changed HEAD --include-dirty --quiet
|
|
31
|
+
language: node
|
|
32
|
+
pass_filenames: false
|
|
33
|
+
always_run: true
|
|
34
|
+
|
|
26
35
|
- id: testguard-probe
|
|
27
36
|
name: TestGuard probe (pre-push)
|
|
28
37
|
description: Injects every claim's faults and fails on any the tests miss. Gated by the committed baseline.
|
|
@@ -32,6 +32,7 @@ node cli/testguard.mjs scaffold src/probe/classify.mjs --json # what the produ
|
|
|
32
32
|
- `testguard.claims.json` — claims about this codebase, probed in CI.
|
|
33
33
|
- `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
|
|
34
34
|
- `src/probe/runners/` — one module per runner (`name`, `testGlobs`, `check`, `tests`, `run`) over `shared.mjs` (jest-compatible report parsing, budgeted process runner). A new runner needs its own fixture copy with the same `expected.json`.
|
|
35
|
+
- `src/gate/changed.mjs` — claim coverage of a change (`gate --changed`). File-level, delta-only; an uncovered file exits 1; every ignore reliance is reported. Its rules are a section of `GATE-SEMANTICS.md`.
|
|
35
36
|
- `src/status/status.mjs` — the state machine every rendering derives from. A new state or action is a spec change (`status.schema.json`) and a skill-template change (`src/init/templates/SKILL.md`) in the same PR.
|
|
36
37
|
|
|
37
38
|
## Rules
|
|
@@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.5.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
The change gate: unclaimed code is now a finding.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- **`testguard gate --changed <ref>`** — the change gate. Every escaped
|
|
17
|
+
defect in the field reports was a *claim gap*: the feature shipped green
|
|
18
|
+
with zero claims, and `probe` is silent about unclaimed code by
|
|
19
|
+
construction. `gate` measures the files changed since
|
|
20
|
+
`merge-base(ref, HEAD)` (or in the working tree with `--include-dirty`)
|
|
21
|
+
and exits `1` when any changed source file carries no fault, does not
|
|
22
|
+
resolve as a defender (test files), and is not excused by an unexpired
|
|
23
|
+
`path` entry in `testguard.ignore.json`. Every reliance on an ignore entry
|
|
24
|
+
is printed with its reason; expired entries excuse nothing. Non-source
|
|
25
|
+
files and documented never-claimed patterns are excluded and listed
|
|
26
|
+
(`--explain`, `--exclude <glob>`); `--strict` fails a change that
|
|
27
|
+
evaluated nothing. The base branch is detected in GitHub Actions and
|
|
28
|
+
GitLab CI (`TESTGUARD_CHANGED_REF` overrides). A new spec kind,
|
|
29
|
+
`gate.schema.json`, with conformance examples.
|
|
30
|
+
- **`status --changed <ref>`** — a new state `unclaimed-changes` and action
|
|
31
|
+
`claim` that precede every evidence state; `next.file` names the first
|
|
32
|
+
unclaimed file. `brief` renders unclaimed files before the findings, and
|
|
33
|
+
briefs them even when there is no evidence yet.
|
|
34
|
+
- `testguard.ignore.json` at the repository root: TestGuard's own excused
|
|
35
|
+
paths (dispatch, thin command wrappers, rendering), with reasons.
|
|
36
|
+
- Nine new self-claims: the gate, status and brief invariants, explicit
|
|
37
|
+
`--changed` being required, `init` idempotency, and — because the gate
|
|
38
|
+
flagged it on its own pull request — the release version-sync check,
|
|
39
|
+
which had no test before. The gate runs on this repository's own pull
|
|
40
|
+
requests.
|
|
41
|
+
- GitHub Action `command: gate` with a `changed-ref` input; pre-commit hook
|
|
42
|
+
`testguard-gate`; a GitLab CI template under `packaging/gitlab/` with
|
|
43
|
+
`testguard:gate` (merge request pipelines, measured against
|
|
44
|
+
`CI_MERGE_REQUEST_DIFF_BASE_SHA`) and `testguard:probe`.
|
|
45
|
+
|
|
10
46
|
## [0.4.0] - 2026-09-17
|
|
11
47
|
|
|
12
48
|
### Added
|
|
@@ -24,7 +24,7 @@ Node ≥ 20. Python ≥ 3.8 only if you touch `testguard_cli/`.
|
|
|
24
24
|
| `spec/lib/` | the validator and the one fingerprint implementation |
|
|
25
25
|
| `spec/conformance/` | one valid example per kind; must-reject documents named for their defect |
|
|
26
26
|
| `src/probe/` | worktree isolation, injector, runner adapter, classifier, ranker, orchestrator |
|
|
27
|
-
| `src/claims/`, `src/baseline/`, `src/brief/` | the other
|
|
27
|
+
| `src/claims/`, `src/baseline/`, `src/brief/`, `src/gate/` | the other commands: claims, baseline, brief, and the change gate |
|
|
28
28
|
| `src/commands/`, `src/cli.mjs`, `cli/` | argument parsing and dispatch |
|
|
29
29
|
| `fixtures/known-answer/` | the oracle: a project with a known verdict for every value in the verdict set |
|
|
30
30
|
| `testguard.claims.json` | TestGuard's claims about itself, probed in CI |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -76,7 +76,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
76
76
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
77
77
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
78
78
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
79
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
79
|
+
| GitHub Action | `uses: raccioly/testguard@v0.5.0` — see [`action.yml`](./action.yml) |
|
|
80
80
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
81
81
|
|
|
82
82
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -92,6 +92,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
92
92
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
93
93
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
94
94
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
95
|
+
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim at all
|
|
95
96
|
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
96
97
|
```
|
|
97
98
|
|
|
@@ -170,6 +171,54 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
170
171
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
171
172
|
so the hook can never break a session.
|
|
172
173
|
|
|
174
|
+
### Every change needs a claim
|
|
175
|
+
|
|
176
|
+
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
177
|
+
reports so far was a **claim gap**: the feature shipped green with zero
|
|
178
|
+
claims, and a verifier with no claim about a feature is silent about it by
|
|
179
|
+
construction. `gate` closes that hole on the delta:
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
npx testguard-cli gate --changed origin/main # in a PR: the files changed since the base branch
|
|
183
|
+
npx testguard-cli gate --changed HEAD --include-dirty # before a commit: the working tree, staged or not
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Every changed source file must carry a fault, resolve as a defender of a
|
|
187
|
+
claim (test files), or be excused by an unexpired `path` entry in
|
|
188
|
+
`testguard.ignore.json` — with a reason a reviewer will accept. One
|
|
189
|
+
unclaimed file exits `1`; there is no percentage. Every reliance on an ignore
|
|
190
|
+
entry is printed, so a reviewer sees *why* the gate passed; an expired entry
|
|
191
|
+
excuses nothing. Non-source files and documented never-claimed patterns
|
|
192
|
+
(`*.d.ts`, `*.config.*`, fixtures, mocks; `--explain` lists them) are
|
|
193
|
+
excluded and said so; `--strict` fails a change that evaluated nothing.
|
|
194
|
+
|
|
195
|
+
With a reference known, `status --changed <ref>` reports `unclaimed-changes`
|
|
196
|
+
**before** any evidence state and makes the claim the next action; the brief
|
|
197
|
+
lists the unclaimed files first. The claim is written before more code.
|
|
198
|
+
|
|
199
|
+
**In CI the base is detected** — GitHub Actions (`GITHUB_BASE_REF`) and GitLab
|
|
200
|
+
merge request pipelines (`CI_MERGE_REQUEST_DIFF_BASE_SHA`, then
|
|
201
|
+
`CI_MERGE_REQUEST_TARGET_BRANCH_NAME`); `TESTGUARD_CHANGED_REF` overrides both.
|
|
202
|
+
The base must exist locally: GitHub — `actions/checkout` with `fetch-depth: 0`;
|
|
203
|
+
GitLab — the diff base sha needs nothing extra on a merge request pipeline,
|
|
204
|
+
the branch name needs `GIT_DEPTH: 0` or a `git fetch origin <target>`. A
|
|
205
|
+
detected base that does not resolve is a warning for `status` and `brief`
|
|
206
|
+
(they keep working) and an error for `gate` (its whole job is the measurement).
|
|
207
|
+
|
|
208
|
+
```yaml
|
|
209
|
+
# GitHub Actions
|
|
210
|
+
- uses: actions/checkout@v4
|
|
211
|
+
with: { fetch-depth: 0 }
|
|
212
|
+
- uses: raccioly/testguard@v0.5.0
|
|
213
|
+
with: { command: gate }
|
|
214
|
+
|
|
215
|
+
# GitLab CI — or include: remote: the template in packaging/gitlab/
|
|
216
|
+
testguard:gate:
|
|
217
|
+
image: node:22
|
|
218
|
+
rules: [{ if: $CI_PIPELINE_SOURCE == "merge_request_event" }]
|
|
219
|
+
script: [npx -y testguard-cli gate .]
|
|
220
|
+
```
|
|
221
|
+
|
|
173
222
|
### Built for agents to run
|
|
174
223
|
|
|
175
224
|
TestGuard is meant to be driven by an AI agent, not typed by a person. Three
|
|
@@ -244,9 +293,10 @@ through every verdict.
|
|
|
244
293
|
|
|
245
294
|
## Status
|
|
246
295
|
|
|
247
|
-
**v0.4.**
|
|
248
|
-
a mechanical scaffold,
|
|
249
|
-
|
|
296
|
+
**v0.4.** Eight commands, vitest and jest runners, hand-authored faults plus
|
|
297
|
+
a mechanical scaffold, an agent operating layer (`status`, `init`) and a
|
|
298
|
+
change gate (`gate`). The contract
|
|
299
|
+
spine — eight JSON Schemas shared with the other Guard tools — is under
|
|
250
300
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
251
301
|
|
|
252
302
|
Not yet: test generation (the two-gate acceptance loop), runners beyond
|
|
@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.5.0` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
58
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -68,6 +68,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
68
68
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
69
69
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
70
70
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
71
|
+
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim at all
|
|
71
72
|
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
72
73
|
```
|
|
73
74
|
|
|
@@ -146,6 +147,54 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
146
147
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
147
148
|
so the hook can never break a session.
|
|
148
149
|
|
|
150
|
+
### Every change needs a claim
|
|
151
|
+
|
|
152
|
+
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
153
|
+
reports so far was a **claim gap**: the feature shipped green with zero
|
|
154
|
+
claims, and a verifier with no claim about a feature is silent about it by
|
|
155
|
+
construction. `gate` closes that hole on the delta:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
npx testguard-cli gate --changed origin/main # in a PR: the files changed since the base branch
|
|
159
|
+
npx testguard-cli gate --changed HEAD --include-dirty # before a commit: the working tree, staged or not
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Every changed source file must carry a fault, resolve as a defender of a
|
|
163
|
+
claim (test files), or be excused by an unexpired `path` entry in
|
|
164
|
+
`testguard.ignore.json` — with a reason a reviewer will accept. One
|
|
165
|
+
unclaimed file exits `1`; there is no percentage. Every reliance on an ignore
|
|
166
|
+
entry is printed, so a reviewer sees *why* the gate passed; an expired entry
|
|
167
|
+
excuses nothing. Non-source files and documented never-claimed patterns
|
|
168
|
+
(`*.d.ts`, `*.config.*`, fixtures, mocks; `--explain` lists them) are
|
|
169
|
+
excluded and said so; `--strict` fails a change that evaluated nothing.
|
|
170
|
+
|
|
171
|
+
With a reference known, `status --changed <ref>` reports `unclaimed-changes`
|
|
172
|
+
**before** any evidence state and makes the claim the next action; the brief
|
|
173
|
+
lists the unclaimed files first. The claim is written before more code.
|
|
174
|
+
|
|
175
|
+
**In CI the base is detected** — GitHub Actions (`GITHUB_BASE_REF`) and GitLab
|
|
176
|
+
merge request pipelines (`CI_MERGE_REQUEST_DIFF_BASE_SHA`, then
|
|
177
|
+
`CI_MERGE_REQUEST_TARGET_BRANCH_NAME`); `TESTGUARD_CHANGED_REF` overrides both.
|
|
178
|
+
The base must exist locally: GitHub — `actions/checkout` with `fetch-depth: 0`;
|
|
179
|
+
GitLab — the diff base sha needs nothing extra on a merge request pipeline,
|
|
180
|
+
the branch name needs `GIT_DEPTH: 0` or a `git fetch origin <target>`. A
|
|
181
|
+
detected base that does not resolve is a warning for `status` and `brief`
|
|
182
|
+
(they keep working) and an error for `gate` (its whole job is the measurement).
|
|
183
|
+
|
|
184
|
+
```yaml
|
|
185
|
+
# GitHub Actions
|
|
186
|
+
- uses: actions/checkout@v4
|
|
187
|
+
with: { fetch-depth: 0 }
|
|
188
|
+
- uses: raccioly/testguard@v0.5.0
|
|
189
|
+
with: { command: gate }
|
|
190
|
+
|
|
191
|
+
# GitLab CI — or include: remote: the template in packaging/gitlab/
|
|
192
|
+
testguard:gate:
|
|
193
|
+
image: node:22
|
|
194
|
+
rules: [{ if: $CI_PIPELINE_SOURCE == "merge_request_event" }]
|
|
195
|
+
script: [npx -y testguard-cli gate .]
|
|
196
|
+
```
|
|
197
|
+
|
|
149
198
|
### Built for agents to run
|
|
150
199
|
|
|
151
200
|
TestGuard is meant to be driven by an AI agent, not typed by a person. Three
|
|
@@ -220,9 +269,10 @@ through every verdict.
|
|
|
220
269
|
|
|
221
270
|
## Status
|
|
222
271
|
|
|
223
|
-
**v0.4.**
|
|
224
|
-
a mechanical scaffold,
|
|
225
|
-
|
|
272
|
+
**v0.4.** Eight commands, vitest and jest runners, hand-authored faults plus
|
|
273
|
+
a mechanical scaffold, an agent operating layer (`status`, `init`) and a
|
|
274
|
+
change gate (`gate`). The contract
|
|
275
|
+
spine — eight JSON Schemas shared with the other Guard tools — is under
|
|
226
276
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
227
277
|
|
|
228
278
|
Not yet: test generation (the two-gate acceptance loop), runners beyond
|
|
@@ -8,7 +8,7 @@ branding:
|
|
|
8
8
|
|
|
9
9
|
inputs:
|
|
10
10
|
command:
|
|
11
|
-
description: 'Command to run: probe, claims, baseline, brief'
|
|
11
|
+
description: 'Command to run: probe, claims, baseline, brief, gate'
|
|
12
12
|
required: false
|
|
13
13
|
default: 'probe'
|
|
14
14
|
working-directory:
|
|
@@ -35,10 +35,18 @@ inputs:
|
|
|
35
35
|
description: 'Skip re-running survivors against the whole suite'
|
|
36
36
|
required: false
|
|
37
37
|
default: 'false'
|
|
38
|
+
changed-ref:
|
|
39
|
+
description: 'gate only: the reference the change is measured against (default: origin/<base branch> of the pull request). Check out with fetch-depth: 0 so it resolves.'
|
|
40
|
+
required: false
|
|
41
|
+
default: ''
|
|
42
|
+
strict:
|
|
43
|
+
description: 'gate only: fail a change whose files were all excluded instead of passing with a note'
|
|
44
|
+
required: false
|
|
45
|
+
default: 'false'
|
|
38
46
|
version:
|
|
39
47
|
description: 'testguard-cli version to run'
|
|
40
48
|
required: false
|
|
41
|
-
default: '0.
|
|
49
|
+
default: '0.5.0'
|
|
42
50
|
node-version:
|
|
43
51
|
description: 'Node.js version (20+)'
|
|
44
52
|
required: false
|
|
@@ -68,5 +76,14 @@ runs:
|
|
|
68
76
|
[ "${{ inputs.no-escalate }}" = "true" ] && ARGS+=(--no-escalate)
|
|
69
77
|
fi
|
|
70
78
|
[ "${{ inputs.command }}" = "brief" ] && ARGS+=(--text)
|
|
79
|
+
if [ "${{ inputs.command }}" = "gate" ]; then
|
|
80
|
+
REF="${{ inputs.changed-ref }}"
|
|
81
|
+
if [ -z "$REF" ] && [ -n "${GITHUB_BASE_REF:-}" ]; then
|
|
82
|
+
git fetch --no-tags --quiet origin "$GITHUB_BASE_REF" || true
|
|
83
|
+
REF="origin/${GITHUB_BASE_REF}"
|
|
84
|
+
fi
|
|
85
|
+
[ -n "$REF" ] && ARGS+=(--changed "$REF")
|
|
86
|
+
[ "${{ inputs.strict }}" = "true" ] && ARGS+=(--strict)
|
|
87
|
+
fi
|
|
71
88
|
echo "evidence=$(pwd)/.testguard/evidence.json" >> "$GITHUB_OUTPUT"
|
|
72
89
|
npx -y "testguard-cli@${{ inputs.version }}" "${{ inputs.command }}" . "${ARGS[@]}"
|
|
@@ -44,3 +44,22 @@ it, and runs `testguard` against the project by path.
|
|
|
44
44
|
| `flaky-defender` | the defenders are not green N/N unmodified — the project's flake, not the tool's; nothing about the fault can be concluded |
|
|
45
45
|
| `killed` where `survived` was expected | either the suite was hardened since, or the previous measurement ran once and a flake read as a detection |
|
|
46
46
|
| `survived` where `killed` was expected | the declared defenders are not the tests that actually catch it; escalation will say whether anything does |
|
|
47
|
+
|
|
48
|
+
## What a field report should carry
|
|
49
|
+
|
|
50
|
+
The two reports that shaped 0.1.2 and 0.1.3 were written from the human
|
|
51
|
+
table. Since 0.3.0 the tool also states, in machine-readable form, where a
|
|
52
|
+
project is and what it would tell an agent to do next. Capture both, from
|
|
53
|
+
the project directory, after the final probe:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
testguard status --json > ~/bench/<project>/status.json # state, next action, changed faults, findings
|
|
57
|
+
testguard brief --text > ~/bench/<project>/brief.txt # exactly what an agent sees at session start
|
|
58
|
+
testguard status --json --changed <base-ref> > ~/bench/<project>/status-changed.json # 0.5.0+: unclaimed changed files
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
and include them with the evidence. They answer the questions a report
|
|
62
|
+
otherwise has to argue in prose: *was anything stale when the numbers were
|
|
63
|
+
taken? did the tool point at the right next fix? would an agent have known
|
|
64
|
+
what to do?* If `status.next` disagrees with what you did next, that
|
|
65
|
+
disagreement is the most useful line in the report.
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "testguard-cli",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.5.0",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"ajv": "8.20.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# TestGuard for GitLab CI. Include it from a project's .gitlab-ci.yml:
|
|
2
|
+
#
|
|
3
|
+
# include:
|
|
4
|
+
# - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.5.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
5
|
+
#
|
|
6
|
+
# Two jobs. `testguard:gate` runs on every merge request pipeline and fails
|
|
7
|
+
# when a changed source file carries no claim (and no excusing ignore entry).
|
|
8
|
+
# `testguard:probe` runs the faults against the tests and is gated by the
|
|
9
|
+
# committed baseline. Override TESTGUARD_VERSION / TESTGUARD_DIR as needed.
|
|
10
|
+
#
|
|
11
|
+
# The gate measures the change against CI_MERGE_REQUEST_DIFF_BASE_SHA, which
|
|
12
|
+
# GitLab provides on merge request pipelines and which needs no extra fetch.
|
|
13
|
+
# On other pipelines set TESTGUARD_CHANGED_REF (and fetch that ref).
|
|
14
|
+
|
|
15
|
+
variables:
|
|
16
|
+
TESTGUARD_VERSION: "0.5.0"
|
|
17
|
+
TESTGUARD_DIR: "."
|
|
18
|
+
GIT_DEPTH: "0" # the probe builds a scratch worktree from HEAD; full history is the safe default
|
|
19
|
+
|
|
20
|
+
.testguard:
|
|
21
|
+
image: node:22
|
|
22
|
+
before_script:
|
|
23
|
+
- cd "$TESTGUARD_DIR"
|
|
24
|
+
- npm ci --ignore-scripts
|
|
25
|
+
|
|
26
|
+
testguard:gate:
|
|
27
|
+
extends: .testguard
|
|
28
|
+
stage: test
|
|
29
|
+
rules:
|
|
30
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
31
|
+
script:
|
|
32
|
+
- npx -y "testguard-cli@${TESTGUARD_VERSION}" gate .
|
|
33
|
+
artifacts:
|
|
34
|
+
when: always
|
|
35
|
+
paths: [".testguard/gate.json"]
|
|
36
|
+
|
|
37
|
+
testguard:probe:
|
|
38
|
+
extends: .testguard
|
|
39
|
+
stage: test
|
|
40
|
+
rules:
|
|
41
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
42
|
+
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
43
|
+
script:
|
|
44
|
+
- npx -y "testguard-cli@${TESTGUARD_VERSION}" probe . --quiet
|
|
45
|
+
artifacts:
|
|
46
|
+
when: always
|
|
47
|
+
paths: [".testguard/evidence.json"]
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
class Testguard < Formula
|
|
14
14
|
desc "Proves a test suite defends the claims a project makes"
|
|
15
15
|
homepage "https://github.com/raccioly/testguard"
|
|
16
|
-
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.
|
|
16
|
+
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.5.0.tgz"
|
|
17
17
|
sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
|
|
18
18
|
license "MIT"
|
|
19
19
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "testguard-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.5.0"
|
|
8
8
|
description = "Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -85,6 +85,57 @@ the suppress-up-to-count semantics are identical.
|
|
|
85
85
|
- An annotation is **strictly additive**. It never changes, suppresses, or
|
|
86
86
|
drops a finding. Ranking may read annotations; verdicts never do.
|
|
87
87
|
|
|
88
|
+
## Claim coverage of a change
|
|
89
|
+
|
|
90
|
+
`probe` asks whether the tests defend the claims that exist. It says nothing
|
|
91
|
+
about code that has no claim, by construction — and every escaped defect the
|
|
92
|
+
field reports share was a *claim gap*, not a defender gap: the feature shipped
|
|
93
|
+
green with zero claims. A repository with ten old claims and one new,
|
|
94
|
+
unclaimed module exits 0 under `probe` forever.
|
|
95
|
+
|
|
96
|
+
The change gate closes that hole. Given a reference, a tool measures the
|
|
97
|
+
**delta** — the files changed since `merge-base(ref, HEAD)`, or since that
|
|
98
|
+
merge-base in the working tree — file by file:
|
|
99
|
+
|
|
100
|
+
- A changed file is **excluded** when it is not source (by extension) or
|
|
101
|
+
matches a documented never-claimed pattern (`*.d.ts`, `*.config.*`,
|
|
102
|
+
fixtures, mocks, snapshots, the tool's own directory). Exclusions are
|
|
103
|
+
listed, never silent.
|
|
104
|
+
- A changed source file is **covered** when at least one fault anchors to it,
|
|
105
|
+
when it is a test file that resolves as a defender of some claim, or when an
|
|
106
|
+
**unexpired** `path` ignore entry excuses it.
|
|
107
|
+
- Anything else is **uncovered**. The unit is the file, on purpose: a fault
|
|
108
|
+
anchors to a file, so the file is the smallest unit the rest of the
|
|
109
|
+
contract already understands.
|
|
110
|
+
|
|
111
|
+
Rules:
|
|
112
|
+
|
|
113
|
+
1. **One uncovered file gates.** Exit `1`. There is no threshold and no
|
|
114
|
+
percentage; a percentage is how the gap hid before.
|
|
115
|
+
2. **Every reliance on an ignore entry is reported.** The gate may pass
|
|
116
|
+
*because of* an excuse; a reviewer reads which entries carried it, with
|
|
117
|
+
their reasons and expiry. An expired entry excuses nothing and is reported
|
|
118
|
+
as expired.
|
|
119
|
+
3. **A test file needs a claim too.** A test that defends no claim is the
|
|
120
|
+
authorship trap the pattern exists for; it is uncovered until a claim
|
|
121
|
+
names it in `defendedBy` (or discovery resolves it as a defender).
|
|
122
|
+
4. **Never a silent pass on nothing.** A non-empty change whose files were
|
|
123
|
+
all excluded passes with an explicit "0 evaluated" line; a tool offers a
|
|
124
|
+
strict mode that fails it instead. An empty change is an honest `0`.
|
|
125
|
+
5. **The reference is never guessed.** An explicit flag, or a CI-provided base
|
|
126
|
+
branch, or an error. An upstream that already contains the change has an
|
|
127
|
+
empty diff and would pass trivially.
|
|
128
|
+
6. **Unclaimed changes precede every evidence state.** When the status
|
|
129
|
+
document knows a reference and finds uncovered files, its state is
|
|
130
|
+
`unclaimed-changes` and its next action is to write the claim, before any
|
|
131
|
+
`unproven` finding is surfaced. The brief renders them first. The claim is
|
|
132
|
+
written before more code.
|
|
133
|
+
|
|
134
|
+
Exit codes: `0` every changed source file is claimed or excused (or nothing
|
|
135
|
+
changed); `1` at least one uncovered file (or strict mode over an
|
|
136
|
+
all-excluded change); `2` the change cannot be evaluated (unresolvable
|
|
137
|
+
reference, invalid claims or ignore file, no repository); `3` no reference.
|
|
138
|
+
|
|
88
139
|
## Severity floor
|
|
89
140
|
|
|
90
141
|
`--severity <level>` gates only findings whose claim severity is at or above
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Guard spec — the contract spine
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Eight formats that tools following the Guard pattern share. They share
|
|
4
4
|
*formats*, not code: [`docguard-cli`](https://www.npmjs.com/package/docguard-cli)
|
|
5
5
|
is Node ESM, [`websec-validator`](https://pypi.org/project/websec-validator/)
|
|
6
6
|
is Python, and porting one runtime into the other is not worth it. A tool
|
|
@@ -21,7 +21,8 @@ The pattern all such tools run:
|
|
|
21
21
|
| `ignore` | [`schemas/ignore.schema.json`](schemas/ignore.schema.json) | Reviewable scoping. Every entry has a reason. |
|
|
22
22
|
| `calibration` | [`schemas/calibration.schema.json`](schemas/calibration.schema.json) | P(finding is real) per bucket, with a Wilson interval and the sample size behind it. |
|
|
23
23
|
| `brief` | [`schemas/brief.schema.json`](schemas/brief.schema.json) | What to tell an agent before it writes code — ranked, capped, never a single score, and carrying the one next action. |
|
|
24
|
-
| `status` | [`schemas/status.schema.json`](schemas/status.schema.json) | Where the project is and what happens next — the single machine-readable truth every human rendering derives from. Surfaces faults whose content changed since they were probed. |
|
|
24
|
+
| `status` | [`schemas/status.schema.json`](schemas/status.schema.json) | Where the project is and what happens next — the single machine-readable truth every human rendering derives from. Surfaces faults whose content changed since they were probed, and changed files that carry no claim. |
|
|
25
|
+
| `gate` | [`schemas/gate.schema.json`](schemas/gate.schema.json) | Claim coverage of one change: which changed files carry a claim, which are excused (and by which ignore entry), which are unclaimed. The delta gate for code that has no claim yet. |
|
|
25
26
|
|
|
26
27
|
Shared definitions (verdicts, fault classes, provenance, annotations) live in
|
|
27
28
|
[`schemas/common.schema.json`](schemas/common.schema.json). Gate behaviour —
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"tool": { "name": "testguard", "version": "0.4.0" },
|
|
4
|
+
"generatedAt": "2026-09-17T02:00:00Z",
|
|
5
|
+
"ref": "origin/main",
|
|
6
|
+
"base": "0f2a4c6e8b1d3f5a7c9e1b3d5f7a9c1e3b5d7f9a",
|
|
7
|
+
"head": "9a7c5e3b1d0f8e6c4a2b0d8f6e4c2a0b8d6f4e2c",
|
|
8
|
+
"includeDirty": false,
|
|
9
|
+
"strict": false,
|
|
10
|
+
"changed": 5,
|
|
11
|
+
"evaluated": 3,
|
|
12
|
+
"excluded": [
|
|
13
|
+
{ "file": "README.md", "by": "non-source" },
|
|
14
|
+
{ "file": "vitest.config.mjs", "by": "default:**/*.config.*" }
|
|
15
|
+
],
|
|
16
|
+
"covered": [
|
|
17
|
+
{ "file": "src/redact.mjs", "by": "fault", "claimIds": ["REDACT-001", "REDACT-003"] },
|
|
18
|
+
{ "file": "src/legacy/format.mjs", "by": "ignore", "pattern": "src/legacy/**" }
|
|
19
|
+
],
|
|
20
|
+
"uncovered": [
|
|
21
|
+
{
|
|
22
|
+
"file": "src/export.mjs",
|
|
23
|
+
"kind": "source",
|
|
24
|
+
"nearestClaimId": "REDACT-001",
|
|
25
|
+
"suggestion": "testguard scaffold src/export.mjs --claim REDACT-001"
|
|
26
|
+
}
|
|
27
|
+
],
|
|
28
|
+
"reliedOn": [
|
|
29
|
+
{
|
|
30
|
+
"pattern": "src/legacy/**",
|
|
31
|
+
"reason": "Scheduled for deletion in Q4; no new claims will be authored against it.",
|
|
32
|
+
"expires": "2026-12-31T00:00:00Z",
|
|
33
|
+
"files": ["src/legacy/format.mjs"]
|
|
34
|
+
}
|
|
35
|
+
],
|
|
36
|
+
"expired": [],
|
|
37
|
+
"exitCode": 1
|
|
38
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"tool": { "name": "testguard", "version": "0.4.0" },
|
|
4
|
+
"generatedAt": "2026-09-17T02:00:00Z",
|
|
5
|
+
"ref": "origin/main",
|
|
6
|
+
"base": "0f2a4c6e8b1d3f5a7c9e1b3d5f7a9c1e3b5d7f9a",
|
|
7
|
+
"head": "9a7c5e3b1d0f8e6c4a2b0d8f6e4c2a0b8d6f4e2c",
|
|
8
|
+
"includeDirty": false,
|
|
9
|
+
"strict": false,
|
|
10
|
+
"changed": 4,
|
|
11
|
+
"evaluated": 1,
|
|
12
|
+
"excluded": [],
|
|
13
|
+
"covered": [
|
|
14
|
+
{ "file": "src/redact.mjs", "by": "fault", "claimIds": ["REDACT-001"] }
|
|
15
|
+
],
|
|
16
|
+
"uncovered": [],
|
|
17
|
+
"reliedOn": [],
|
|
18
|
+
"expired": [],
|
|
19
|
+
"exitCode": 0
|
|
20
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"tool": { "name": "testguard", "version": "0.4.0" },
|
|
4
|
+
"generatedAt": "2026-09-17T02:00:00Z",
|
|
5
|
+
"ref": "origin/main",
|
|
6
|
+
"base": "0f2a4c6e8b1d3f5a7c9e1b3d5f7a9c1e3b5d7f9a",
|
|
7
|
+
"head": "9a7c5e3b1d0f8e6c4a2b0d8f6e4c2a0b8d6f4e2c",
|
|
8
|
+
"includeDirty": false,
|
|
9
|
+
"strict": false,
|
|
10
|
+
"changed": 1,
|
|
11
|
+
"evaluated": 1,
|
|
12
|
+
"excluded": [],
|
|
13
|
+
"covered": [],
|
|
14
|
+
"uncovered": [
|
|
15
|
+
{ "file": "src/export.mjs", "kind": "source", "suggestion": "testguard scaffold src/export.mjs" }
|
|
16
|
+
],
|
|
17
|
+
"reliedOn": [],
|
|
18
|
+
"expired": [],
|
|
19
|
+
"exitCode": 0
|
|
20
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"tool": { "name": "testguard", "version": "0.4.0" },
|
|
4
|
+
"generatedAt": "2026-09-17T02:00:00Z",
|
|
5
|
+
"ref": "origin/main",
|
|
6
|
+
"base": "0f2a4c6e8b1d3f5a7c9e1b3d5f7a9c1e3b5d7f9a",
|
|
7
|
+
"head": "9a7c5e3b1d0f8e6c4a2b0d8f6e4c2a0b8d6f4e2c",
|
|
8
|
+
"includeDirty": false,
|
|
9
|
+
"strict": false,
|
|
10
|
+
"changed": 1,
|
|
11
|
+
"evaluated": 1,
|
|
12
|
+
"excluded": [],
|
|
13
|
+
"covered": [
|
|
14
|
+
{ "file": "src/legacy/format.mjs", "by": "ignore" }
|
|
15
|
+
],
|
|
16
|
+
"uncovered": [],
|
|
17
|
+
"reliedOn": [],
|
|
18
|
+
"expired": [],
|
|
19
|
+
"exitCode": 0
|
|
20
|
+
}
|