testguard-cli 0.3.1__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/scripts/sync-release-version.mjs +3 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/workflows/ci.yml +27 -0
- testguard_cli-0.5.0/.gitignore +23 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.pre-commit-hooks.yaml +9 -0
- testguard_cli-0.5.0/.testguard/baseline.json +11 -0
- testguard_cli-0.5.0/.testguard/status.json +32 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/AGENTS.md +2 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/CHANGELOG.md +56 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/CONTRIBUTING.md +1 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/PKG-INFO +63 -8
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/README.md +62 -7
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/action.yml +19 -2
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/bench/README.md +19 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/.gitignore +2 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/README.md +7 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/expected.json +54 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/jest.config.js +1 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/package.json +7 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/src/export.js +5 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/src/redact.js +62 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/test/flaky.test.js +13 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/test/redact.test.js +35 -0
- testguard_cli-0.5.0/fixtures/known-answer-jest/testguard.claims.json +297 -0
- testguard_cli-0.5.0/package-lock.json +5700 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/package.json +2 -1
- testguard_cli-0.5.0/packaging/gitlab/testguard.gitlab-ci.yml +47 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/packaging/homebrew/testguard.rb +1 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/pyproject.toml +1 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/GATE-SEMANTICS.md +51 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/README.md +3 -2
- testguard_cli-0.5.0/spec/conformance/examples/gate.json +38 -0
- testguard_cli-0.5.0/spec/conformance/invalid/gate.counts-disagree.json +20 -0
- testguard_cli-0.5.0/spec/conformance/invalid/gate.exit-zero-with-uncovered.json +20 -0
- testguard_cli-0.5.0/spec/conformance/invalid/gate.ignore-cover-without-pattern.json +20 -0
- testguard_cli-0.5.0/spec/conformance/invalid/status.claim-without-file.json +29 -0
- testguard_cli-0.5.0/spec/conformance/invalid/status.unclaimed-hidden-behind-unproven.json +40 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/lib/validate.mjs +19 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/brief.schema.json +11 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/common.schema.json +27 -0
- testguard_cli-0.5.0/spec/schemas/gate.schema.json +52 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/status.schema.json +26 -4
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/brief/brief.mjs +26 -2
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/cli.mjs +23 -3
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/brief.mjs +18 -10
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/claims.mjs +1 -1
- testguard_cli-0.5.0/src/commands/gate.mjs +47 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/probe.mjs +3 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/status.mjs +2 -1
- testguard_cli-0.5.0/src/gate/changed.mjs +222 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/init/init.mjs +1 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/init/templates/SKILL.md +19 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/probe/discover.mjs +1 -1
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/probe/probe.mjs +17 -13
- testguard_cli-0.5.0/src/probe/runners/index.mjs +20 -0
- testguard_cli-0.5.0/src/probe/runners/jest.mjs +9 -0
- testguard_cli-0.3.1/src/probe/runner-vitest.mjs → testguard_cli-0.5.0/src/probe/runners/shared.mjs +29 -21
- testguard_cli-0.5.0/src/probe/runners/vitest.mjs +7 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/status/status.mjs +43 -3
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/brief.test.mjs +20 -1
- testguard_cli-0.5.0/test/gate.test.mjs +227 -0
- testguard_cli-0.5.0/test/probe.jest.test.mjs +52 -0
- testguard_cli-0.5.0/test/release-sync.test.mjs +57 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/runner-command.test.mjs +3 -3
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/runner-vitest.test.mjs +9 -3
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/status.test.mjs +29 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/testguard.claims.json +288 -0
- testguard_cli-0.5.0/testguard.ignore.json +34 -0
- testguard_cli-0.3.1/.gitignore +0 -18
- testguard_cli-0.3.1/package-lock.json +0 -1231
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.codex/hooks.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.gitattributes +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/dependabot.yml +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/scripts/install-smoke.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/workflows/auto-merge.yml +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/workflows/release.yml +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/workflows/scheduled-release.yml +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.github/workflows/supply-chain.yml +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.npmignore +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/.npmrc +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/CLAUDE.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/LICENSE +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/PRIVACY.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/SECURITY.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/SUPPORT.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/cli/testguard.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/README.md +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/expected.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/testguard.claims.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/baseline.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/evidence-provisional.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/evidence.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/examples/status.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.provisional-flag-on-confirmed-run.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.single-run-without-provisional.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/status.clean-with-new-findings.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/invalid/status.write-test-without-target.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/conformance/schemas.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/baseline.schema.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/claims.schema.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/evidence.schema.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/baseline/baseline.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/claims/load.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/baseline.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/init.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/commands/scaffold.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/git.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/probe/classify.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/probe/rank.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/probe/worktree.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/render.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/scaffold/producers.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/scaffold/scaffold.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/util/glob.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/src/util/hash.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/classify.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/discover.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/glob.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/init.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/inject.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/probe-preconditions.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/probe.fixture.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/rank-aliases.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/scaffold.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/snapshot.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/worktree.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/test/writer.test.mjs +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.3.1 → testguard_cli-0.5.0}/vitest.config.mjs +0 -0
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* - pyproject.toml `version = "x.y.z"`
|
|
6
6
|
* - action.yml default of the `version` input
|
|
7
7
|
* - packaging/homebrew/*.rb the tarball URL (sha256 is set after publish)
|
|
8
|
+
* - packaging/gitlab/*.yml the include URL tag and TESTGUARD_VERSION
|
|
8
9
|
*
|
|
9
10
|
* `--check` verifies instead of writing (used by release.yml).
|
|
10
11
|
*/
|
|
@@ -21,6 +22,8 @@ const surfaces = [
|
|
|
21
22
|
['action.yml', /(\n version:\n description: [^\n]*\n required: false\n default: ')[^']*(')/, `$1${version}$2`],
|
|
22
23
|
['packaging/homebrew/testguard.rb', /testguard-cli-\d+\.\d+\.\d+\.tgz/g, `testguard-cli-${version}.tgz`],
|
|
23
24
|
['README.md', /raccioly\/testguard@v\d+\.\d+\.\d+/g, `raccioly/testguard@v${version}`],
|
|
25
|
+
['packaging/gitlab/testguard.gitlab-ci.yml', /testguard\/v\d+\.\d+\.\d+\/packaging/g, `testguard/v${version}/packaging`],
|
|
26
|
+
['packaging/gitlab/testguard.gitlab-ci.yml', /TESTGUARD_VERSION: "\d+\.\d+\.\d+"/, `TESTGUARD_VERSION: "${version}"`],
|
|
24
27
|
];
|
|
25
28
|
|
|
26
29
|
let drift = 0;
|
|
@@ -102,6 +102,33 @@ jobs:
|
|
|
102
102
|
console.log('fixture verdicts match expected.json:', ev.records.length);
|
|
103
103
|
"
|
|
104
104
|
|
|
105
|
+
- name: probe the jest fixture (through the bin, --runner jest)
|
|
106
|
+
shell: bash
|
|
107
|
+
run: |
|
|
108
|
+
S=$(mktemp -d)
|
|
109
|
+
cp -R fixtures/known-answer-jest/. "$S/"
|
|
110
|
+
rm -rf "$S/node_modules" "$S/.flake-counter" "$S/.testguard"
|
|
111
|
+
ln -s "$GITHUB_WORKSPACE/node_modules" "$S/node_modules"
|
|
112
|
+
git -C "$S" init -q && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci add -A && git -C "$S" -c user.email=ci@example.invalid -c user.name=ci commit -qm fixture
|
|
113
|
+
status=0; node cli/testguard.mjs probe "$S" --runner jest --budget 60000 --quiet --no-escalate || status=$?
|
|
114
|
+
[ "$status" -eq 1 ] || { echo "::error::expected exit 1, got $status"; exit 1; }
|
|
115
|
+
node --input-type=module -e "
|
|
116
|
+
import { readFileSync } from 'node:fs';
|
|
117
|
+
const ev = JSON.parse(readFileSync('$S/.testguard/evidence.json','utf8'));
|
|
118
|
+
const exp = JSON.parse(readFileSync('fixtures/known-answer-jest/expected.json','utf8')).expected;
|
|
119
|
+
if (ev.run.runner.name !== 'jest') { console.error('runner was', ev.run.runner.name); process.exit(1); }
|
|
120
|
+
const bad = ev.records.filter(r => exp[r.claim.id+'/'+r.subject.id].verdict !== r.verdict);
|
|
121
|
+
if (bad.length) { console.error('MISMATCH', bad.map(r => r.claim.id+'/'+r.subject.id+':'+r.verdict)); process.exit(1); }
|
|
122
|
+
console.log('jest fixture verdicts match expected.json:', ev.records.length);
|
|
123
|
+
"
|
|
124
|
+
|
|
125
|
+
# ── Every change needs a claim: the gate runs on this repository's own
|
|
126
|
+
# pull requests. A new source file without a claim (or an excusing
|
|
127
|
+
# ignore entry with a reason) turns the PR red. ──
|
|
128
|
+
- name: gate — every changed source file carries a claim
|
|
129
|
+
if: github.event_name == 'pull_request'
|
|
130
|
+
run: node cli/testguard.mjs gate . --changed "origin/${{ github.base_ref }}"
|
|
131
|
+
|
|
105
132
|
# ── Self-verification: TestGuard is governed by the standard it defines.
|
|
106
133
|
# testguard.claims.json states invariants of the tool itself; every
|
|
107
134
|
# fault must be killed by the tool's own tests. Exit 0 = all defended. ──
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
node_modules/
|
|
2
|
+
dist/
|
|
3
|
+
coverage/
|
|
4
|
+
.DS_Store
|
|
5
|
+
*.log
|
|
6
|
+
# TestGuard: baseline.json and status.json ARE committed (the frozen contract and its snapshot); the rest is regenerated per run
|
|
7
|
+
.testguard/evidence.json
|
|
8
|
+
.testguard/evidence-provisional.json
|
|
9
|
+
.testguard/evidence-partial.json
|
|
10
|
+
.testguard/brief.json
|
|
11
|
+
.testguard/gate.json
|
|
12
|
+
.testguard/scaffold-*.json
|
|
13
|
+
# Local agent tooling — not part of the public project
|
|
14
|
+
.wolf/
|
|
15
|
+
.claude/
|
|
16
|
+
CLAUDE.local.md
|
|
17
|
+
graphify-out/
|
|
18
|
+
.vitest/
|
|
19
|
+
# Python build artefacts
|
|
20
|
+
dist/
|
|
21
|
+
build/
|
|
22
|
+
*.egg-info/
|
|
23
|
+
__pycache__/
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
# rev: v0.1.0
|
|
11
11
|
# hooks:
|
|
12
12
|
# - id: testguard-claims # fast: validates the claims file + drift
|
|
13
|
+
# - id: testguard-gate # fast: every changed source file must carry a claim
|
|
13
14
|
# - id: testguard-probe # slow: pre-push stage, runs the faults
|
|
14
15
|
#
|
|
15
16
|
# # install the push-stage hook once per clone:
|
|
@@ -23,6 +24,14 @@
|
|
|
23
24
|
pass_filenames: false
|
|
24
25
|
always_run: true
|
|
25
26
|
|
|
27
|
+
- id: testguard-gate
|
|
28
|
+
name: TestGuard gate (every change needs a claim)
|
|
29
|
+
description: Fails when a source file in this commit carries no claim, does not defend one, and is not excused by testguard.ignore.json. Fast; runs no tests.
|
|
30
|
+
entry: testguard gate --changed HEAD --include-dirty --quiet
|
|
31
|
+
language: node
|
|
32
|
+
pass_filenames: false
|
|
33
|
+
always_run: true
|
|
34
|
+
|
|
26
35
|
- id: testguard-probe
|
|
27
36
|
name: TestGuard probe (pre-push)
|
|
28
37
|
description: Injects every claim's faults and fails on any the tests miss. Gated by the committed baseline.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 1,
|
|
3
|
+
"tool": {
|
|
4
|
+
"name": "testguard",
|
|
5
|
+
"version": "0.3.0"
|
|
6
|
+
},
|
|
7
|
+
"generatedAt": "2026-09-17T15:24:13.654Z",
|
|
8
|
+
"state": "clean",
|
|
9
|
+
"next": {
|
|
10
|
+
"action": "none",
|
|
11
|
+
"command": "testguard probe",
|
|
12
|
+
"why": "Every claim is defended. Re-probe after changing code, tests or claims; add a claim for every new invariant."
|
|
13
|
+
},
|
|
14
|
+
"provisional": false,
|
|
15
|
+
"counts": {
|
|
16
|
+
"claims": 12,
|
|
17
|
+
"faults": 12,
|
|
18
|
+
"byVerdict": {
|
|
19
|
+
"killed": 12
|
|
20
|
+
},
|
|
21
|
+
"new": 0,
|
|
22
|
+
"baselined": 0
|
|
23
|
+
},
|
|
24
|
+
"stale": [],
|
|
25
|
+
"changedFaults": [],
|
|
26
|
+
"findings": [],
|
|
27
|
+
"paths": {
|
|
28
|
+
"claims": "testguard.claims.json",
|
|
29
|
+
"evidence": ".testguard/evidence.json",
|
|
30
|
+
"baseline": ".testguard/baseline.json"
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -31,6 +31,8 @@ node cli/testguard.mjs scaffold src/probe/classify.mjs --json # what the produ
|
|
|
31
31
|
- `fixtures/known-answer/expected.json` — the oracle. Never edit it to match output.
|
|
32
32
|
- `testguard.claims.json` — claims about this codebase, probed in CI.
|
|
33
33
|
- `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
|
|
34
|
+
- `src/probe/runners/` — one module per runner (`name`, `testGlobs`, `check`, `tests`, `run`) over `shared.mjs` (jest-compatible report parsing, budgeted process runner). A new runner needs its own fixture copy with the same `expected.json`.
|
|
35
|
+
- `src/gate/changed.mjs` — claim coverage of a change (`gate --changed`). File-level, delta-only; an uncovered file exits 1; every ignore reliance is reported. Its rules are a section of `GATE-SEMANTICS.md`.
|
|
34
36
|
- `src/status/status.mjs` — the state machine every rendering derives from. A new state or action is a spec change (`status.schema.json`) and a skill-template change (`src/init/templates/SKILL.md`) in the same PR.
|
|
35
37
|
|
|
36
38
|
## Rules
|
|
@@ -7,6 +7,62 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.5.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
The change gate: unclaimed code is now a finding.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
|
|
16
|
+
- **`testguard gate --changed <ref>`** — the change gate. Every escaped
|
|
17
|
+
defect in the field reports was a *claim gap*: the feature shipped green
|
|
18
|
+
with zero claims, and `probe` is silent about unclaimed code by
|
|
19
|
+
construction. `gate` measures the files changed since
|
|
20
|
+
`merge-base(ref, HEAD)` (or in the working tree with `--include-dirty`)
|
|
21
|
+
and exits `1` when any changed source file carries no fault, does not
|
|
22
|
+
resolve as a defender (test files), and is not excused by an unexpired
|
|
23
|
+
`path` entry in `testguard.ignore.json`. Every reliance on an ignore entry
|
|
24
|
+
is printed with its reason; expired entries excuse nothing. Non-source
|
|
25
|
+
files and documented never-claimed patterns are excluded and listed
|
|
26
|
+
(`--explain`, `--exclude <glob>`); `--strict` fails a change that
|
|
27
|
+
evaluated nothing. The base branch is detected in GitHub Actions and
|
|
28
|
+
GitLab CI (`TESTGUARD_CHANGED_REF` overrides). A new spec kind,
|
|
29
|
+
`gate.schema.json`, with conformance examples.
|
|
30
|
+
- **`status --changed <ref>`** — a new state `unclaimed-changes` and action
|
|
31
|
+
`claim` that precede every evidence state; `next.file` names the first
|
|
32
|
+
unclaimed file. `brief` renders unclaimed files before the findings, and
|
|
33
|
+
briefs them even when there is no evidence yet.
|
|
34
|
+
- `testguard.ignore.json` at the repository root: TestGuard's own excused
|
|
35
|
+
paths (dispatch, thin command wrappers, rendering), with reasons.
|
|
36
|
+
- Nine new self-claims: the gate, status and brief invariants, explicit
|
|
37
|
+
`--changed` being required, `init` idempotency, and — because the gate
|
|
38
|
+
flagged it on its own pull request — the release version-sync check,
|
|
39
|
+
which had no test before. The gate runs on this repository's own pull
|
|
40
|
+
requests.
|
|
41
|
+
- GitHub Action `command: gate` with a `changed-ref` input; pre-commit hook
|
|
42
|
+
`testguard-gate`; a GitLab CI template under `packaging/gitlab/` with
|
|
43
|
+
`testguard:gate` (merge request pipelines, measured against
|
|
44
|
+
`CI_MERGE_REQUEST_DIFF_BASE_SHA`) and `testguard:probe`.
|
|
45
|
+
|
|
46
|
+
## [0.4.0] - 2026-09-17
|
|
47
|
+
|
|
48
|
+
### Added
|
|
49
|
+
|
|
50
|
+
- **jest runner.** `--runner vitest|jest|auto` (default `auto`: the first
|
|
51
|
+
that resolves in the project). jest runs with `--ci --json --runTestsByPath`
|
|
52
|
+
so file arguments are exact paths, not regexes; its `Exceeded timeout`
|
|
53
|
+
wording is recognised as a timeout, never a kill; `__tests__/` files are
|
|
54
|
+
collected for escalation. `run.runner.name` records which runner produced
|
|
55
|
+
the evidence.
|
|
56
|
+
- A CommonJS edition of the known-answer fixture (`fixtures/known-answer-jest`)
|
|
57
|
+
with the same claims and the same expected verdicts; the acceptance suite
|
|
58
|
+
and CI prove jest against it exactly as vitest is proven against the ESM
|
|
59
|
+
one.
|
|
60
|
+
|
|
61
|
+
### Changed
|
|
62
|
+
|
|
63
|
+
- Runners are now modules behind one interface (`src/probe/runners/`); the
|
|
64
|
+
vitest path is unchanged and still reproduces its oracle.
|
|
65
|
+
|
|
10
66
|
## [0.3.1] - 2026-09-17
|
|
11
67
|
|
|
12
68
|
Dogfooding `init` and `status` on this repository.
|
|
@@ -24,7 +24,7 @@ Node ≥ 20. Python ≥ 3.8 only if you touch `testguard_cli/`.
|
|
|
24
24
|
| `spec/lib/` | the validator and the one fingerprint implementation |
|
|
25
25
|
| `spec/conformance/` | one valid example per kind; must-reject documents named for their defect |
|
|
26
26
|
| `src/probe/` | worktree isolation, injector, runner adapter, classifier, ranker, orchestrator |
|
|
27
|
-
| `src/claims/`, `src/baseline/`, `src/brief/` | the other
|
|
27
|
+
| `src/claims/`, `src/baseline/`, `src/brief/`, `src/gate/` | the other commands: claims, baseline, brief, and the change gate |
|
|
28
28
|
| `src/commands/`, `src/cli.mjs`, `cli/` | argument parsing and dispatch |
|
|
29
29
|
| `fixtures/known-answer/` | the oracle: a project with a known verdict for every value in the verdict set |
|
|
30
30
|
| `testguard.claims.json` | TestGuard's claims about itself, probed in CI |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -76,7 +76,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
76
76
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
77
77
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
78
78
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
79
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
79
|
+
| GitHub Action | `uses: raccioly/testguard@v0.5.0` — see [`action.yml`](./action.yml) |
|
|
80
80
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
81
81
|
|
|
82
82
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -92,6 +92,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
92
92
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
93
93
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
94
94
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
95
|
+
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim at all
|
|
95
96
|
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
96
97
|
```
|
|
97
98
|
|
|
@@ -135,6 +136,11 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
135
136
|
files that import the fault's target, by relative path or resolved alias.
|
|
136
137
|
`NOCOVER` then means exactly "no test file imports this source".
|
|
137
138
|
|
|
139
|
+
Runners: **vitest** and **jest** (`--runner auto` picks the first that
|
|
140
|
+
resolves; both read the same jest-compatible JSON report). Anything else
|
|
141
|
+
goes through `--runner-cmd`. Each runner is proven against its own copy
|
|
142
|
+
of the known-answer fixture.
|
|
143
|
+
|
|
138
144
|
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
139
145
|
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
140
146
|
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
@@ -165,6 +171,54 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
165
171
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
166
172
|
so the hook can never break a session.
|
|
167
173
|
|
|
174
|
+
### Every change needs a claim
|
|
175
|
+
|
|
176
|
+
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
177
|
+
reports so far was a **claim gap**: the feature shipped green with zero
|
|
178
|
+
claims, and a verifier with no claim about a feature is silent about it by
|
|
179
|
+
construction. `gate` closes that hole on the delta:
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
npx testguard-cli gate --changed origin/main # in a PR: the files changed since the base branch
|
|
183
|
+
npx testguard-cli gate --changed HEAD --include-dirty # before a commit: the working tree, staged or not
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Every changed source file must carry a fault, resolve as a defender of a
|
|
187
|
+
claim (test files), or be excused by an unexpired `path` entry in
|
|
188
|
+
`testguard.ignore.json` — with a reason a reviewer will accept. One
|
|
189
|
+
unclaimed file exits `1`; there is no percentage. Every reliance on an ignore
|
|
190
|
+
entry is printed, so a reviewer sees *why* the gate passed; an expired entry
|
|
191
|
+
excuses nothing. Non-source files and documented never-claimed patterns
|
|
192
|
+
(`*.d.ts`, `*.config.*`, fixtures, mocks; `--explain` lists them) are
|
|
193
|
+
excluded and said so; `--strict` fails a change that evaluated nothing.
|
|
194
|
+
|
|
195
|
+
With a reference known, `status --changed <ref>` reports `unclaimed-changes`
|
|
196
|
+
**before** any evidence state and makes the claim the next action; the brief
|
|
197
|
+
lists the unclaimed files first. The claim is written before more code.
|
|
198
|
+
|
|
199
|
+
**In CI the base is detected** — GitHub Actions (`GITHUB_BASE_REF`) and GitLab
|
|
200
|
+
merge request pipelines (`CI_MERGE_REQUEST_DIFF_BASE_SHA`, then
|
|
201
|
+
`CI_MERGE_REQUEST_TARGET_BRANCH_NAME`); `TESTGUARD_CHANGED_REF` overrides both.
|
|
202
|
+
The base must exist locally: GitHub — `actions/checkout` with `fetch-depth: 0`;
|
|
203
|
+
GitLab — the diff base sha needs nothing extra on a merge request pipeline,
|
|
204
|
+
the branch name needs `GIT_DEPTH: 0` or a `git fetch origin <target>`. A
|
|
205
|
+
detected base that does not resolve is a warning for `status` and `brief`
|
|
206
|
+
(they keep working) and an error for `gate` (its whole job is the measurement).
|
|
207
|
+
|
|
208
|
+
```yaml
|
|
209
|
+
# GitHub Actions
|
|
210
|
+
- uses: actions/checkout@v4
|
|
211
|
+
with: { fetch-depth: 0 }
|
|
212
|
+
- uses: raccioly/testguard@v0.5.0
|
|
213
|
+
with: { command: gate }
|
|
214
|
+
|
|
215
|
+
# GitLab CI — or include: remote: the template in packaging/gitlab/
|
|
216
|
+
testguard:gate:
|
|
217
|
+
image: node:22
|
|
218
|
+
rules: [{ if: $CI_PIPELINE_SOURCE == "merge_request_event" }]
|
|
219
|
+
script: [npx -y testguard-cli gate .]
|
|
220
|
+
```
|
|
221
|
+
|
|
168
222
|
### Built for agents to run
|
|
169
223
|
|
|
170
224
|
TestGuard is meant to be driven by an AI agent, not typed by a person. Three
|
|
@@ -239,14 +293,15 @@ through every verdict.
|
|
|
239
293
|
|
|
240
294
|
## Status
|
|
241
295
|
|
|
242
|
-
**v0.
|
|
243
|
-
mechanical scaffold,
|
|
244
|
-
|
|
296
|
+
**v0.4.** Eight commands, vitest and jest runners, hand-authored faults plus
|
|
297
|
+
a mechanical scaffold, an agent operating layer (`status`, `init`) and a
|
|
298
|
+
change gate (`gate`). The contract
|
|
299
|
+
spine — eight JSON Schemas shared with the other Guard tools — is under
|
|
245
300
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
246
301
|
|
|
247
|
-
Not yet: test generation (the two-gate acceptance loop),
|
|
248
|
-
AST-aware producers, and calibration of fault classes
|
|
249
|
-
bugs. Each is designed for; none is claimed.
|
|
302
|
+
Not yet: test generation (the two-gate acceptance loop), runners beyond
|
|
303
|
+
vitest and jest, AST-aware producers, and calibration of fault classes
|
|
304
|
+
against real escaped bugs. Each is designed for; none is claimed.
|
|
250
305
|
|
|
251
306
|
## Licence
|
|
252
307
|
|
|
@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.5.0` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
58
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -68,6 +68,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
68
68
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
69
69
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
70
70
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
71
|
+
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim at all
|
|
71
72
|
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
72
73
|
```
|
|
73
74
|
|
|
@@ -111,6 +112,11 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
111
112
|
files that import the fault's target, by relative path or resolved alias.
|
|
112
113
|
`NOCOVER` then means exactly "no test file imports this source".
|
|
113
114
|
|
|
115
|
+
Runners: **vitest** and **jest** (`--runner auto` picks the first that
|
|
116
|
+
resolves; both read the same jest-compatible JSON report). Anything else
|
|
117
|
+
goes through `--runner-cmd`. Each runner is proven against its own copy
|
|
118
|
+
of the known-answer fixture.
|
|
119
|
+
|
|
114
120
|
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
115
121
|
suite N times per survivor); iterate on one claim with `--claim <ID>` and
|
|
116
122
|
either `--include-dirty` or `--in-place` (only fault target files must be
|
|
@@ -141,6 +147,54 @@ npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to
|
|
|
141
147
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
142
148
|
so the hook can never break a session.
|
|
143
149
|
|
|
150
|
+
### Every change needs a claim
|
|
151
|
+
|
|
152
|
+
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
153
|
+
reports so far was a **claim gap**: the feature shipped green with zero
|
|
154
|
+
claims, and a verifier with no claim about a feature is silent about it by
|
|
155
|
+
construction. `gate` closes that hole on the delta:
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
npx testguard-cli gate --changed origin/main # in a PR: the files changed since the base branch
|
|
159
|
+
npx testguard-cli gate --changed HEAD --include-dirty # before a commit: the working tree, staged or not
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Every changed source file must carry a fault, resolve as a defender of a
|
|
163
|
+
claim (test files), or be excused by an unexpired `path` entry in
|
|
164
|
+
`testguard.ignore.json` — with a reason a reviewer will accept. One
|
|
165
|
+
unclaimed file exits `1`; there is no percentage. Every reliance on an ignore
|
|
166
|
+
entry is printed, so a reviewer sees *why* the gate passed; an expired entry
|
|
167
|
+
excuses nothing. Non-source files and documented never-claimed patterns
|
|
168
|
+
(`*.d.ts`, `*.config.*`, fixtures, mocks; `--explain` lists them) are
|
|
169
|
+
excluded and said so; `--strict` fails a change that evaluated nothing.
|
|
170
|
+
|
|
171
|
+
With a reference known, `status --changed <ref>` reports `unclaimed-changes`
|
|
172
|
+
**before** any evidence state and makes the claim the next action; the brief
|
|
173
|
+
lists the unclaimed files first. The claim is written before more code.
|
|
174
|
+
|
|
175
|
+
**In CI the base is detected** — GitHub Actions (`GITHUB_BASE_REF`) and GitLab
|
|
176
|
+
merge request pipelines (`CI_MERGE_REQUEST_DIFF_BASE_SHA`, then
|
|
177
|
+
`CI_MERGE_REQUEST_TARGET_BRANCH_NAME`); `TESTGUARD_CHANGED_REF` overrides both.
|
|
178
|
+
The base must exist locally: GitHub — `actions/checkout` with `fetch-depth: 0`;
|
|
179
|
+
GitLab — the diff base sha needs nothing extra on a merge request pipeline,
|
|
180
|
+
the branch name needs `GIT_DEPTH: 0` or a `git fetch origin <target>`. A
|
|
181
|
+
detected base that does not resolve is a warning for `status` and `brief`
|
|
182
|
+
(they keep working) and an error for `gate` (its whole job is the measurement).
|
|
183
|
+
|
|
184
|
+
```yaml
|
|
185
|
+
# GitHub Actions
|
|
186
|
+
- uses: actions/checkout@v4
|
|
187
|
+
with: { fetch-depth: 0 }
|
|
188
|
+
- uses: raccioly/testguard@v0.5.0
|
|
189
|
+
with: { command: gate }
|
|
190
|
+
|
|
191
|
+
# GitLab CI — or include: remote: the template in packaging/gitlab/
|
|
192
|
+
testguard:gate:
|
|
193
|
+
image: node:22
|
|
194
|
+
rules: [{ if: $CI_PIPELINE_SOURCE == "merge_request_event" }]
|
|
195
|
+
script: [npx -y testguard-cli gate .]
|
|
196
|
+
```
|
|
197
|
+
|
|
144
198
|
### Built for agents to run
|
|
145
199
|
|
|
146
200
|
TestGuard is meant to be driven by an AI agent, not typed by a person. Three
|
|
@@ -215,14 +269,15 @@ through every verdict.
|
|
|
215
269
|
|
|
216
270
|
## Status
|
|
217
271
|
|
|
218
|
-
**v0.
|
|
219
|
-
mechanical scaffold,
|
|
220
|
-
|
|
272
|
+
**v0.4.** Eight commands, vitest and jest runners, hand-authored faults plus
|
|
273
|
+
a mechanical scaffold, an agent operating layer (`status`, `init`) and a
|
|
274
|
+
change gate (`gate`). The contract
|
|
275
|
+
spine — eight JSON Schemas shared with the other Guard tools — is under
|
|
221
276
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
222
277
|
|
|
223
|
-
Not yet: test generation (the two-gate acceptance loop),
|
|
224
|
-
AST-aware producers, and calibration of fault classes
|
|
225
|
-
bugs. Each is designed for; none is claimed.
|
|
278
|
+
Not yet: test generation (the two-gate acceptance loop), runners beyond
|
|
279
|
+
vitest and jest, AST-aware producers, and calibration of fault classes
|
|
280
|
+
against real escaped bugs. Each is designed for; none is claimed.
|
|
226
281
|
|
|
227
282
|
## Licence
|
|
228
283
|
|
|
@@ -8,7 +8,7 @@ branding:
|
|
|
8
8
|
|
|
9
9
|
inputs:
|
|
10
10
|
command:
|
|
11
|
-
description: 'Command to run: probe, claims, baseline, brief'
|
|
11
|
+
description: 'Command to run: probe, claims, baseline, brief, gate'
|
|
12
12
|
required: false
|
|
13
13
|
default: 'probe'
|
|
14
14
|
working-directory:
|
|
@@ -35,10 +35,18 @@ inputs:
|
|
|
35
35
|
description: 'Skip re-running survivors against the whole suite'
|
|
36
36
|
required: false
|
|
37
37
|
default: 'false'
|
|
38
|
+
changed-ref:
|
|
39
|
+
description: 'gate only: the reference the change is measured against (default: origin/<base branch> of the pull request). Check out with fetch-depth: 0 so it resolves.'
|
|
40
|
+
required: false
|
|
41
|
+
default: ''
|
|
42
|
+
strict:
|
|
43
|
+
description: 'gate only: fail a change whose files were all excluded instead of passing with a note'
|
|
44
|
+
required: false
|
|
45
|
+
default: 'false'
|
|
38
46
|
version:
|
|
39
47
|
description: 'testguard-cli version to run'
|
|
40
48
|
required: false
|
|
41
|
-
default: '0.
|
|
49
|
+
default: '0.5.0'
|
|
42
50
|
node-version:
|
|
43
51
|
description: 'Node.js version (20+)'
|
|
44
52
|
required: false
|
|
@@ -68,5 +76,14 @@ runs:
|
|
|
68
76
|
[ "${{ inputs.no-escalate }}" = "true" ] && ARGS+=(--no-escalate)
|
|
69
77
|
fi
|
|
70
78
|
[ "${{ inputs.command }}" = "brief" ] && ARGS+=(--text)
|
|
79
|
+
if [ "${{ inputs.command }}" = "gate" ]; then
|
|
80
|
+
REF="${{ inputs.changed-ref }}"
|
|
81
|
+
if [ -z "$REF" ] && [ -n "${GITHUB_BASE_REF:-}" ]; then
|
|
82
|
+
git fetch --no-tags --quiet origin "$GITHUB_BASE_REF" || true
|
|
83
|
+
REF="origin/${GITHUB_BASE_REF}"
|
|
84
|
+
fi
|
|
85
|
+
[ -n "$REF" ] && ARGS+=(--changed "$REF")
|
|
86
|
+
[ "${{ inputs.strict }}" = "true" ] && ARGS+=(--strict)
|
|
87
|
+
fi
|
|
71
88
|
echo "evidence=$(pwd)/.testguard/evidence.json" >> "$GITHUB_OUTPUT"
|
|
72
89
|
npx -y "testguard-cli@${{ inputs.version }}" "${{ inputs.command }}" . "${ARGS[@]}"
|
|
@@ -44,3 +44,22 @@ it, and runs `testguard` against the project by path.
|
|
|
44
44
|
| `flaky-defender` | the defenders are not green N/N unmodified — the project's flake, not the tool's; nothing about the fault can be concluded |
|
|
45
45
|
| `killed` where `survived` was expected | either the suite was hardened since, or the previous measurement ran once and a flake read as a detection |
|
|
46
46
|
| `survived` where `killed` was expected | the declared defenders are not the tests that actually catch it; escalation will say whether anything does |
|
|
47
|
+
|
|
48
|
+
## What a field report should carry
|
|
49
|
+
|
|
50
|
+
The two reports that shaped 0.1.2 and 0.1.3 were written from the human
|
|
51
|
+
table. Since 0.3.0 the tool also states, in machine-readable form, where a
|
|
52
|
+
project is and what it would tell an agent to do next. Capture both, from
|
|
53
|
+
the project directory, after the final probe:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
testguard status --json > ~/bench/<project>/status.json # state, next action, changed faults, findings
|
|
57
|
+
testguard brief --text > ~/bench/<project>/brief.txt # exactly what an agent sees at session start
|
|
58
|
+
testguard status --json --changed <base-ref> > ~/bench/<project>/status-changed.json # 0.5.0+: unclaimed changed files
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
and include them with the evidence. They answer the questions a report
|
|
62
|
+
otherwise has to argue in prose: *was anything stale when the numbers were
|
|
63
|
+
taken? did the tool point at the right next fix? would an agent have known
|
|
64
|
+
what to do?* If `status.next` disagrees with what you did next, that
|
|
65
|
+
disagreement is the most useful line in the report.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Known-answer fixture — jest edition
|
|
2
|
+
|
|
3
|
+
The same nine claims and eleven faults as [`../known-answer`](../known-answer), in
|
|
4
|
+
CommonJS, run by **jest** (`jest.config.js`: `testTimeout: 1000`). Every verdict in
|
|
5
|
+
`expected.json` is the same. Its test suite is flaky by design (`test/flaky.test.js`).
|
|
6
|
+
|
|
7
|
+
It proves the jest runner adapter reproduces the oracle; the ESM fixture proves vitest.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"confirmRuns": 3,
|
|
3
|
+
"expected": {
|
|
4
|
+
"REDACT-001/F1": {
|
|
5
|
+
"verdict": "survived",
|
|
6
|
+
"why": "The audit-row test asserts with objectContaining and omits `content`; a row carrying the raw input passes."
|
|
7
|
+
},
|
|
8
|
+
"REDACT-001/F2": {
|
|
9
|
+
"verdict": "killed",
|
|
10
|
+
"why": "The masked-output test asserts the secret is absent."
|
|
11
|
+
},
|
|
12
|
+
"REDACT-002/F1": {
|
|
13
|
+
"verdict": "killed",
|
|
14
|
+
"why": "The invalid-rule test asserts compileRules does not throw."
|
|
15
|
+
},
|
|
16
|
+
"REDACT-003/F1": {
|
|
17
|
+
"verdict": "survived",
|
|
18
|
+
"why": "No test exercises a missing scope; the fail-closed guard is undefended."
|
|
19
|
+
},
|
|
20
|
+
"REDACT-004/F1": {
|
|
21
|
+
"verdict": "timeout",
|
|
22
|
+
"why": "Tests awaiting redact() hit the 1s jest testTimeout; a timeout is not an assertion failure and is not a kill."
|
|
23
|
+
},
|
|
24
|
+
"REDACT-005/F1": {
|
|
25
|
+
"verdict": "unverifiable",
|
|
26
|
+
"reason": "anchor-missing",
|
|
27
|
+
"why": "The find string is not present in the source."
|
|
28
|
+
},
|
|
29
|
+
"REDACT-005/F2": {
|
|
30
|
+
"verdict": "unverifiable",
|
|
31
|
+
"reason": "anchor-ambiguous",
|
|
32
|
+
"why": "`return null;` occurs twice; expectHits is 1."
|
|
33
|
+
},
|
|
34
|
+
"REDACT-006/F1": {
|
|
35
|
+
"verdict": "fault-invalid",
|
|
36
|
+
"reason": "replacement-does-not-compile",
|
|
37
|
+
"why": "The replacement introduces a syntax error; the suite cannot load."
|
|
38
|
+
},
|
|
39
|
+
"EXPORT-001/F1": {
|
|
40
|
+
"verdict": "nocover",
|
|
41
|
+
"why": "test/export.test.js does not exist; no defender resolves."
|
|
42
|
+
},
|
|
43
|
+
"FLAKY-001/F1": {
|
|
44
|
+
"verdict": "flaky-defender",
|
|
45
|
+
"reason": "defenders-not-green",
|
|
46
|
+
"why": "test/flaky.test.js fails every other run; the baseline is not green N/N."
|
|
47
|
+
},
|
|
48
|
+
"DISCOVER-001/F1": {
|
|
49
|
+
"verdict": "killed",
|
|
50
|
+
"why": "No defendedBy; test/redact.test.js is discovered because it imports src/redact.js, and its masked-output test fails."
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"runner": "jest"
|
|
54
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = { testEnvironment: 'node', testTimeout: 1000, testMatch: ['**/test/**/*.test.js'] };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "testguard-known-answer-fixture-jest",
|
|
3
|
+
"private": true,
|
|
4
|
+
"version": "0.0.0",
|
|
5
|
+
"description": "The known-answer fixture in CommonJS for jest. Same claims, same expected verdicts. FLAKY BY DESIGN (test/flaky.test.js).",
|
|
6
|
+
"scripts": { "test": "jest" }
|
|
7
|
+
}
|