testguard-cli 0.10.3__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/CHANGELOG.md +43 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/PKG-INFO +72 -6
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/README.md +71 -5
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/action.yml +1 -1
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs/testguard-explained.html +1 -1
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs/testguard-explained.pdf +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs-canonical/ARCHITECTURE.md +4 -1
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/package-lock.json +2 -2
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/package.json +1 -1
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/packaging/gitlab/testguard.gitlab-ci.yml +2 -2
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/packaging/homebrew/testguard.rb +2 -2
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/pyproject.toml +1 -1
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/lib/validate.mjs +43 -1
- testguard_cli-0.11.0/spec/schemas/sweep.schema.json +127 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/cli.mjs +8 -1
- testguard_cli-0.11.0/src/commands/sweep.mjs +74 -0
- testguard_cli-0.11.0/src/supply/persistence.mjs +173 -0
- testguard_cli-0.11.0/src/supply/select.mjs +168 -0
- testguard_cli-0.11.0/src/sweep/sweep.mjs +274 -0
- testguard_cli-0.11.0/test/supply-persistence.test.mjs +108 -0
- testguard_cli-0.11.0/test/supply-select.test.mjs +128 -0
- testguard_cli-0.11.0/test/sweep.fixture.test.mjs +129 -0
- testguard_cli-0.11.0/test/sweep.test.mjs +190 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/testguard.claims.json +191 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/testguard.cost-budget.json +1 -1
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/testguard.ignore.json +7 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.agents/skills/testguard/SKILL.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.codex/hooks.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.docguard.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.docguardignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.gitattributes +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/dependabot.yml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/approve-held-runs.sh +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/build-one-pager.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/check-cost-budget.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/check-python-fixture.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/install-smoke.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/publish-homebrew-tap.sh +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/scripts/sync-release-version.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/workflows/auto-merge.yml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/workflows/ci.yml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/workflows/release.yml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/workflows/scheduled-release.yml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.github/workflows/supply-chain.yml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.gitignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.npmignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.npmrc +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.pre-commit-config.yaml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/.pre-commit-hooks.yaml +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/AGENTS.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/CLAUDE.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/CONTRIBUTING.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/DRIFT-LOG.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/LICENSE +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/PRIVACY.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/SECURITY.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/SUPPORT.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/bench/README.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/cli/testguard.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs-canonical/DATA-MODEL.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs-canonical/ENVIRONMENT.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs-canonical/REQUIREMENTS.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs-canonical/SECURITY.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/docs-canonical/TEST-SPEC.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/README.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/expected.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/test/export-mocked.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/testguard.claims.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/.gitignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/README.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/expected.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/jest.config.js +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/package.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/src/export.js +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/src/redact.js +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/test/export-mocked.test.js +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/test/flaky.test.js +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/test/redact.test.js +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-jest/testguard.claims.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/.gitignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/README.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/e2e/flaky.spec.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/e2e/toggle.spec.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/expected.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/package-lock.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/package.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/playwright.config.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/src/toggle.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/test/toggle.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/test-results/.last-run.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/testguard.claims.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-playwright/vitest.config.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/.gitignore +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/README.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/demo/__init__.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/demo/export.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/demo/orphan.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/demo/pipeline.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/demo/redact.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/expected.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/testguard.claims.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/tests/test_export_mocked.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/tests/test_flaky.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/tests/test_patched.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/fixtures/known-answer-python/tests/test_redact.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/GATE-SEMANTICS.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/README.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/baseline.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/calibration-websec.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/claims-methods.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/evidence-disturbed-run.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/evidence-negative-control.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/evidence-provisional.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/evidence-python.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/evidence-scan-method.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/evidence.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/gate.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/replay.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/examples/status.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.backoff-cell-not-wilson.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.breakdown-does-not-sum-to-n.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.ci-not-wilson.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.compound-key-arity-mismatch.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.fallback-without-basis.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.p-not-positives-over-n.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.fault-injection-without-a-substitution.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.method-detail-on-a-specified-method.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.annotated-signal-without-reason.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.attribute-signal-not-a-defender.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.attribute-signal-without-attributes.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.byrunner-file-not-resolved.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.contention-without-runners.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.discovered-defender-mocks-subject.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.fault-injection-without-its-evidence.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.ignored-dirty-on-snapshot.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.method-detail-on-an-injecting-run.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.negative-control-not-reached-but-survived.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.negative-control-reached-on-a-kill.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.probe-error-without-message.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.provisional-flag-on-confirmed-run.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.restore-skipped-in-place.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.single-run-without-provisional.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.target-not-imported-but-killed.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/gate.counts-disagree.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/gate.exit-zero-with-uncovered.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/gate.ignore-cover-without-pattern.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/replay.blind-with-a-failing-run.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/replay.duplicate-patch-id.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.claim-without-file.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.clean-with-new-findings.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.invalid-anchor-hidden.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.partial-scope-counts-disagree.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.surface-counts-disagree.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.unclaimed-hidden-behind-unproven.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/invalid/status.write-test-without-target.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/conformance/schemas.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/lib/wilson.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/baseline.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/brief.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/claims.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/common.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/evidence.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/gate.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/replay.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/spec/schemas/status.schema.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/admit/admit.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/baseline/baseline.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/brief/brief.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/claims/anchors.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/claims/load.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/claims/removed.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/admit.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/baseline.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/brief.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/claims.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/gate.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/init.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/mcp.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/probe.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/replay.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/scaffold.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/commands/status.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/gate/changed.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/git.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/init/init.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/init/templates/SKILL.md +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/mcp/server.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/mcp/tools.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/attribution.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/classify.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/contention.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/cost.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/discover.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/independence.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/mocks.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/probe.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/progress.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/pyimports.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/rank.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/index.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/jest.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/playwright.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/python/_testguard_pytest_plugin.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/python/_testguard_report.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/python/_testguard_unittest_main.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/python.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/shared.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/runners/vitest.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/probe/worktree.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/render.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/replay/label.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/replay/replay.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/scaffold/producers.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/scaffold/producers.python.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/scaffold/scaffold.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/status/status.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/status/surface.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/util/glob.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/src/util/hash.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/admit.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/attribution.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/brief.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/calibration.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/claim-selection.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/claims-anchors.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/claims-since.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/classify.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/contention.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/cost.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/discover.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/gate.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/gitlab-template.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/glob.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/independence.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/init.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/inject.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/label.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/mcp.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/mocks.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/negative-control.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/one-pager.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/probe-error.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/probe-preconditions.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/probe.fixture.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/probe.jest.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/probe.playwright.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/probe.python.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/progress.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/pyimports.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/rank-aliases.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/release-approval.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/release-sync.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/replay-verdict.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/replay.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/runner-argv.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/runner-command.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/runner-playwright.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/runner-python.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/runner-vitest.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/samples/playwright-report.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/samples/python-report-collect-error.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/samples/python-report.json +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/scaffold.python.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/scaffold.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/snapshot.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/status.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/surface.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/vitest-config.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/worktree.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/test/writer.test.mjs +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.10.3 → testguard_cli-0.11.0}/vitest.config.mjs +0 -0
|
@@ -7,6 +7,49 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.11.0] - 2026-09-21
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.10.4`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- feat(sweep): findings on a repository that has no claims yet (#151)
|
|
17
|
+
- chore(homebrew): sha256 for v0.10.4 (#149)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- `testguard sweep [dir] --changed <ref>` — the cold start. Proposes faults for
|
|
23
|
+
the changed source files that carry no claim, probes a bounded selection, and
|
|
24
|
+
reports what a green suite did not notice. Never writes
|
|
25
|
+
`testguard.claims.json`; its evidence never replaces
|
|
26
|
+
`.testguard/evidence.json`; only `survived` and `nocover` exit 1, so a
|
|
27
|
+
proposal the tool could not anchor or compile is reported and never gates.
|
|
28
|
+
Offline, like every other verifying command.
|
|
29
|
+
- Selection follows Google's mutation service: at most 7 × |files| candidates,
|
|
30
|
+
spread round-robin across files, ordered on the measured productivity of the
|
|
31
|
+
fault class and shrunk toward a neutral prior so one observation never
|
|
32
|
+
dominates (`src/supply/select.mjs`).
|
|
33
|
+
- `persistence-payload-unasserted` — why a survivor on a write path was missed:
|
|
34
|
+
the defender mocks the persistence layer and the file's call assertions never
|
|
35
|
+
name an exact payload, so a field dropped from a write cannot fail them.
|
|
36
|
+
Descriptive, never predictive; attached only to a survivor on a write path
|
|
37
|
+
(`src/supply/persistence.mjs`).
|
|
38
|
+
- `urn:claimspec:v1:sweep` — the sweep document, with semantic rules that check
|
|
39
|
+
a cap never hides its own remainder.
|
|
40
|
+
- Five self-claims (8 faults) covering the cap arithmetic, the exit-code rule,
|
|
41
|
+
where the persistence signal may be attached, per-file cap fairness, and that
|
|
42
|
+
one assertion's arguments are read without reading the next one's.
|
|
43
|
+
|
|
44
|
+
## [0.10.4] - 2026-09-21
|
|
45
|
+
|
|
46
|
+
Automated weekly release — everything merged since `v0.10.3`.
|
|
47
|
+
|
|
48
|
+
### Changed
|
|
49
|
+
|
|
50
|
+
- chore(homebrew): sha256 for v0.10.3 (#147)
|
|
51
|
+
|
|
52
|
+
|
|
10
53
|
## [0.10.3] - 2026-09-20
|
|
11
54
|
|
|
12
55
|
Automated weekly release — everything merged since `v0.10.2`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest. Ships the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -151,9 +151,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
151
151
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
152
152
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
153
153
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
154
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
154
|
+
| GitHub Action | `uses: raccioly/testguard@v0.11.0` — see [`action.yml`](./action.yml) |
|
|
155
155
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
156
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
156
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.11.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
157
157
|
|
|
158
158
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
159
159
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -171,6 +171,7 @@ npx testguard-cli probe # try to falsify each claim; report
|
|
|
171
171
|
npx testguard-cli admit test/auth.test.ts --claim AUTH-ADMIN # does this test satisfy the two-gate rule?
|
|
172
172
|
npx testguard-cli baseline # freeze today's unproven findings; only new ones gate
|
|
173
173
|
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim
|
|
174
|
+
npx testguard-cli sweep --changed origin/main # no claims yet? propose faults for the changed files and report what nothing noticed
|
|
174
175
|
npx testguard-cli brief --text # tell the agent where the suite is blind, before it writes
|
|
175
176
|
npx testguard-cli replay --since v1.0..HEAD # would this suite have caught the bugs that escaped?
|
|
176
177
|
npx testguard-cli mcp # serve the read-only loop over MCP, on stdio
|
|
@@ -424,6 +425,71 @@ on-demand helper, not a hook**: the session-start hook never touches the
|
|
|
424
425
|
network, and the helper exits 0 with a message when the CLI or the artifact
|
|
425
426
|
is missing. TestGuard itself still makes no network calls.
|
|
426
427
|
|
|
428
|
+
### Starting from zero claims
|
|
429
|
+
|
|
430
|
+
`gate` names the changed files that carry no claim, and stops there —
|
|
431
|
+
correctly, because stating a claim is a human act. But a project adopting this
|
|
432
|
+
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
433
|
+
The bottleneck was never verification; it is **oracle supply**.
|
|
434
|
+
|
|
435
|
+
`sweep` is the lowest rung of that supply. No claim, no concern, nothing
|
|
436
|
+
written by anybody:
|
|
437
|
+
|
|
438
|
+
```bash
|
|
439
|
+
npx testguard-cli sweep --changed origin/main # propose, probe a bounded selection, report
|
|
440
|
+
npx testguard-cli sweep --changed HEAD --include-dirty --cap 20
|
|
441
|
+
```
|
|
442
|
+
|
|
443
|
+
It takes the files `gate` just called uncovered, proposes faults with the same
|
|
444
|
+
mechanical producers `scaffold` uses, probes a bounded selection, and reports
|
|
445
|
+
what a green suite did not notice. A fault that survives needs no claim to be
|
|
446
|
+
alarming: something was deliberately broken and not one test failed.
|
|
447
|
+
|
|
448
|
+
```
|
|
449
|
+
swept 13 of 14 unclaimed changed files against origin/main.
|
|
450
|
+
proposed 510 faults, probed 10 (cap 10), deferred 500.
|
|
451
|
+
The 500 deferred are not a verdict: raise --cap, or sweep a smaller change.
|
|
452
|
+
|
|
453
|
+
9 findings — a deliberate break that no test noticed:
|
|
454
|
+
|
|
455
|
+
SURVIVED src/app/actions/register.ts:118 [write path]
|
|
456
|
+
[line 118] Field dropped: `password_hash` is no longer written.
|
|
457
|
+
src/tests/actions/register.test.ts mocks @/lib/prisma; 1 exact, 8 partial,
|
|
458
|
+
31 argument-free call assertions in the file. A field dropped from the write
|
|
459
|
+
payload fails only against an assertion that names that field exactly.
|
|
460
|
+
Assert the whole object written by the call this fault changes, including
|
|
461
|
+
`password_hash` — or read the record back and assert on what was stored.
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
Three rules make it safe to run on a repository that has never seen this tool:
|
|
465
|
+
|
|
466
|
+
- **It never writes `testguard.claims.json`**, and never will. A claim is a
|
|
467
|
+
sentence someone is willing to stand behind; a sentence nobody wrote is not
|
|
468
|
+
one. Its drafts land beside the evidence to keep or drop.
|
|
469
|
+
- **Its evidence never replaces `.testguard/evidence.json`.** A sweep probes
|
|
470
|
+
faults nobody stated, under TODO statements; folding that into the document
|
|
471
|
+
`status` and `baseline` read would corrupt the record of what the project
|
|
472
|
+
actually claims.
|
|
473
|
+
- **It does not fail on its own bad guess.** Only `survived` and `nocover`
|
|
474
|
+
exit 1. A proposal that would not compile, or an anchor that did not locate,
|
|
475
|
+
is reported and never gates — a tool that fails because its own guess was bad
|
|
476
|
+
is a tool people switch off.
|
|
477
|
+
|
|
478
|
+
The cap and the ordering are Google's: their mutation service surfaces at most
|
|
479
|
+
7 × |files| mutants per change and orders candidates on the measured
|
|
480
|
+
productivity of their operator in similar context, which took their productive
|
|
481
|
+
rate from 15% to 89%
|
|
482
|
+
([Petrović et al., *Practical Mutation Testing at Scale*, 2021](https://arxiv.org/abs/2102.11378)).
|
|
483
|
+
TestGuard's ordering prior is measured the same way — from probed faults on a
|
|
484
|
+
real AI-authored codebase — and shrunk toward a neutral value, so one
|
|
485
|
+
observation of a class surviving never dominates a sweep. The cap is spread
|
|
486
|
+
across files: a single file with many candidates cannot make the others look
|
|
487
|
+
clean.
|
|
488
|
+
|
|
489
|
+
A sweep is weaker evidence than a probe, and says so. Nobody stated that the
|
|
490
|
+
behaviour mattered. It is stronger than nothing, which is what a repository
|
|
491
|
+
with no claims has — and the survivors worth defending become the first claims.
|
|
492
|
+
|
|
427
493
|
### Every change needs a claim
|
|
428
494
|
|
|
429
495
|
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
@@ -505,12 +571,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
505
571
|
# GitHub Actions
|
|
506
572
|
- uses: actions/checkout@v4
|
|
507
573
|
with: { fetch-depth: 0 }
|
|
508
|
-
- uses: raccioly/testguard@v0.
|
|
574
|
+
- uses: raccioly/testguard@v0.11.0
|
|
509
575
|
with: { command: gate }
|
|
510
576
|
|
|
511
577
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
512
578
|
include:
|
|
513
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
579
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.11.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
514
580
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
515
581
|
```
|
|
516
582
|
|
|
@@ -848,7 +914,7 @@ verdicts.
|
|
|
848
914
|
## Status
|
|
849
915
|
|
|
850
916
|
**v0.5.** Eleven commands (`status`, `init`, `claims`, `probe`, `admit`,
|
|
851
|
-
`replay`, `baseline`, `brief`, `gate`, `scaffold`, `mcp`), vitest, jest,
|
|
917
|
+
`replay`, `baseline`, `brief`, `gate`, `scaffold`, `sweep`, `mcp`), vitest, jest,
|
|
852
918
|
Playwright and Python (pytest / stdlib unittest) runners, hand-authored faults
|
|
853
919
|
plus a mechanical scaffold for JavaScript and Python, an agent operating layer
|
|
854
920
|
(`status`, `init`) and a change gate (`gate`). The contract spine — eight JSON
|
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.11.0` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.11.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -147,6 +147,7 @@ npx testguard-cli probe # try to falsify each claim; report
|
|
|
147
147
|
npx testguard-cli admit test/auth.test.ts --claim AUTH-ADMIN # does this test satisfy the two-gate rule?
|
|
148
148
|
npx testguard-cli baseline # freeze today's unproven findings; only new ones gate
|
|
149
149
|
npx testguard-cli gate --changed origin/main # fail when a changed source file carries no claim
|
|
150
|
+
npx testguard-cli sweep --changed origin/main # no claims yet? propose faults for the changed files and report what nothing noticed
|
|
150
151
|
npx testguard-cli brief --text # tell the agent where the suite is blind, before it writes
|
|
151
152
|
npx testguard-cli replay --since v1.0..HEAD # would this suite have caught the bugs that escaped?
|
|
152
153
|
npx testguard-cli mcp # serve the read-only loop over MCP, on stdio
|
|
@@ -400,6 +401,71 @@ on-demand helper, not a hook**: the session-start hook never touches the
|
|
|
400
401
|
network, and the helper exits 0 with a message when the CLI or the artifact
|
|
401
402
|
is missing. TestGuard itself still makes no network calls.
|
|
402
403
|
|
|
404
|
+
### Starting from zero claims
|
|
405
|
+
|
|
406
|
+
`gate` names the changed files that carry no claim, and stops there —
|
|
407
|
+
correctly, because stating a claim is a human act. But a project adopting this
|
|
408
|
+
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
409
|
+
The bottleneck was never verification; it is **oracle supply**.
|
|
410
|
+
|
|
411
|
+
`sweep` is the lowest rung of that supply. No claim, no concern, nothing
|
|
412
|
+
written by anybody:
|
|
413
|
+
|
|
414
|
+
```bash
|
|
415
|
+
npx testguard-cli sweep --changed origin/main # propose, probe a bounded selection, report
|
|
416
|
+
npx testguard-cli sweep --changed HEAD --include-dirty --cap 20
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
It takes the files `gate` just called uncovered, proposes faults with the same
|
|
420
|
+
mechanical producers `scaffold` uses, probes a bounded selection, and reports
|
|
421
|
+
what a green suite did not notice. A fault that survives needs no claim to be
|
|
422
|
+
alarming: something was deliberately broken and not one test failed.
|
|
423
|
+
|
|
424
|
+
```
|
|
425
|
+
swept 13 of 14 unclaimed changed files against origin/main.
|
|
426
|
+
proposed 510 faults, probed 10 (cap 10), deferred 500.
|
|
427
|
+
The 500 deferred are not a verdict: raise --cap, or sweep a smaller change.
|
|
428
|
+
|
|
429
|
+
9 findings — a deliberate break that no test noticed:
|
|
430
|
+
|
|
431
|
+
SURVIVED src/app/actions/register.ts:118 [write path]
|
|
432
|
+
[line 118] Field dropped: `password_hash` is no longer written.
|
|
433
|
+
src/tests/actions/register.test.ts mocks @/lib/prisma; 1 exact, 8 partial,
|
|
434
|
+
31 argument-free call assertions in the file. A field dropped from the write
|
|
435
|
+
payload fails only against an assertion that names that field exactly.
|
|
436
|
+
Assert the whole object written by the call this fault changes, including
|
|
437
|
+
`password_hash` — or read the record back and assert on what was stored.
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
Three rules make it safe to run on a repository that has never seen this tool:
|
|
441
|
+
|
|
442
|
+
- **It never writes `testguard.claims.json`**, and never will. A claim is a
|
|
443
|
+
sentence someone is willing to stand behind; a sentence nobody wrote is not
|
|
444
|
+
one. Its drafts land beside the evidence to keep or drop.
|
|
445
|
+
- **Its evidence never replaces `.testguard/evidence.json`.** A sweep probes
|
|
446
|
+
faults nobody stated, under TODO statements; folding that into the document
|
|
447
|
+
`status` and `baseline` read would corrupt the record of what the project
|
|
448
|
+
actually claims.
|
|
449
|
+
- **It does not fail on its own bad guess.** Only `survived` and `nocover`
|
|
450
|
+
exit 1. A proposal that would not compile, or an anchor that did not locate,
|
|
451
|
+
is reported and never gates — a tool that fails because its own guess was bad
|
|
452
|
+
is a tool people switch off.
|
|
453
|
+
|
|
454
|
+
The cap and the ordering are Google's: their mutation service surfaces at most
|
|
455
|
+
7 × |files| mutants per change and orders candidates on the measured
|
|
456
|
+
productivity of their operator in similar context, which took their productive
|
|
457
|
+
rate from 15% to 89%
|
|
458
|
+
([Petrović et al., *Practical Mutation Testing at Scale*, 2021](https://arxiv.org/abs/2102.11378)).
|
|
459
|
+
TestGuard's ordering prior is measured the same way — from probed faults on a
|
|
460
|
+
real AI-authored codebase — and shrunk toward a neutral value, so one
|
|
461
|
+
observation of a class surviving never dominates a sweep. The cap is spread
|
|
462
|
+
across files: a single file with many candidates cannot make the others look
|
|
463
|
+
clean.
|
|
464
|
+
|
|
465
|
+
A sweep is weaker evidence than a probe, and says so. Nobody stated that the
|
|
466
|
+
behaviour mattered. It is stronger than nothing, which is what a repository
|
|
467
|
+
with no claims has — and the survivors worth defending become the first claims.
|
|
468
|
+
|
|
403
469
|
### Every change needs a claim
|
|
404
470
|
|
|
405
471
|
`probe` can only verify claims that exist. Every escaped defect in the field
|
|
@@ -481,12 +547,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
481
547
|
# GitHub Actions
|
|
482
548
|
- uses: actions/checkout@v4
|
|
483
549
|
with: { fetch-depth: 0 }
|
|
484
|
-
- uses: raccioly/testguard@v0.
|
|
550
|
+
- uses: raccioly/testguard@v0.11.0
|
|
485
551
|
with: { command: gate }
|
|
486
552
|
|
|
487
553
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
488
554
|
include:
|
|
489
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
555
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.11.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
490
556
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
491
557
|
```
|
|
492
558
|
|
|
@@ -824,7 +890,7 @@ verdicts.
|
|
|
824
890
|
## Status
|
|
825
891
|
|
|
826
892
|
**v0.5.** Eleven commands (`status`, `init`, `claims`, `probe`, `admit`,
|
|
827
|
-
`replay`, `baseline`, `brief`, `gate`, `scaffold`, `mcp`), vitest, jest,
|
|
893
|
+
`replay`, `baseline`, `brief`, `gate`, `scaffold`, `sweep`, `mcp`), vitest, jest,
|
|
828
894
|
Playwright and Python (pytest / stdlib unittest) runners, hand-authored faults
|
|
829
895
|
plus a mechanical scaffold for JavaScript and Python, an agent operating layer
|
|
830
896
|
(`status`, `init`) and a change gate (`gate`). The contract spine — eight JSON
|
|
@@ -144,7 +144,7 @@
|
|
|
144
144
|
<div class="page">
|
|
145
145
|
<div class="masthead">
|
|
146
146
|
<div class="wordmark">Test<span class="mark">Guard</span></div>
|
|
147
|
-
<div class="meta">Technical Brief · v0.
|
|
147
|
+
<div class="meta">Technical Brief · v0.11.0 · MIT</div>
|
|
148
148
|
</div>
|
|
149
149
|
|
|
150
150
|
<h1>Your tests pass. That is not the same as your tests working.</h1>
|
|
Binary file
|
|
@@ -49,6 +49,8 @@ code were wrong, would anything fail?*
|
|
|
49
49
|
| Cost | What a probe spent, per claim and per defender file, derived from evidence | `src/probe/cost.mjs` |
|
|
50
50
|
| Progress | How a running probe reports itself, per reader: tty, plain, ndjson | `src/probe/progress.mjs` |
|
|
51
51
|
| Change gate | Claim coverage of a diff | `src/gate/changed.mjs` |
|
|
52
|
+
| Sweep | The cold start: propose faults for changed files that carry no claim, probe a bounded selection, report what nothing noticed | `src/sweep/sweep.mjs` |
|
|
53
|
+
| Claim supply | Which proposals are worth probing, and why a survivor on a write path was missed | `src/supply/select.mjs`, `src/supply/persistence.mjs` |
|
|
52
54
|
| Baseline | Freeze debt; gate the delta; re-stamp | `src/baseline/baseline.mjs` |
|
|
53
55
|
| Status | The single state machine every rendering derives from | `src/status/status.mjs` |
|
|
54
56
|
| Brief | The agent's session-start blind-spot block | `src/brief/brief.mjs` |
|
|
@@ -68,7 +70,8 @@ never the layers above.
|
|
|
68
70
|
|---|---|---|
|
|
69
71
|
| `cli.mjs` | commands | engine modules directly |
|
|
70
72
|
| `src/commands/` | engine modules, `spec/lib` | other commands |
|
|
71
|
-
| Engine (`probe`, `gate`, `baseline`, `status`, `brief`, `admit`, `replay`) | `spec/lib`, `src/util`, `src/git.mjs`, runners | commands, `cli.mjs` |
|
|
73
|
+
| Engine (`probe`, `gate`, `baseline`, `status`, `brief`, `admit`, `replay`, `sweep`) | `spec/lib`, `src/util`, `src/git.mjs`, runners, `src/supply` | commands, `cli.mjs` |
|
|
74
|
+
| Claim supply (`src/supply/`) | `src/util` | the probe orchestrator, commands, `cli.mjs` |
|
|
72
75
|
| `src/probe/runners/` | `src/util`, `shared.mjs` | the probe orchestrator |
|
|
73
76
|
| `spec/lib/` | nothing in `src/` | all of `src/` |
|
|
74
77
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "testguard-cli",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.11.0",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"ajv": "8.20.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# Include it from a project's .gitlab-ci.yml, pinned to a release:
|
|
4
4
|
#
|
|
5
5
|
# include:
|
|
6
|
-
# - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
6
|
+
# - remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.11.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
7
7
|
# inputs:
|
|
8
8
|
# dir: backend # where testguard.claims.json lives (default ".")
|
|
9
9
|
# post_note: true # post the brief as a merge-request note (needs TESTGUARD_GITLAB_TOKEN, api scope)
|
|
@@ -29,7 +29,7 @@ spec:
|
|
|
29
29
|
inputs:
|
|
30
30
|
version:
|
|
31
31
|
description: testguard-cli version to run
|
|
32
|
-
default: "0.
|
|
32
|
+
default: "0.11.0"
|
|
33
33
|
dir:
|
|
34
34
|
description: directory whose test runner is invoked (where testguard.claims.json lives)
|
|
35
35
|
default: "."
|
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
class Testguard < Formula
|
|
16
16
|
desc "Proves a test suite defends the claims a project makes"
|
|
17
17
|
homepage "https://github.com/raccioly/testguard"
|
|
18
|
-
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.
|
|
19
|
-
sha256 "
|
|
18
|
+
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.11.0.tgz"
|
|
19
|
+
sha256 "c2e36e07108e8d40197aa37cf8d23589c38dc265fddd589a4c890e4c6377cac4"
|
|
20
20
|
license "MIT"
|
|
21
21
|
|
|
22
22
|
depends_on "node"
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "testguard-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.11.0"
|
|
8
8
|
description = "Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest. Ships the Node.js CLI (requires Node.js 20+)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -7,7 +7,7 @@ import { reproduces, decimals, MAX_PLACES } from './wilson.mjs';
|
|
|
7
7
|
|
|
8
8
|
const schemaDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'schemas');
|
|
9
9
|
|
|
10
|
-
export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status', 'gate', 'replay']);
|
|
10
|
+
export const KINDS = Object.freeze(['claims', 'evidence', 'baseline', 'ignore', 'calibration', 'brief', 'status', 'gate', 'replay', 'sweep']);
|
|
11
11
|
/**
|
|
12
12
|
* How a document says it tried to falsify its claims. Absent means
|
|
13
13
|
* `fault-injection`, so every document written before the field existed is
|
|
@@ -305,6 +305,48 @@ const semantic = {
|
|
|
305
305
|
return errors;
|
|
306
306
|
},
|
|
307
307
|
|
|
308
|
+
sweep(doc) {
|
|
309
|
+
const errors = [];
|
|
310
|
+
const s = doc.selection;
|
|
311
|
+
// A cap that hides its remainder is a coverage claim nobody made, so the
|
|
312
|
+
// arithmetic that proves nothing was dropped is checked, not trusted.
|
|
313
|
+
if (s.proposed !== s.selected + s.deferred) errors.push({ path: '/selection/proposed', message: `proposed (${s.proposed}) must equal selected (${s.selected}) + deferred (${s.deferred})` });
|
|
314
|
+
if (s.selected > s.cap) errors.push({ path: '/selection/selected', message: `selected (${s.selected}) exceeds the cap (${s.cap})` });
|
|
315
|
+
if (doc.scope.swept > doc.scope.targets) errors.push({ path: '/scope/swept', message: `swept (${doc.scope.swept}) cannot exceed targets (${doc.scope.targets})` });
|
|
316
|
+
|
|
317
|
+
const byClass = Object.values(s.byClass);
|
|
318
|
+
const proposedByClass = byClass.reduce((a, c) => a + c.proposed, 0);
|
|
319
|
+
const selectedByClass = byClass.reduce((a, c) => a + c.selected, 0);
|
|
320
|
+
if (proposedByClass !== s.proposed) errors.push({ path: '/selection/byClass', message: `byClass proposed sums to ${proposedByClass}; selection.proposed is ${s.proposed}` });
|
|
321
|
+
if (selectedByClass !== s.selected) errors.push({ path: '/selection/byClass', message: `byClass selected sums to ${selectedByClass}; selection.selected is ${s.selected}` });
|
|
322
|
+
for (const [k, c] of Object.entries(s.byClass)) {
|
|
323
|
+
if (c.selected > c.proposed) errors.push({ path: `/selection/byClass/${k}`, message: `selected (${c.selected}) exceeds proposed (${c.proposed})` });
|
|
324
|
+
if (c.writePath > c.proposed) errors.push({ path: `/selection/byClass/${k}`, message: `writePath (${c.writePath}) exceeds proposed (${c.proposed})` });
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const probed = Object.values(doc.counts).reduce((a, n) => a + n, 0);
|
|
328
|
+
if (probed > s.selected) errors.push({ path: '/counts', message: `${probed} verdicts recorded for ${s.selected} selected faults` });
|
|
329
|
+
// Findings are every record that was not killed. Stating that here keeps a
|
|
330
|
+
// document from quietly reporting fewer findings than it reached verdicts.
|
|
331
|
+
const notKilled = probed - (doc.counts.killed ?? 0);
|
|
332
|
+
if (doc.findings.length !== notKilled) errors.push({ path: '/findings', message: `${doc.findings.length} findings for ${notKilled} records that were not killed; findings are every non-killed record` });
|
|
333
|
+
|
|
334
|
+
const gating = doc.findings.filter((f) => f.verdict === 'survived' || f.verdict === 'nocover').length;
|
|
335
|
+
if (gating > 0 && doc.exitCode !== 1) errors.push({ path: '/exitCode', message: 'a survived or nocover finding must exit 1; a sweep never passes over a deliberate break nothing noticed' });
|
|
336
|
+
if (gating === 0 && doc.exitCode !== 0) errors.push({ path: '/exitCode', message: 'exit 1 requires a survived or nocover finding; a sweep does not fail on its own unanchorable proposal' });
|
|
337
|
+
|
|
338
|
+
doc.findings.forEach((f, i) => {
|
|
339
|
+
if (f.verdict === 'killed') errors.push({ path: `/findings/${i}/verdict`, message: 'a killed fault is not a finding' });
|
|
340
|
+
// The signal explains why a defender could not notice a changed payload.
|
|
341
|
+
// On any other verdict, or off the write path, it explains nothing and
|
|
342
|
+
// would be exactly the non-actionable noise that gets a check ignored.
|
|
343
|
+
if (f.signals && !(f.verdict === 'survived' && f.writePath)) {
|
|
344
|
+
errors.push({ path: `/findings/${i}/signals`, message: 'signals belong to a survived finding on the write path' });
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
return errors;
|
|
348
|
+
},
|
|
349
|
+
|
|
308
350
|
brief(doc) {
|
|
309
351
|
const errors = [];
|
|
310
352
|
if (!doc.text.startsWith(doc.heading)) errors.push({ path: '/text', message: 'text must begin with heading' });
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$id": "urn:claimspec:v1:sweep",
|
|
3
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
4
|
+
"title": "claimspec v1 — sweep",
|
|
5
|
+
"description": "What a sweep proposed and what survived it. A sweep probes faults NOBODY STATED: it takes the changed source files that carry no claim, proposes faults with the mechanical producers, and reports the ones a green suite did not notice. Its findings are therefore weaker evidence than a probe's — no human said the behaviour mattered — and stronger than nothing, which is what a repository with no claims has. The drafts it carries are proposals for a human to keep or drop; they are not claims, and nothing here belongs in a claims file until someone states the sentence it defends.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"required": ["schemaVersion", "tool", "generatedAt", "ref", "base", "head", "includeDirty", "scope", "selection", "counts", "findings", "exitCode"],
|
|
8
|
+
"additionalProperties": false,
|
|
9
|
+
"properties": {
|
|
10
|
+
"$schema": { "type": "string" },
|
|
11
|
+
"schemaVersion": { "const": 1 },
|
|
12
|
+
"tool": { "$ref": "urn:claimspec:v1:common#/$defs/tool" },
|
|
13
|
+
"generatedAt": { "$ref": "urn:claimspec:v1:common#/$defs/isoDateTime" },
|
|
14
|
+
"ref": { "type": "string", "description": "The reference the change was measured against." },
|
|
15
|
+
"base": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
16
|
+
"head": { "$ref": "urn:claimspec:v1:common#/$defs/gitSha" },
|
|
17
|
+
"includeDirty": { "type": "boolean" },
|
|
18
|
+
"scope": {
|
|
19
|
+
"type": "object",
|
|
20
|
+
"required": ["changed", "targets", "swept"],
|
|
21
|
+
"additionalProperties": false,
|
|
22
|
+
"properties": {
|
|
23
|
+
"changed": { "type": "integer", "minimum": 0, "description": "Files in the change, before any filter." },
|
|
24
|
+
"targets": { "type": "integer", "minimum": 0, "description": "Changed SOURCE files carrying no claim — what the gate calls uncovered. A sweep proposes against these and nothing else." },
|
|
25
|
+
"swept": { "type": "integer", "minimum": 0, "description": "Targets that yielded at least one proposal. Never more than `targets`: a file the producers cannot touch is skipped, not swept." },
|
|
26
|
+
"skipped": {
|
|
27
|
+
"type": "array",
|
|
28
|
+
"description": "Targets that produced nothing, and why. Recorded rather than dropped: a file that silently yielded no proposal reads as a file with no findings.",
|
|
29
|
+
"items": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"required": ["file", "reason"],
|
|
32
|
+
"additionalProperties": false,
|
|
33
|
+
"properties": {
|
|
34
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
35
|
+
"reason": { "type": "string", "minLength": 1 }
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"selection": {
|
|
42
|
+
"type": "object",
|
|
43
|
+
"description": "What was proposed and what was probed. A sweep is capped — past a few findings per file nobody reads them, and the runner is spent either way — so the count NOT probed is reported beside the count that was. A cap that hides its own remainder is a coverage claim nobody made.",
|
|
44
|
+
"required": ["proposed", "selected", "deferred", "cap", "byClass"],
|
|
45
|
+
"additionalProperties": false,
|
|
46
|
+
"properties": {
|
|
47
|
+
"proposed": { "type": "integer", "minimum": 0 },
|
|
48
|
+
"selected": { "type": "integer", "minimum": 0 },
|
|
49
|
+
"deferred": { "type": "integer", "minimum": 0, "description": "Proposed and not probed. Not a verdict of any kind." },
|
|
50
|
+
"cap": { "type": "integer", "minimum": 1 },
|
|
51
|
+
"byClass": {
|
|
52
|
+
"type": "object",
|
|
53
|
+
"description": "Per fault class: how many were proposed, how many probed, and how many sat on a write payload.",
|
|
54
|
+
"additionalProperties": {
|
|
55
|
+
"type": "object",
|
|
56
|
+
"required": ["proposed", "selected", "writePath"],
|
|
57
|
+
"additionalProperties": false,
|
|
58
|
+
"properties": {
|
|
59
|
+
"proposed": { "type": "integer", "minimum": 0 },
|
|
60
|
+
"selected": { "type": "integer", "minimum": 0 },
|
|
61
|
+
"writePath": { "type": "integer", "minimum": 0 }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"counts": {
|
|
68
|
+
"type": "object",
|
|
69
|
+
"description": "Every verdict the probe reached, including `killed`. A sweep whose faults were all caught is a real result and must be readable as one.",
|
|
70
|
+
"additionalProperties": { "type": "integer", "minimum": 0 },
|
|
71
|
+
"propertyNames": { "$ref": "urn:claimspec:v1:common#/$defs/verdict" }
|
|
72
|
+
},
|
|
73
|
+
"findings": {
|
|
74
|
+
"type": "array",
|
|
75
|
+
"description": "Every record that was not killed, most actionable first. `survived` and `nocover` are statements about the test suite; the rest are statements about the PROPOSAL and do not gate, because a sweep's faults are machine-made and a tool that fails on its own bad guess gets switched off.",
|
|
76
|
+
"items": {
|
|
77
|
+
"type": "object",
|
|
78
|
+
"required": ["file", "claimId", "faultId", "faultClass", "verdict", "description", "writePath", "hint"],
|
|
79
|
+
"additionalProperties": false,
|
|
80
|
+
"properties": {
|
|
81
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
82
|
+
"line": { "type": "integer", "minimum": 1 },
|
|
83
|
+
"claimId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
84
|
+
"faultId": { "$ref": "urn:claimspec:v1:common#/$defs/identifier" },
|
|
85
|
+
"faultClass": { "$ref": "urn:claimspec:v1:common#/$defs/faultClass" },
|
|
86
|
+
"verdict": { "$ref": "urn:claimspec:v1:common#/$defs/verdict" },
|
|
87
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
88
|
+
"description": { "type": "string", "minLength": 1 },
|
|
89
|
+
"writePath": { "type": "boolean", "description": "The fault sits on a database write or the payload of one. Every surviving fault in the measurement that prompted this field was such a line." },
|
|
90
|
+
"hint": { "type": "string", "minLength": 1, "description": "One line naming the mechanism, never only the verdict." },
|
|
91
|
+
"signals": {
|
|
92
|
+
"type": "array",
|
|
93
|
+
"minItems": 1,
|
|
94
|
+
"description": "Static, annotation-grade signals about the defenders of a SURVIVOR on the write path. They never change a verdict. `persistence-payload-unasserted`: the defender mocks the persistence layer and the file's call assertions never name an exact payload, so a field dropped from a write cannot fail them. Descriptive, never predictive: no threshold on the assertion mix separates the files whose payload faults survive from the files whose do not.",
|
|
95
|
+
"items": {
|
|
96
|
+
"type": "object",
|
|
97
|
+
"required": ["file", "signal", "reason"],
|
|
98
|
+
"additionalProperties": false,
|
|
99
|
+
"properties": {
|
|
100
|
+
"file": { "$ref": "urn:claimspec:v1:common#/$defs/repoPath" },
|
|
101
|
+
"signal": { "enum": ["persistence-payload-unasserted"] },
|
|
102
|
+
"reason": { "type": "string", "minLength": 1 },
|
|
103
|
+
"specifiers": { "type": "array", "items": { "type": "string", "minLength": 1 } },
|
|
104
|
+
"counts": {
|
|
105
|
+
"type": "object",
|
|
106
|
+
"required": ["exact", "partial", "argless"],
|
|
107
|
+
"additionalProperties": false,
|
|
108
|
+
"properties": {
|
|
109
|
+
"exact": { "type": "integer", "minimum": 0 },
|
|
110
|
+
"partial": { "type": "integer", "minimum": 0 },
|
|
111
|
+
"argless": { "type": "integer", "minimum": 0 }
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
"exitCode": { "enum": [0, 1] },
|
|
121
|
+
"drafts": {
|
|
122
|
+
"type": "array",
|
|
123
|
+
"description": "The draft claims documents the proposals came from, carried so a human can keep one without re-running the sweep. Drafts, never claims: every statement is a TODO nobody has written yet.",
|
|
124
|
+
"items": { "type": "object" }
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
@@ -18,6 +18,7 @@ import { gateCommand } from './commands/gate.mjs';
|
|
|
18
18
|
import { replayCommand } from './commands/replay.mjs';
|
|
19
19
|
import { mcpCommand } from './commands/mcp.mjs';
|
|
20
20
|
import { admitCommand } from './commands/admit.mjs';
|
|
21
|
+
import { sweepCommand } from './commands/sweep.mjs';
|
|
21
22
|
|
|
22
23
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
23
24
|
const ISSUES = 'https://github.com/raccioly/testguard/issues';
|
|
@@ -32,6 +33,7 @@ export const USAGE = `testguard ${VERSION} — proves a test suite defends the c
|
|
|
32
33
|
testguard brief [dir] emit the blind-spot block for an agent's session-start context
|
|
33
34
|
testguard gate [dir] fail when a changed source file carries no claim and no excusing ignore entry
|
|
34
35
|
testguard scaffold <file> propose faults mechanically for one source file, as a draft claims document
|
|
36
|
+
testguard sweep [dir] propose faults for the changed source files that carry no claim, probe a bounded selection, and report what nothing noticed
|
|
35
37
|
testguard mcp serve the read-only status/brief/claims/evidence tools over MCP on stdio, so the loop works in any agent harness
|
|
36
38
|
testguard replay [dir] would this suite have caught the bugs that already escaped? Replays real fix commits and calibrates fault classes against them
|
|
37
39
|
testguard admit <test> --claim <ID> the two-gate rule as one verb: is this test green on HEAD and does it fail on every fault of the claim?
|
|
@@ -74,6 +76,10 @@ probe
|
|
|
74
76
|
--json the status document plus this run's result (records, newSinceBaseline, exitCode)
|
|
75
77
|
|
|
76
78
|
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
79
|
+
sweep --changed <ref> (required; CI bases and a safe local default are detected) --cap <n> (default 7 x unclaimed files)
|
|
80
|
+
--include-dirty --exclude <glob> --confirm <n> --budget <ms> --max <n> --out <path> --json
|
|
81
|
+
the cold start: no claim and no concern needed. Everything it proposes is a DRAFT — it never writes testguard.claims.json,
|
|
82
|
+
and its evidence never replaces .testguard/evidence.json. exit 1 on a fault that survived, or a file no test imports
|
|
77
83
|
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
78
84
|
· security flag/window/cost literal weakened · verify/validate/check call removed
|
|
79
85
|
· field dropped from a payload/allow-list/schema/merge · parameter-derived argument swapped for undefined/{}
|
|
@@ -117,7 +123,7 @@ brief --evidence <path> --baseline <path> --max <n> --text (print only;
|
|
|
117
123
|
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or unclaimed changes) · 2 precondition failed · 3 usage
|
|
118
124
|
`;
|
|
119
125
|
|
|
120
|
-
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand };
|
|
126
|
+
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand, sweep: sweepCommand };
|
|
121
127
|
|
|
122
128
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
123
129
|
let parsed;
|
|
@@ -143,6 +149,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
143
149
|
changed: { type: 'string' },
|
|
144
150
|
since: { type: 'string' },
|
|
145
151
|
exclude: { type: 'string', multiple: true },
|
|
152
|
+
cap: { type: 'string' },
|
|
146
153
|
strict: { type: 'boolean', default: false },
|
|
147
154
|
explain: { type: 'boolean', default: false },
|
|
148
155
|
ignore: { type: 'string' },
|