testguard-cli 0.15.0__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testguard_cli-0.16.0/.github/scripts/install-smoke.mjs +94 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/CHANGELOG.md +64 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/PKG-INFO +63 -9
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/README.md +62 -8
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/action.yml +6 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs/testguard-explained.html +1 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs/testguard-explained.pdf +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/DATA-MODEL.md +20 -6
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/REQUIREMENTS.md +5 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/TEST-SPEC.md +13 -6
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/package-lock.json +2 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/package.json +1 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/packaging/gitlab/testguard.gitlab-ci.yml +2 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/packaging/homebrew/testguard.rb +2 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/pyproject.toml +1 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/GATE-SEMANTICS.md +55 -5
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/schemas.test.mjs +64 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/lib/validate.mjs +35 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/claims.schema.json +6 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/common.schema.json +10 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/evidence.schema.json +4 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/gate.schema.json +1 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/status.schema.json +27 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/cli.mjs +30 -3
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/admit.mjs +8 -3
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/claims.mjs +40 -4
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/probe.mjs +13 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/gate/changed.mjs +45 -4
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/init/init.mjs +6 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/init/templates/SKILL.md +44 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/attribution.mjs +9 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/cost.mjs +16 -8
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/probe.mjs +28 -24
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/discovery.mjs +4 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/shared.mjs +16 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/status/status.mjs +15 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/status/surface.mjs +3 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/attribution.test.mjs +20 -1
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/claim-selection.test.mjs +52 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/cost.test.mjs +20 -3
- testguard_cli-0.16.0/test/fault-defender-orchestration.test.mjs +44 -0
- testguard_cli-0.16.0/test/fault-defenders.test.mjs +95 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/gate.test.mjs +53 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/init.test.mjs +36 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-discovery.test.mjs +35 -2
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-vitest.test.mjs +51 -3
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/status.test.mjs +22 -3
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/testguard.claims.json +90 -4
- testguard_cli-0.15.0/.github/scripts/install-smoke.mjs +0 -55
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.agents/skills/testguard/SKILL.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.docguard.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.docguardignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.gitattributes +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/dependabot.yml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/scripts/approve-held-runs.sh +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/scripts/build-one-pager.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/scripts/check-cost-budget.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/scripts/check-python-fixture.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/scripts/publish-homebrew-tap.sh +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/scripts/sync-release-version.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/workflows/auto-merge.yml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/workflows/ci.yml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/workflows/release.yml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/workflows/scheduled-release.yml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.github/workflows/supply-chain.yml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.gitignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.npmignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.npmrc +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.pre-commit-config.yaml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/.pre-commit-hooks.yaml +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/AGENTS.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/CLAUDE.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/CONTRIBUTING.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/DRIFT-LOG.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/LICENSE +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/PRIVACY.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/SECURITY.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/SUPPORT.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/bench/README.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/cli/testguard.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/ARCHITECTURE.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/ENVIRONMENT.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/PRIOR-ART.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/docs-canonical/SECURITY.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/README.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/expected.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/test/export-mocked.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/testguard.claims.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/.gitignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/README.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/expected.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/jest.config.js +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/package.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/src/export.js +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/src/redact.js +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/test/export-mocked.test.js +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/test/flaky.test.js +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/test/redact.test.js +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-jest/testguard.claims.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/.gitignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/README.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/e2e/flaky.spec.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/e2e/toggle.spec.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/expected.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/package-lock.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/package.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/playwright.config.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/src/toggle.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/test/toggle.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/testguard.claims.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-playwright/vitest.config.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/.gitignore +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/README.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/demo/__init__.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/demo/export.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/demo/orphan.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/demo/pipeline.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/demo/redact.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/expected.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/testguard.claims.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/tests/test_export_mocked.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/tests/test_flaky.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/tests/test_patched.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/fixtures/known-answer-python/tests/test_redact.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/README.md +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/baseline.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/calibration-websec.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/claims-methods.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence-disturbed-run.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence-negative-control.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence-persistence-signal.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence-provisional.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence-python.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence-scan-method.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/evidence.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/gate.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/replay.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/examples/status.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.backoff-cell-not-wilson.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.breakdown-does-not-sum-to-n.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.ci-not-wilson.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.compound-key-arity-mismatch.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.fallback-without-basis.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.p-not-positives-over-n.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.fault-injection-without-a-substitution.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.method-detail-on-a-specified-method.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.annotated-signal-without-reason.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.attribute-signal-not-a-defender.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.attribute-signal-without-attributes.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.byrunner-file-not-resolved.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.contention-without-runners.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.discovered-defender-mocks-subject.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.fault-injection-without-its-evidence.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.ignored-dirty-on-snapshot.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.method-detail-on-an-injecting-run.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.negative-control-not-reached-but-survived.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.negative-control-reached-on-a-kill.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.persistence-signal-not-a-defender.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.persistence-signal-on-a-non-survivor.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.persistence-signal-without-reason.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.probe-error-without-message.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.provisional-flag-on-confirmed-run.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.restore-skipped-in-place.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.single-run-without-provisional.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.target-not-imported-but-killed.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/gate.counts-disagree.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/gate.exit-zero-with-uncovered.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/gate.ignore-cover-without-pattern.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/replay.blind-with-a-failing-run.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/replay.duplicate-patch-id.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/replay.nocover-when-no-test-file-remained.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.claim-without-file.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.clean-with-new-findings.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.invalid-anchor-hidden.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.partial-scope-counts-disagree.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.surface-counts-disagree.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.unclaimed-hidden-behind-unproven.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/conformance/invalid/status.write-test-without-target.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/lib/wilson.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/baseline.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/brief.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/concerns.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/replay.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/spec/schemas/sweep.schema.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/admit/admit.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/baseline/baseline.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/brief/brief.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/claims/anchors.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/claims/load.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/claims/removed.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/command-budget.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/baseline.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/brief.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/concerns.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/gate.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/init.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/mcp.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/replay.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/scaffold.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/status.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/commands/sweep.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/git.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/mcp/server.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/mcp/tools.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/classify.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/contention.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/discover.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/imports.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/independence.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/mocks.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/progress.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/pyimports.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/rank.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/index.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/jest.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/playwright.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/python/_testguard_pytest_plugin.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/python/_testguard_report.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/python/_testguard_unittest_main.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/python.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/runners/vitest.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/probe/worktree.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/render.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/replay/label.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/replay/replay.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/scaffold/producers.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/scaffold/producers.python.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/scaffold/scaffold.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/supply/concerns.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/supply/feedback.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/supply/persistence.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/supply/savepath.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/supply/select.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/sweep/sweep.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/util/glob.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/src/util/hash.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/admit-decision.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/admit.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/brief.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/calibration.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/claims-anchors.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/claims-since.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/classify.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/command-budget-write-boundaries.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/command-budget.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/contention.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/discover.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/evidence-persistence-signal.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/gitlab-template.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/glob.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/imports.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/independence.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/inject.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/label.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/mcp.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/mocks.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/negative-control.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/one-pager.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe-dirty-paths.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe-error.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe-native-manifest.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe-persistence-signal.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe-preconditions.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe.fixture.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe.jest.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe.playwright.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/probe.python.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/progress.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/pyimports.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/rank-aliases.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/release-approval.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/release-sync.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/replay-mixed-runners.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/replay-no-test-file-left.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/replay-python-defenders.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/replay-verdict.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/replay.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-argv.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-command.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-playwright.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-python.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/runner-selection-blind.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/samples/playwright-report.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/samples/python-report-collect-error.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/samples/python-report.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/scaffold.python.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/scaffold.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/snapshot.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/supply-concerns.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/supply-feedback.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/supply-persistence.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/supply-savepath.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/supply-select.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/surface.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/sweep.fixture.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/sweep.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/vitest-config.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/worktree.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/test/writer.test.mjs +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/testguard.cost-budget.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/testguard.ignore.json +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.15.0 → testguard_cli-0.16.0}/vitest.config.mjs +0 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Prove the package works AS INSTALLED, not as checked out: pack it, install
|
|
4
|
+
* the tarball into a scratch project with production dependencies only, and
|
|
5
|
+
* run the CLI from there. This is the gate that would have caught 0.1.0,
|
|
6
|
+
* which shipped with its schema validator's dependency declared as a
|
|
7
|
+
* devDependency and crashed on startup for every npm/npx/pip/brew user.
|
|
8
|
+
*/
|
|
9
|
+
import { execFileSync } from 'node:child_process';
|
|
10
|
+
import { mkdtempSync, mkdirSync, rmSync, cpSync, readdirSync, writeFileSync, existsSync } from 'node:fs';
|
|
11
|
+
import { tmpdir } from 'node:os';
|
|
12
|
+
import { join, resolve } from 'node:path';
|
|
13
|
+
|
|
14
|
+
const root = resolve(process.argv[1], '..', '..', '..');
|
|
15
|
+
const run = (cmd, args, cwd) => execFileSync(cmd, args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] });
|
|
16
|
+
const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
|
|
17
|
+
|
|
18
|
+
const scratch = mkdtempSync(join(tmpdir(), 'testguard-install-smoke-'));
|
|
19
|
+
try {
|
|
20
|
+
const packDir = join(scratch, 'pack');
|
|
21
|
+
mkdirSync(packDir);
|
|
22
|
+
run(npm, ['pack', '--pack-destination', packDir, '--ignore-scripts'], root);
|
|
23
|
+
const tgz = join(packDir, readdirSync(packDir).find((f) => f.endsWith('.tgz')));
|
|
24
|
+
|
|
25
|
+
const consumer = join(scratch, 'consumer');
|
|
26
|
+
cpSync(join(root, 'fixtures', 'known-answer'), consumer, { recursive: true, filter: (s) => !/node_modules|\.flake-counter|\.testguard/.test(s) });
|
|
27
|
+
run(npm, ['init', '-y'], consumer);
|
|
28
|
+
run(npm, ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tgz], consumer);
|
|
29
|
+
|
|
30
|
+
const bin = join(consumer, 'node_modules', '.bin', process.platform === 'win32' ? 'testguard.cmd' : 'testguard');
|
|
31
|
+
const version = run(bin, ['--version'], consumer).trim();
|
|
32
|
+
const emptyProject = join(scratch, 'empty-adoption');
|
|
33
|
+
mkdirSync(emptyProject);
|
|
34
|
+
writeFileSync(join(emptyProject, 'testguard.claims.json'), JSON.stringify({ schemaVersion: 1, claims: [] }));
|
|
35
|
+
const emptyStatus = JSON.parse(run(bin, ['probe', emptyProject, '--allow-empty', '--json'], consumer));
|
|
36
|
+
if (emptyStatus.state !== 'no-claims' || emptyStatus.run || existsSync(join(emptyProject, '.testguard', 'evidence.json'))) throw new Error('installed empty-adoption probe invented verification evidence');
|
|
37
|
+
const replayHelp = run(bin, ['replay', '--help'], consumer);
|
|
38
|
+
if (!replayHelp.includes('--since') || replayHelp.includes('--node-modules')) throw new Error('installed command help is not command-specific');
|
|
39
|
+
writeFileSync(join(emptyProject, 'vite.config.mjs'), "export default { base: '/', plugins: [{ configureServer(s) { s.middlewares.use('/__rooms', () => {}); const parts = '/__room'.split('/'); } }] };\n");
|
|
40
|
+
const installedDiscovery = join(consumer, 'node_modules', 'testguard-cli', 'src', 'probe', 'runners', 'discovery.mjs');
|
|
41
|
+
run(process.execPath, ['--input-type=module', '-e',
|
|
42
|
+
"import { pathToFileURL } from 'node:url'; const { hashDiscoveryConfigs } = await import(pathToFileURL(process.argv[1])); hashDiscoveryConfigs(process.argv[2], ['vite.config.mjs']);",
|
|
43
|
+
installedDiscovery, emptyProject,
|
|
44
|
+
], consumer);
|
|
45
|
+
const installedRunner = join(consumer, 'node_modules', 'testguard-cli', 'src', 'probe', 'runners', 'shared.mjs');
|
|
46
|
+
run(process.execPath, ['--input-type=module', '-e',
|
|
47
|
+
`import { pathToFileURL } from 'node:url'; const { parseReport } = await import(pathToFileURL(process.argv[1]));
|
|
48
|
+
const result = parseReport({ success: false, numTotalTests: 2, numFailedTests: 2, testResults: [{ name: 'test.mjs', status: 'failed', assertionResults: [
|
|
49
|
+
{ status: 'failed', failureMessages: ['AssertionError: expected /specify timed out/'] },
|
|
50
|
+
{ status: 'failed', failureMessages: ['Error: Test timed out in 1000ms.'] },
|
|
51
|
+
] }] }, 1);
|
|
52
|
+
if (result.timeouts !== 1 || result.run.assertionFailures !== 1) throw new Error('installed runner confuses assertion text with timeout identity');`,
|
|
53
|
+
installedRunner,
|
|
54
|
+
], consumer);
|
|
55
|
+
const claims = run(bin, ['claims', '.'], consumer);
|
|
56
|
+
const draft = JSON.parse(run(bin, ['scaffold', 'src/redact.mjs', '--json'], consumer));
|
|
57
|
+
const installedValidator = join(consumer, 'node_modules', 'testguard-cli', 'spec', 'lib', 'validate.mjs');
|
|
58
|
+
const candidateEvidence = join(root, 'spec', 'conformance', 'examples', 'evidence.json');
|
|
59
|
+
const validatesCandidateEvidence = run(process.execPath, [
|
|
60
|
+
'--input-type=module',
|
|
61
|
+
'-e',
|
|
62
|
+
`import { pathToFileURL } from 'node:url'; import { readFileSync } from 'node:fs';
|
|
63
|
+
const { validate } = await import(pathToFileURL(process.argv[1]));
|
|
64
|
+
const candidate = JSON.parse(readFileSync(process.argv[2], 'utf8'));
|
|
65
|
+
for (const r of candidate.records) {
|
|
66
|
+
r.defenders.selectionSource = r.defenders.requested.length ? 'claim' : 'discovery';
|
|
67
|
+
if (!r.defenders.requested.length) r.defenders.discovered = true;
|
|
68
|
+
}
|
|
69
|
+
const result = validate('evidence', candidate);
|
|
70
|
+
if (!result.ok) throw new Error(result.errors.map((error) => \`${'${error.path}: ${error.message}'}\`).join('; '));
|
|
71
|
+
const claims = JSON.parse(readFileSync(process.argv[4], 'utf8'));
|
|
72
|
+
claims.claims[0].faults[0].defendedBy = [];
|
|
73
|
+
if (!validate('claims', claims).ok) throw new Error('installed package rejected a fault discovery override');
|
|
74
|
+
claims.claims[0].faults[0].defendedBy = ['test/unit.test.mjs'];
|
|
75
|
+
if (!validate('claims', claims).ok) throw new Error('installed package rejected a fault defender override');
|
|
76
|
+
const gate = JSON.parse(readFileSync(process.argv[3], 'utf8'));
|
|
77
|
+
gate.nested = [{ project: 'backend', files: ['backend/src/guard.mjs'] }]; gate.changed++;
|
|
78
|
+
const nestedResult = validate('gate', gate);
|
|
79
|
+
if (!nestedResult.ok) throw new Error('installed package rejected the nested-project gate contract');
|
|
80
|
+
process.stdout.write('ok');`,
|
|
81
|
+
installedValidator,
|
|
82
|
+
candidateEvidence,
|
|
83
|
+
join(root, 'spec', 'conformance', 'examples', 'gate.json'),
|
|
84
|
+
join(root, 'spec', 'conformance', 'examples', 'claims.json'),
|
|
85
|
+
], consumer).trim();
|
|
86
|
+
if (!draft.claims?.length) throw new Error('scaffold produced no claims from the installed tarball');
|
|
87
|
+
if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
|
|
88
|
+
if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
|
|
89
|
+
if (validatesCandidateEvidence !== 'ok') throw new Error('installed package did not validate candidate-format evidence');
|
|
90
|
+
const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
|
|
91
|
+
console.log(`install smoke OK — testguard ${version} runs from the installed tarball; consumer deps: ${deps.join(', ')}`);
|
|
92
|
+
} finally {
|
|
93
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
94
|
+
}
|
|
@@ -7,6 +7,70 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.16.0] - 2026-09-30
|
|
11
|
+
|
|
12
|
+
Automated weekly release — everything merged since `v0.15.1`.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- feat: fault-specific defenders, provenance and cost reporting (#184)
|
|
17
|
+
- Add advisory AI update guidance and stabilize cleanup regression (#183)
|
|
18
|
+
- chore(homebrew): sha256 for v0.15.1 (#182)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Fault-specific `defendedBy` overrides, with explicit empty-list discovery,
|
|
24
|
+
evidence selection provenance, per-fault cost reporting and prior-kill
|
|
25
|
+
narrowing warnings (#90). Selection changes invalidate reuse and status;
|
|
26
|
+
admission and changed-file coverage use the actual selected defenders.
|
|
27
|
+
|
|
28
|
+
### Changed
|
|
29
|
+
|
|
30
|
+
- New AI integrations include permission-aware, once-per-session update advice
|
|
31
|
+
that identifies the actual CLI and requests approval before any upgrade.
|
|
32
|
+
CLI commands and session-start hooks remain offline. Existing customized
|
|
33
|
+
instructions remain untouched; safe manual refresh is documented.
|
|
34
|
+
|
|
35
|
+
### Fixed
|
|
36
|
+
|
|
37
|
+
- The daemon-cleanup regression test now requires explicit startup and
|
|
38
|
+
post-cleanup release signals instead of assuming a detached process starts
|
|
39
|
+
and writes within fixed sleeps. Timeout and cleanup semantics are unchanged.
|
|
40
|
+
|
|
41
|
+
## [0.15.1] - 2026-09-30
|
|
42
|
+
|
|
43
|
+
Automated weekly release — everything merged since `v0.15.0`.
|
|
44
|
+
|
|
45
|
+
### Changed
|
|
46
|
+
|
|
47
|
+
- fix: runner timeout identity and reported adoption regressions (#180)
|
|
48
|
+
- chore(homebrew): sha256 for v0.15.0 (#177)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
### Fixed
|
|
52
|
+
|
|
53
|
+
- Assertion messages quoting "timed out" no longer become runner timeouts.
|
|
54
|
+
Native timeout headers and structured timeout failures remain non-detections
|
|
55
|
+
(#179).
|
|
56
|
+
- Configuration dependency hashing no longer rejects harmless absolute-looking
|
|
57
|
+
strings such as Vite URL bases, middleware routes, or `split('/')`. Existing
|
|
58
|
+
outside-project file dependencies and symlinks still fail closed (#178).
|
|
59
|
+
- Discovery failures no longer assert that dependencies are missing when the
|
|
60
|
+
runner resolved successfully but its configuration could not be discovered.
|
|
61
|
+
- Each subcommand's `--help` now shows its own options and an example; global
|
|
62
|
+
`--help` exits successfully and points to command-specific help (#120).
|
|
63
|
+
- Parent gates, status change reports, and claimed-surface counts respect valid
|
|
64
|
+
nested TestGuard project boundaries. Delegated files are explicitly reported
|
|
65
|
+
and require a separate child gate; invalid or symlinked markers fail (#150).
|
|
66
|
+
|
|
67
|
+
### Added
|
|
68
|
+
|
|
69
|
+
- Explicit `probe --allow-empty` adoption mode, also available as an opt-in
|
|
70
|
+
GitHub Action input. A valid empty claims file skips verification without
|
|
71
|
+
writing evidence; JSON keeps the `no-claims` state. Default empty probes,
|
|
72
|
+
invalid claims, and explicit claim selections still fail (#176).
|
|
73
|
+
|
|
10
74
|
## [0.15.0] - 2026-09-29
|
|
11
75
|
|
|
12
76
|
Automated weekly release — everything merged since `v0.14.2`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.16.0
|
|
4
4
|
Summary: Breaks your code on purpose and reports every promise your tests did not notice breaking. Mutation testing bound to the claims a project makes — vitest, jest, Playwright, pytest and unittest. Ships the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -151,9 +151,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
151
151
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
152
152
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
153
153
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
154
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
154
|
+
| GitHub Action | `uses: raccioly/testguard@v0.16.0` — see [`action.yml`](./action.yml) |
|
|
155
155
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
156
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
156
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
157
157
|
|
|
158
158
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
159
159
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -245,6 +245,14 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
245
245
|
the evidence records them as `repo.ignoredDirty`, so the run says what it
|
|
246
246
|
did not look at. Every summary names the commit probed.
|
|
247
247
|
|
|
248
|
+
A fault may declare its own `defendedBy` to override the claim's set.
|
|
249
|
+
Omit the fault field to inherit; set it to `[]` to request discovery rather
|
|
250
|
+
than inheritance. Evidence names the selection origin and selected files.
|
|
251
|
+
Changing either invalidates reuse and makes status stale. A narrowed fault
|
|
252
|
+
set after a prior kill is warned about by `claims`; re-probe before trusting
|
|
253
|
+
it. For example, a unit-level fault can select `["test/unit.test.mjs"]`
|
|
254
|
+
while an integration fault inherits the claim's broader set.
|
|
255
|
+
|
|
248
256
|
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
249
257
|
files that import the fault's target, by relative path or resolved alias
|
|
250
258
|
(tsconfig `paths`, through `extends` and `references`, vite/vitest
|
|
@@ -352,6 +360,34 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
352
360
|
offline. The brief's first line says which install answered
|
|
353
361
|
(`local install` or `global`), so a stale one is visible.
|
|
354
362
|
|
|
363
|
+
### Advisory updates for AI sessions
|
|
364
|
+
|
|
365
|
+
New integrations created by `testguard init` tell the AI to identify the actual
|
|
366
|
+
CLI (project-local before global) and, once per session when network policy
|
|
367
|
+
permits, read the registry's stable version:
|
|
368
|
+
|
|
369
|
+
```bash
|
|
370
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
This is an optional check performed by the AI, not by TestGuard's CLI or hook.
|
|
374
|
+
A newer stable version produces a suggestion and a request for approval, never
|
|
375
|
+
an automatic install. Project pins and the original package manager still
|
|
376
|
+
apply; prereleases and development checkouts are not replaced. Offline or
|
|
377
|
+
failed checks do not change verdicts, evidence or exit codes, and do not prove
|
|
378
|
+
that an installation is current. AI instructions are guidance, not a guarantee
|
|
379
|
+
that every harness performs the check.
|
|
380
|
+
|
|
381
|
+
Upgrading the package does **not** refresh an existing copied skill or an
|
|
382
|
+
existing managed section already listing the project. Compare the upgraded
|
|
383
|
+
package's `src/init/templates/SKILL.md` with
|
|
384
|
+
`.claude/skills/testguard/SKILL.md` and merge approved changes, preserving local
|
|
385
|
+
customizations. For an existing `AGENTS.md` section, add the same advisory
|
|
386
|
+
policy after reviewing it. `testguard init --force` replaces the whole skill
|
|
387
|
+
but does not refresh an already-listed managed section; use it only with
|
|
388
|
+
explicit overwrite approval and review the diff. No new automatic refresh
|
|
389
|
+
mechanism is introduced.
|
|
390
|
+
|
|
355
391
|
### Would this suite have caught the bugs that already escaped?
|
|
356
392
|
|
|
357
393
|
An injected fault is a fault somebody thought of. A bug that actually shipped
|
|
@@ -431,6 +467,20 @@ is missing. TestGuard itself still makes no network calls.
|
|
|
431
467
|
|
|
432
468
|
### Starting from zero claims
|
|
433
469
|
|
|
470
|
+
During gradual adoption, use `testguard probe --allow-empty` to skip a valid
|
|
471
|
+
empty claims file without blocking a push. It explicitly reports that no
|
|
472
|
+
verification was performed and writes no evidence; `--json` retains the
|
|
473
|
+
`no-claims` state. Missing or invalid claims files still fail. Run `gate`
|
|
474
|
+
separately to enforce changed-file coverage. For the pre-commit probe hook,
|
|
475
|
+
set `args: [--allow-empty]`; for the GitHub Action, set `allow-empty: 'true'`.
|
|
476
|
+
The default probe still exits 2 when no claims exist.
|
|
477
|
+
|
|
478
|
+
In a monorepo, a valid nested `testguard.claims.json` marks a separate project.
|
|
479
|
+
The parent gate reports its changed files as delegated and does not count them
|
|
480
|
+
as parent coverage. Run `gate` in each child project as well; a passing parent
|
|
481
|
+
gate does not establish that child coverage passed. Invalid child claims fail
|
|
482
|
+
evaluation instead of hiding files.
|
|
483
|
+
|
|
434
484
|
`gate` names the changed files that carry no claim, and stops there —
|
|
435
485
|
correctly, because stating a claim is a human act. But a project adopting this
|
|
436
486
|
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
@@ -687,12 +737,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
687
737
|
# GitHub Actions
|
|
688
738
|
- uses: actions/checkout@v4
|
|
689
739
|
with: { fetch-depth: 0 }
|
|
690
|
-
- uses: raccioly/testguard@v0.
|
|
740
|
+
- uses: raccioly/testguard@v0.16.0
|
|
691
741
|
with: { command: gate }
|
|
692
742
|
|
|
693
743
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
694
744
|
include:
|
|
695
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
745
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
696
746
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
697
747
|
```
|
|
698
748
|
|
|
@@ -965,19 +1015,23 @@ most expensive claims
|
|
|
965
1015
|
4m 58s TG-FAULT-EDIT-VISIBLE 1 fault · 6 runs
|
|
966
1016
|
|
|
967
1017
|
defender files, by the time of the runs that included them
|
|
968
|
-
(a
|
|
1018
|
+
(a fault runs its selected defender set at once, so these overlap and do not sum to the total)
|
|
969
1019
|
24m 3s test/probe.fixture.test.mjs 5 claims
|
|
970
1020
|
|
|
971
|
-
shared defenders — each
|
|
1021
|
+
shared defenders — each fault selects the file; durations include recorded baseline sharing
|
|
972
1022
|
24m 3s test/probe.fixture.test.mjs
|
|
973
|
-
named by TG-ESCALATION-N-RUNS, TG-FAULT-EDIT-VISIBLE, …
|
|
1023
|
+
named by TG-ESCALATION-N-RUNS/F1, TG-FAULT-EDIT-VISIBLE/F1, …
|
|
974
1024
|
```
|
|
975
1025
|
|
|
976
1026
|
Every number is read back out of the `durationMs` the probe already records.
|
|
977
1027
|
`--cost` re-measures nothing, spawns nothing, and costs the price of reading
|
|
978
1028
|
one JSON file.
|
|
979
1029
|
|
|
980
|
-
|
|
1030
|
+
Cost output also lists individual faults, their selected defenders and selection
|
|
1031
|
+
origin. Sharing is counted across distinct fault identities, including siblings
|
|
1032
|
+
within one claim; repeated confirmation runs alone are not sharing.
|
|
1033
|
+
|
|
1034
|
+
**The per-file figures overlap.** A run executes a fault's whole defender set
|
|
981
1035
|
at once, so the runner never says how much of it belonged to which file. A
|
|
982
1036
|
file's `ms` is the time of every run that *included* it: an upper bound on
|
|
983
1037
|
what removing it could save, which is the decision you are making. Only
|
|
@@ -127,9 +127,9 @@ raises coverage. TestGuard admits it because it fails when the claim is false.
|
|
|
127
127
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
128
128
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
129
129
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
130
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
130
|
+
| GitHub Action | `uses: raccioly/testguard@v0.16.0` — see [`action.yml`](./action.yml) |
|
|
131
131
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
132
|
-
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
132
|
+
| GitLab CI | `include: - remote: https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml` with `inputs:` — see [`packaging/gitlab/`](./packaging/gitlab/testguard.gitlab-ci.yml) |
|
|
133
133
|
|
|
134
134
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
135
135
|
less than that many days ago (`ENOVERSIONS`); install that one with
|
|
@@ -221,6 +221,14 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
221
221
|
the evidence records them as `repo.ignoredDirty`, so the run says what it
|
|
222
222
|
did not look at. Every summary names the commit probed.
|
|
223
223
|
|
|
224
|
+
A fault may declare its own `defendedBy` to override the claim's set.
|
|
225
|
+
Omit the fault field to inherit; set it to `[]` to request discovery rather
|
|
226
|
+
than inheritance. Evidence names the selection origin and selected files.
|
|
227
|
+
Changing either invalidates reuse and makes status stale. A narrowed fault
|
|
228
|
+
set after a prior kill is warned about by `claims`; re-probe before trusting
|
|
229
|
+
it. For example, a unit-level fault can select `["test/unit.test.mjs"]`
|
|
230
|
+
while an integration fault inherits the claim's broader set.
|
|
231
|
+
|
|
224
232
|
A claim with no `defendedBy` has its defenders **discovered**: the test
|
|
225
233
|
files that import the fault's target, by relative path or resolved alias
|
|
226
234
|
(tsconfig `paths`, through `extends` and `references`, vite/vitest
|
|
@@ -328,6 +336,34 @@ npx testguard-cli admit test/x.test.ts --claim X # is this test green on HEAD
|
|
|
328
336
|
offline. The brief's first line says which install answered
|
|
329
337
|
(`local install` or `global`), so a stale one is visible.
|
|
330
338
|
|
|
339
|
+
### Advisory updates for AI sessions
|
|
340
|
+
|
|
341
|
+
New integrations created by `testguard init` tell the AI to identify the actual
|
|
342
|
+
CLI (project-local before global) and, once per session when network policy
|
|
343
|
+
permits, read the registry's stable version:
|
|
344
|
+
|
|
345
|
+
```bash
|
|
346
|
+
npm view testguard-cli dist-tags.latest --json --fetch-retries=0 --fetch-timeout=5000
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
This is an optional check performed by the AI, not by TestGuard's CLI or hook.
|
|
350
|
+
A newer stable version produces a suggestion and a request for approval, never
|
|
351
|
+
an automatic install. Project pins and the original package manager still
|
|
352
|
+
apply; prereleases and development checkouts are not replaced. Offline or
|
|
353
|
+
failed checks do not change verdicts, evidence or exit codes, and do not prove
|
|
354
|
+
that an installation is current. AI instructions are guidance, not a guarantee
|
|
355
|
+
that every harness performs the check.
|
|
356
|
+
|
|
357
|
+
Upgrading the package does **not** refresh an existing copied skill or an
|
|
358
|
+
existing managed section already listing the project. Compare the upgraded
|
|
359
|
+
package's `src/init/templates/SKILL.md` with
|
|
360
|
+
`.claude/skills/testguard/SKILL.md` and merge approved changes, preserving local
|
|
361
|
+
customizations. For an existing `AGENTS.md` section, add the same advisory
|
|
362
|
+
policy after reviewing it. `testguard init --force` replaces the whole skill
|
|
363
|
+
but does not refresh an already-listed managed section; use it only with
|
|
364
|
+
explicit overwrite approval and review the diff. No new automatic refresh
|
|
365
|
+
mechanism is introduced.
|
|
366
|
+
|
|
331
367
|
### Would this suite have caught the bugs that already escaped?
|
|
332
368
|
|
|
333
369
|
An injected fault is a fault somebody thought of. A bug that actually shipped
|
|
@@ -407,6 +443,20 @@ is missing. TestGuard itself still makes no network calls.
|
|
|
407
443
|
|
|
408
444
|
### Starting from zero claims
|
|
409
445
|
|
|
446
|
+
During gradual adoption, use `testguard probe --allow-empty` to skip a valid
|
|
447
|
+
empty claims file without blocking a push. It explicitly reports that no
|
|
448
|
+
verification was performed and writes no evidence; `--json` retains the
|
|
449
|
+
`no-claims` state. Missing or invalid claims files still fail. Run `gate`
|
|
450
|
+
separately to enforce changed-file coverage. For the pre-commit probe hook,
|
|
451
|
+
set `args: [--allow-empty]`; for the GitHub Action, set `allow-empty: 'true'`.
|
|
452
|
+
The default probe still exits 2 when no claims exist.
|
|
453
|
+
|
|
454
|
+
In a monorepo, a valid nested `testguard.claims.json` marks a separate project.
|
|
455
|
+
The parent gate reports its changed files as delegated and does not count them
|
|
456
|
+
as parent coverage. Run `gate` in each child project as well; a passing parent
|
|
457
|
+
gate does not establish that child coverage passed. Invalid child claims fail
|
|
458
|
+
evaluation instead of hiding files.
|
|
459
|
+
|
|
410
460
|
`gate` names the changed files that carry no claim, and stops there —
|
|
411
461
|
correctly, because stating a claim is a human act. But a project adopting this
|
|
412
462
|
tool reads that list, has nothing to compare it against, and closes the tab.
|
|
@@ -663,12 +713,12 @@ detected base that does not resolve is a warning for `status` and `brief`
|
|
|
663
713
|
# GitHub Actions
|
|
664
714
|
- uses: actions/checkout@v4
|
|
665
715
|
with: { fetch-depth: 0 }
|
|
666
|
-
- uses: raccioly/testguard@v0.
|
|
716
|
+
- uses: raccioly/testguard@v0.16.0
|
|
667
717
|
with: { command: gate }
|
|
668
718
|
|
|
669
719
|
# GitLab CI — the component-shaped template: gate + probe, brief as an artifact and, opted in, as a merge-request note
|
|
670
720
|
include:
|
|
671
|
-
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.
|
|
721
|
+
- remote: 'https://raw.githubusercontent.com/raccioly/testguard/v0.16.0/packaging/gitlab/testguard.gitlab-ci.yml'
|
|
672
722
|
inputs: { dir: backend, post_note: true } # post_note needs TESTGUARD_GITLAB_TOKEN (api scope); one note, updated in place
|
|
673
723
|
```
|
|
674
724
|
|
|
@@ -941,19 +991,23 @@ most expensive claims
|
|
|
941
991
|
4m 58s TG-FAULT-EDIT-VISIBLE 1 fault · 6 runs
|
|
942
992
|
|
|
943
993
|
defender files, by the time of the runs that included them
|
|
944
|
-
(a
|
|
994
|
+
(a fault runs its selected defender set at once, so these overlap and do not sum to the total)
|
|
945
995
|
24m 3s test/probe.fixture.test.mjs 5 claims
|
|
946
996
|
|
|
947
|
-
shared defenders — each
|
|
997
|
+
shared defenders — each fault selects the file; durations include recorded baseline sharing
|
|
948
998
|
24m 3s test/probe.fixture.test.mjs
|
|
949
|
-
named by TG-ESCALATION-N-RUNS, TG-FAULT-EDIT-VISIBLE, …
|
|
999
|
+
named by TG-ESCALATION-N-RUNS/F1, TG-FAULT-EDIT-VISIBLE/F1, …
|
|
950
1000
|
```
|
|
951
1001
|
|
|
952
1002
|
Every number is read back out of the `durationMs` the probe already records.
|
|
953
1003
|
`--cost` re-measures nothing, spawns nothing, and costs the price of reading
|
|
954
1004
|
one JSON file.
|
|
955
1005
|
|
|
956
|
-
|
|
1006
|
+
Cost output also lists individual faults, their selected defenders and selection
|
|
1007
|
+
origin. Sharing is counted across distinct fault identities, including siblings
|
|
1008
|
+
within one claim; repeated confirmation runs alone are not sharing.
|
|
1009
|
+
|
|
1010
|
+
**The per-file figures overlap.** A run executes a fault's whole defender set
|
|
957
1011
|
at once, so the runner never says how much of it belonged to which file. A
|
|
958
1012
|
file's `ms` is the time of every run that *included* it: an upper bound on
|
|
959
1013
|
what removing it could save, which is the decision you are making. Only
|
|
@@ -35,6 +35,10 @@ inputs:
|
|
|
35
35
|
description: 'Skip re-running survivors against the whole suite'
|
|
36
36
|
required: false
|
|
37
37
|
default: 'false'
|
|
38
|
+
allow-empty:
|
|
39
|
+
description: 'probe only: explicitly skip a valid empty claims file during adoption; writes no evidence. Run gate separately.'
|
|
40
|
+
required: false
|
|
41
|
+
default: 'false'
|
|
38
42
|
changed-ref:
|
|
39
43
|
description: 'gate only: the reference the change is measured against (default: origin/<base branch> of the pull request). Check out with fetch-depth: 0 so it resolves.'
|
|
40
44
|
required: false
|
|
@@ -46,7 +50,7 @@ inputs:
|
|
|
46
50
|
version:
|
|
47
51
|
description: 'testguard-cli version to run'
|
|
48
52
|
required: false
|
|
49
|
-
default: '0.
|
|
53
|
+
default: '0.16.0'
|
|
50
54
|
node-version:
|
|
51
55
|
description: 'Node.js version (20+)'
|
|
52
56
|
required: false
|
|
@@ -82,6 +86,7 @@ runs:
|
|
|
82
86
|
if [ "${{ inputs.command }}" = "probe" ]; then
|
|
83
87
|
ARGS+=(--severity "${{ inputs.severity }}" --confirm "${{ inputs.confirm }}" --budget "${{ inputs.budget }}")
|
|
84
88
|
[ "${{ inputs.no-escalate }}" = "true" ] && ARGS+=(--no-escalate)
|
|
89
|
+
[ "${{ inputs.allow-empty }}" = "true" ] && ARGS+=(--allow-empty)
|
|
85
90
|
[ "${{ inputs.runner }}" != "auto" ] && ARGS+=(--runner "${{ inputs.runner }}")
|
|
86
91
|
[ -n "${{ inputs.python }}" ] && ARGS+=(--python "${{ inputs.python }}")
|
|
87
92
|
fi
|
|
@@ -144,7 +144,7 @@
|
|
|
144
144
|
<div class="page">
|
|
145
145
|
<div class="masthead">
|
|
146
146
|
<div class="wordmark">Test<span class="mark">Guard</span></div>
|
|
147
|
-
<div class="meta">Technical Brief · v0.
|
|
147
|
+
<div class="meta">Technical Brief · v0.16.0 · MIT</div>
|
|
148
148
|
</div>
|
|
149
149
|
|
|
150
150
|
<h1>Your tests pass. That is not the same as your tests working.</h1>
|
|
Binary file
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Data Model
|
|
2
2
|
|
|
3
|
-
<!-- docguard:version 1.
|
|
3
|
+
<!-- docguard:version 1.1.0 -->
|
|
4
4
|
<!-- docguard:status approved -->
|
|
5
|
-
<!-- docguard:last-reviewed 2026-09-
|
|
5
|
+
<!-- docguard:last-reviewed 2026-09-30 -->
|
|
6
6
|
<!-- docguard:owner @raccioly -->
|
|
7
7
|
<!-- docguard:quality negation-load off — the data model is largely a set of prohibitions (no database, no server state, fingerprints never derived from fault text). -->
|
|
8
8
|
<!-- docguard:quality passive-voice off — this document describes what happens to documents as they flow through validation, where the artifact is the subject and the actor is irrelevant. -->
|
|
@@ -60,12 +60,23 @@ be told from a mechanically derived one.
|
|
|
60
60
|
`{find, replace}` against one file, with `expectHits` and `occurrence` so the
|
|
61
61
|
anchor is exact. `faultClass` is one of thirteen values and exists so that
|
|
62
62
|
calibration (fault class → real-bug predictiveness) can be computed later.
|
|
63
|
+
An optional fault-level `defendedBy` overrides the claim's declaration.
|
|
64
|
+
Presence matters: `[]` requests discovery, while absence inherits a non-empty
|
|
65
|
+
claim list or discovers when the claim has none.
|
|
63
66
|
|
|
64
67
|
### Evidence record
|
|
65
68
|
|
|
66
69
|
The verdict plus everything it rests on: every baseline run, every probe run,
|
|
67
70
|
the resolved defenders, which of them merely mock the subject, the input
|
|
68
71
|
hashes that make verdict reuse sound, and the rank.
|
|
72
|
+
New records carry `defenders.selectionSource` (`fault`, `claim`, `discovery`)
|
|
73
|
+
alongside requested and resolved selections. Empty selections carry
|
|
74
|
+
`discovered: true`. Origin or selection changes invalidate reuse and status;
|
|
75
|
+
the field remains optional so new readers can validate legacy evidence.
|
|
76
|
+
|
|
77
|
+
Status cost reports include per-fault recorded durations and the selected
|
|
78
|
+
defenders. File-level entries list distinct `claimId/faultId` identities;
|
|
79
|
+
multiple confirmation runs for one fault do not constitute sharing.
|
|
69
80
|
|
|
70
81
|
### Verdict
|
|
71
82
|
|
|
@@ -78,7 +89,7 @@ Closed set of seven. Only `killed` is a pass:
|
|
|
78
89
|
```
|
|
79
90
|
claims.json ──1:N──> claim ──1:N──> fault ──1:1──> evidence record
|
|
80
91
|
│ │
|
|
81
|
-
└── defendedBy
|
|
92
|
+
└── default defendedBy ─> fault selection ─> test files
|
|
82
93
|
│
|
|
83
94
|
baseline.json ──suppresses by fingerprint────────────┘
|
|
84
95
|
│
|
|
@@ -107,9 +118,11 @@ target file to importing tests for defender discovery.
|
|
|
107
118
|
The contract is versioned by `schemaVersion` inside every document and by the
|
|
108
119
|
`urn:claimspec:v1:` identifier on every schema.
|
|
109
120
|
|
|
110
|
-
- **Additive change** (a new optional field):
|
|
111
|
-
|
|
112
|
-
|
|
121
|
+
- **Additive change** (a new optional field): new readers accept old documents
|
|
122
|
+
without migration. Older strict readers may reject the new field because
|
|
123
|
+
schemas use `additionalProperties: false`; upgrade those consumers before
|
|
124
|
+
asking them to validate new-format artifacts. Add the field to the schema
|
|
125
|
+
in the same change that emits it.
|
|
113
126
|
- **Semantic change** (a new verdict, a new fault class, a changed rule): a
|
|
114
127
|
spec change. The schema, `GATE-SEMANTICS.md`, the validator's semantic rules,
|
|
115
128
|
a conformance example, a must-reject document and the known-answer fixture
|
|
@@ -125,4 +138,5 @@ The contract is versioned by `schemaVersion` inside every document and by the
|
|
|
125
138
|
|
|
126
139
|
| Version | Date | Change | Author |
|
|
127
140
|
|---|---|---|---|
|
|
141
|
+
| 1.1.0 | 2026-09-30 | Fault defender overrides, selection provenance and per-fault costs; clarify strict-reader compatibility | @raccioly |
|
|
128
142
|
| 1.0.0 | 2026-09-18 | First canonical data model, written against v0.6.0 | @raccioly |
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Requirements
|
|
2
2
|
|
|
3
|
-
<!-- docguard:version 1.
|
|
3
|
+
<!-- docguard:version 1.2.0 -->
|
|
4
4
|
<!-- docguard:status approved -->
|
|
5
|
-
<!-- docguard:last-reviewed 2026-09-
|
|
5
|
+
<!-- docguard:last-reviewed 2026-09-30 -->
|
|
6
6
|
<!-- docguard:owner @raccioly -->
|
|
7
7
|
<!-- docguard:quality passive-voice off — requirements are written in the standard "the system shall ..." register, where the agent is the system throughout. -->
|
|
8
8
|
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
| FR-14 | Serve the read-only operating loop over MCP so the loop survives a change of agent harness | P3 |
|
|
29
29
|
| FR-15 | Validate every exact fault anchor and the syntax of supported in-memory replacements before starting a test runner (`claims --check-anchors`), without guessing repairs | P1 |
|
|
30
30
|
| FR-16 | Preserve every repeated or comma-separated `probe --claim` selection in first-seen order, collapse duplicates, and report requested versus probed scope without allowing an empty selection to pass | P1 |
|
|
31
|
+
| FR-17 | Select defenders per fault: an explicit override takes precedence, an empty override requests discovery, and absence inherits the claim. Record the origin, invalidate stale/reused selections, report per-fault costs and warn on narrowing after a prior kill | P1 |
|
|
31
32
|
|
|
32
33
|
## Non-Functional Requirements
|
|
33
34
|
|
|
@@ -103,6 +104,7 @@ fails the pull request naming the unclaimed file, and `status` reports
|
|
|
103
104
|
| FR-14 | `src/mcp/` | `test/mcp.test.mjs` |
|
|
104
105
|
| FR-15 | `src/claims/anchors.mjs`, `src/commands/claims.mjs`, `src/status/status.mjs` | `test/claims-anchors.test.mjs`, `test/status.test.mjs`, `TG-ANCHOR-PREFLIGHT-FAILS-FAST` |
|
|
105
106
|
| FR-16 | `src/cli.mjs`, `src/commands/probe.mjs`, `src/probe/probe.mjs`, `src/commands/scaffold.mjs`, `src/commands/admit.mjs` | `test/claim-selection.test.mjs`, `test/probe.fixture.test.mjs`, `TG-REPEATED-CLAIMS-ARE-ALL-PROBED` |
|
|
107
|
+
| FR-17 | `src/probe/attribution.mjs`, `src/probe/probe.mjs`, `src/commands/claims.mjs`, `src/status/status.mjs`, `src/probe/cost.mjs` | `test/fault-defenders.test.mjs`, `test/attribution.test.mjs`, `test/status.test.mjs`, `test/cost.test.mjs`, `TG-FAULT-DEFENDER-SELECTION`, `TG-FAULT-DEFENDER-ORCHESTRATION` |
|
|
106
108
|
| NFR-01 | `src/init/init.mjs`, runner resolution | `TG-INIT-HOOK-NO-NETWORK`, `TG-README-HOOK-MATCHES-THE-CODE` (every surface, not only the README), `TG-GITIGNORE-ADVICE-IS-ONE-LIST` |
|
|
107
109
|
| NFR-02 | `package.json` | `npm run test:install` in CI |
|
|
108
110
|
| NFR-03 | `src/probe/inject.mjs`, `src/probe/probe.mjs` | `TG-DIRTY-DEFENDERS-REFUSED`, `TG-IGNORED-DIRTY-RECORDED`, `TG-IN-PLACE-STILL-RAISES-A-MISSING-TARGET`, `TG-CONTROL-RESTORES-THE-SUBJECT` |
|
|
@@ -116,5 +118,6 @@ fails the pull request naming the unclaimed file, and `status` reports
|
|
|
116
118
|
|
|
117
119
|
| Version | Date | Change | Author |
|
|
118
120
|
|---|---|---|---|
|
|
121
|
+
| 1.2.0 | 2026-09-30 | Approved fault-specific defender selection and attributable costs | @raccioly |
|
|
119
122
|
| 1.1.0 | 2026-09-19 | Require lossless, visible partial claim selection | @raccioly |
|
|
120
123
|
| 1.0.0 | 2026-09-18 | First canonical requirements, written against v0.6.0 | @raccioly |
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Test Specification
|
|
2
2
|
|
|
3
|
-
<!-- docguard:version 1.
|
|
3
|
+
<!-- docguard:version 1.1.0 -->
|
|
4
4
|
<!-- docguard:status approved -->
|
|
5
|
-
<!-- docguard:last-reviewed 2026-09-
|
|
5
|
+
<!-- docguard:last-reviewed 2026-09-30 -->
|
|
6
6
|
<!-- docguard:owner @raccioly -->
|
|
7
7
|
<!-- docguard:quality negation-load off — the governing rule is a prohibition (a test that would pass with the behaviour broken is not a test), and the verdict rules are defined by what does not count as detection. -->
|
|
8
8
|
|
|
@@ -21,7 +21,7 @@ the test that defends it, and CI fails if any injected fault survives.
|
|
|
21
21
|
| Conformance | `spec/conformance/` | Every valid example validates; every must-reject document is rejected for the defect its filename names | ~1 s |
|
|
22
22
|
| Anchor preflight | `test/claims-anchors.test.mjs` | Exact anchors still locate, supported replacements parse, the check stays below one second, and CI reaches it before self-probe | <1 s |
|
|
23
23
|
| Fixture acceptance | `test/probe.fixture.test.mjs`, `test/probe.jest.test.mjs`, `test/probe.playwright.test.mjs` | The whole pipeline reproduces a known-answer oracle, one per runner. The Playwright fixture carries its own dependencies: without `npm ci --prefix fixtures/known-answer-playwright` that file skips, and a skipped acceptance is not a passing one | 10–50 s each |
|
|
24
|
-
| Self-verification | `testguard.claims.json`, run by `npm run self:probe` | The tool's own invariants survive fault injection | **20.2 min**
|
|
24
|
+
| Self-verification | `testguard.claims.json`, run by `npm run self:probe` | The tool's own invariants survive fault injection | Historical cold measurement: **20.2 min** in the v0.10.1 release job on 2026-09-19 at `70e9d61` (154 faults, 924 defender runs, 1210 s). PR CI restores previous evidence for reuse. Current committed ceilings in `testguard.cost-budget.json`: 2250 s total and 100 s per claim; both gate CI, and neither changes for fault-specific selection |
|
|
25
25
|
| Install smoke | `.github/scripts/install-smoke.mjs` | The **packed tarball** runs with production dependencies only | ~20 s |
|
|
26
26
|
| Runtime budget | `ci.yml` | The suite has not silently started walking the wrong tree | gate, not a test |
|
|
27
27
|
|
|
@@ -31,7 +31,7 @@ the test that defends it, and CI fails if any injected fault survives.
|
|
|
31
31
|
`durationMs` already on every run in an evidence document. They re-measure
|
|
32
32
|
nothing.
|
|
33
33
|
|
|
34
|
-
A probe's wall
|
|
34
|
+
A probe's historical wall-clock model, before fault-specific selection, was per claim:
|
|
35
35
|
`(baseline runs + faults × --confirm) × the cost of that claim's defender
|
|
36
36
|
SET`. One slow acceptance test named by several claims is therefore paid for
|
|
37
37
|
once per claim, which is what made the gate take 24 minutes: a single 49.5 s
|
|
@@ -39,8 +39,15 @@ fixture test defended five claims. Those claims now point at pure functions
|
|
|
39
39
|
with unit-test defenders (issue #67), and the gate measures **9.6 minutes**
|
|
40
40
|
from scratch, serially, for 68 faults.
|
|
41
41
|
|
|
42
|
+
Fault-specific selection runs each fault's selected set. Baselines are cached
|
|
43
|
+
by that set, not claim identity; per-fault cost records expose which files
|
|
44
|
+
actually ran. Repeated confirmations alone are not shared-defender usage.
|
|
45
|
+
`test/fault-defenders.test.mjs` covers inheritance, overrides, explicit-empty
|
|
46
|
+
discovery, origin-sensitive reuse, dirty-input refusal, runner ownership,
|
|
47
|
+
narrowing warnings and admission through the actual selected set.
|
|
48
|
+
|
|
42
49
|
Per-file cost figures are an **upper bound, not a share**: a run executes a
|
|
43
|
-
|
|
50
|
+
fault's whole selected defender set at once, so the runner never attributes time to
|
|
44
51
|
one file. They overlap and do not sum to the total.
|
|
45
52
|
|
|
46
53
|
## Coverage Rules
|
|
@@ -69,7 +76,7 @@ What is gated instead:
|
|
|
69
76
|
| `src/probe/classify.mjs` | `test/classify.test.mjs`, `test/negative-control.test.mjs` |
|
|
70
77
|
| `src/probe/inject.mjs` | `test/inject.test.mjs` |
|
|
71
78
|
| `src/claims/anchors.mjs` | `test/claims-anchors.test.mjs` |
|
|
72
|
-
| `src/probe/probe.mjs` | `test/probe.fixture.test.mjs`, `test/probe-preconditions.test.mjs`, `test/probe-error.test.mjs` |
|
|
79
|
+
| `src/probe/probe.mjs` | `test/probe.fixture.test.mjs`, `test/probe-preconditions.test.mjs`, `test/probe-error.test.mjs`, `test/fault-defenders.test.mjs` |
|
|
73
80
|
| `src/probe/discover.mjs`, `src/probe/mocks.mjs` | `test/discover.test.mjs`, `test/mocks.test.mjs` |
|
|
74
81
|
| `src/probe/rank.mjs` | `test/rank-aliases.test.mjs`, `test/discover.test.mjs` |
|
|
75
82
|
| `src/probe/runners/` | `test/runner-vitest.test.mjs`, `test/runner-command.test.mjs`, `test/runner-playwright.test.mjs` |
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "testguard-cli",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.16.0",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"ajv": "8.20.0"
|