testguard-cli 0.1.0__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/dependabot.yml +1 -2
- testguard_cli-0.1.2/.github/scripts/install-smoke.mjs +39 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/scripts/sync-release-version.mjs +1 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/workflows/auto-merge.yml +1 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/workflows/ci.yml +8 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/workflows/release.yml +2 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/workflows/scheduled-release.yml +1 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/workflows/supply-chain.yml +2 -2
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.npmrc +2 -2
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/AGENTS.md +3 -1
- testguard_cli-0.1.2/CHANGELOG.md +104 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/CONTRIBUTING.md +5 -3
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/PKG-INFO +26 -7
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/README.md +25 -6
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/SECURITY.md +3 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/action.yml +1 -1
- testguard_cli-0.1.2/cli/testguard.mjs +14 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/package-lock.json +8 -8
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/package.json +6 -3
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/packaging/homebrew/testguard.rb +3 -3
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/pyproject.toml +1 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/GATE-SEMANTICS.md +2 -2
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/common.schema.json +1 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/cli.mjs +7 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/commands/baseline.mjs +3 -1
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/commands/claims.mjs +3 -2
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/commands/probe.mjs +8 -3
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/probe/probe.mjs +41 -10
- testguard_cli-0.1.2/src/probe/rank.mjs +130 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/probe/runner-vitest.mjs +61 -2
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/probe/worktree.mjs +28 -11
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/render.mjs +3 -2
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/brief.test.mjs +11 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/classify.test.mjs +3 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/probe.fixture.test.mjs +35 -0
- testguard_cli-0.1.2/test/rank-aliases.test.mjs +35 -0
- testguard_cli-0.1.2/test/runner-command.test.mjs +34 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/worktree.test.mjs +28 -1
- testguard_cli-0.1.0/CHANGELOG.md +0 -39
- testguard_cli-0.1.0/cli/testguard.mjs +0 -6
- testguard_cli-0.1.0/src/probe/rank.mjs +0 -51
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.gitattributes +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.gitignore +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.npmignore +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/.pre-commit-hooks.yaml +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/CLAUDE.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/LICENSE +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/PRIVACY.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/SUPPORT.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/bench/README.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/README.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/expected.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/testguard.claims.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/README.md +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/examples/baseline.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/examples/evidence.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/conformance/schemas.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/lib/validate.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/baseline.schema.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/brief.schema.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/claims.schema.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/evidence.schema.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/baseline/baseline.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/brief/brief.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/claims/load.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/commands/brief.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/git.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/probe/classify.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/util/glob.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/src/util/hash.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/glob.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/inject.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/runner-vitest.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/test/writer.test.mjs +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/testguard.claims.json +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.1.0 → testguard_cli-0.1.2}/vitest.config.mjs +0 -0
|
@@ -7,8 +7,7 @@ updates:
|
|
|
7
7
|
groups:
|
|
8
8
|
all-npm:
|
|
9
9
|
patterns: ["*"]
|
|
10
|
-
#
|
|
11
|
-
# runner). The published package has zero runtime dependencies.
|
|
10
|
+
# One runtime dependency (ajv, exact-pinned) plus vitest for tests.
|
|
12
11
|
- package-ecosystem: "pip"
|
|
13
12
|
directory: "/"
|
|
14
13
|
schedule:
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Prove the package works AS INSTALLED, not as checked out: pack it, install
|
|
4
|
+
* the tarball into a scratch project with production dependencies only, and
|
|
5
|
+
* run the CLI from there. This is the gate that would have caught 0.1.0,
|
|
6
|
+
* which shipped with its schema validator's dependency declared as a
|
|
7
|
+
* devDependency and crashed on startup for every npm/npx/pip/brew user.
|
|
8
|
+
*/
|
|
9
|
+
import { execFileSync } from 'node:child_process';
|
|
10
|
+
import { mkdtempSync, mkdirSync, rmSync, cpSync, readdirSync } from 'node:fs';
|
|
11
|
+
import { tmpdir } from 'node:os';
|
|
12
|
+
import { join, resolve } from 'node:path';
|
|
13
|
+
|
|
14
|
+
const root = resolve(process.argv[1], '..', '..', '..');
|
|
15
|
+
const run = (cmd, args, cwd) => execFileSync(cmd, args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'inherit'] });
|
|
16
|
+
const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
|
|
17
|
+
|
|
18
|
+
const scratch = mkdtempSync(join(tmpdir(), 'testguard-install-smoke-'));
|
|
19
|
+
try {
|
|
20
|
+
const packDir = join(scratch, 'pack');
|
|
21
|
+
mkdirSync(packDir);
|
|
22
|
+
run(npm, ['pack', '--pack-destination', packDir, '--ignore-scripts'], root);
|
|
23
|
+
const tgz = join(packDir, readdirSync(packDir).find((f) => f.endsWith('.tgz')));
|
|
24
|
+
|
|
25
|
+
const consumer = join(scratch, 'consumer');
|
|
26
|
+
cpSync(join(root, 'fixtures', 'known-answer'), consumer, { recursive: true, filter: (s) => !/node_modules|\.flake-counter|\.testguard/.test(s) });
|
|
27
|
+
run(npm, ['init', '-y'], consumer);
|
|
28
|
+
run(npm, ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tgz], consumer);
|
|
29
|
+
|
|
30
|
+
const bin = join(consumer, 'node_modules', '.bin', process.platform === 'win32' ? 'testguard.cmd' : 'testguard');
|
|
31
|
+
const version = run(bin, ['--version'], consumer).trim();
|
|
32
|
+
const claims = run(bin, ['claims', '.'], consumer);
|
|
33
|
+
if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
|
|
34
|
+
if (!claims.includes('8 claims')) throw new Error(`claims did not list the fixture:\n${claims}`);
|
|
35
|
+
const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
|
|
36
|
+
console.log(`install smoke OK — testguard ${version} runs from the installed tarball; consumer deps: ${deps.join(', ')}`);
|
|
37
|
+
} finally {
|
|
38
|
+
rmSync(scratch, { recursive: true, force: true });
|
|
39
|
+
}
|
|
@@ -20,6 +20,7 @@ const surfaces = [
|
|
|
20
20
|
['pyproject.toml', /^version = "[^"]+"/m, `version = "${version}"`],
|
|
21
21
|
['action.yml', /(\n version:\n description: [^\n]*\n required: false\n default: ')[^']*(')/, `$1${version}$2`],
|
|
22
22
|
['packaging/homebrew/testguard.rb', /testguard-cli-\d+\.\d+\.\d+\.tgz/g, `testguard-cli-${version}.tgz`],
|
|
23
|
+
['README.md', /raccioly\/testguard@v\d+\.\d+\.\d+/g, `raccioly/testguard@v${version}`],
|
|
23
24
|
];
|
|
24
25
|
|
|
25
26
|
let drift = 0;
|
|
@@ -46,7 +46,7 @@ jobs:
|
|
|
46
46
|
|
|
47
47
|
const { data: files } = await github.rest.pulls.listFiles({ owner, repo, pull_number: num, per_page: 300 });
|
|
48
48
|
const paths = files.map((f) => f.filename);
|
|
49
|
-
const RELEASE_SURFACES = new Set(['package.json', 'package-lock.json', 'pyproject.toml', 'CHANGELOG.md', 'action.yml', 'packaging/homebrew/testguard.rb']);
|
|
49
|
+
const RELEASE_SURFACES = new Set(['package.json', 'package-lock.json', 'pyproject.toml', 'CHANGELOG.md', 'action.yml', 'packaging/homebrew/testguard.rb', 'README.md']);
|
|
50
50
|
|
|
51
51
|
if (author === 'dependabot[bot]') {
|
|
52
52
|
const m = /from (\d+)\.(\d+)\.\S+ to (\d+)\.(\d+)\./.exec(pr.title || '');
|
|
@@ -30,7 +30,7 @@ jobs:
|
|
|
30
30
|
with:
|
|
31
31
|
node-version: ${{ matrix.node-version }}
|
|
32
32
|
|
|
33
|
-
- name: Install
|
|
33
|
+
- name: Install
|
|
34
34
|
run: npm ci
|
|
35
35
|
|
|
36
36
|
# ── Syntax-check EVERY source, not just what the entry point eagerly
|
|
@@ -38,6 +38,13 @@ jobs:
|
|
|
38
38
|
- name: Syntax-check all sources
|
|
39
39
|
run: find cli src spec -name '*.mjs' -print0 | xargs -0 -n1 node --check
|
|
40
40
|
|
|
41
|
+
# ── Install-from-tarball smoke: the package must run as INSTALLED, with
|
|
42
|
+
# production dependencies only. 0.1.0 shipped unrunnable because the
|
|
43
|
+
# schema validator's dependency was a devDependency; the suite, which
|
|
44
|
+
# runs from the checkout, could not see it. ──
|
|
45
|
+
- name: Install smoke (pack → install --omit=dev → run)
|
|
46
|
+
run: npm run test:install
|
|
47
|
+
|
|
41
48
|
- name: Python wrapper imports (PyPI surface)
|
|
42
49
|
run: python3 -c "import testguard_cli.wrapper; print('testguard_cli.wrapper OK')"
|
|
43
50
|
|
|
@@ -77,6 +77,8 @@ jobs:
|
|
|
77
77
|
node-version: 24
|
|
78
78
|
- run: npm ci
|
|
79
79
|
- run: npm test
|
|
80
|
+
- name: Install smoke — the tarball must run with production deps only
|
|
81
|
+
run: npm run test:install
|
|
80
82
|
- name: Self-probe must be fully defended before anything ships
|
|
81
83
|
run: node cli/testguard.mjs probe . --budget 180000 --out "$RUNNER_TEMP/self-evidence.json" --quiet
|
|
82
84
|
- name: Version surfaces are in sync
|
|
@@ -102,7 +102,7 @@ jobs:
|
|
|
102
102
|
# Only release surfaces may change in a release PR.
|
|
103
103
|
CHANGED=$(git diff --name-only HEAD --)
|
|
104
104
|
for f in $CHANGED; do
|
|
105
|
-
case "$f" in package.json|package-lock.json|pyproject.toml|CHANGELOG.md|action.yml|packaging/homebrew/testguard.rb) ;;
|
|
105
|
+
case "$f" in package.json|package-lock.json|pyproject.toml|CHANGELOG.md|action.yml|packaging/homebrew/testguard.rb|README.md) ;;
|
|
106
106
|
*) echo "::error::unexpected file in release diff: $f"; exit 1;;
|
|
107
107
|
esac
|
|
108
108
|
done
|
|
@@ -17,8 +17,8 @@ permissions:
|
|
|
17
17
|
jobs:
|
|
18
18
|
osv-scan-pr:
|
|
19
19
|
if: github.event_name == 'pull_request'
|
|
20
|
-
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@
|
|
20
|
+
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable-pr.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc" # v2.6.0
|
|
21
21
|
|
|
22
22
|
osv-scan-scheduled:
|
|
23
23
|
if: github.event_name != 'pull_request'
|
|
24
|
-
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@
|
|
24
|
+
uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@a345acffa64b0eaede81a3d9aae6141214d9c8fc" # v2.6.0
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Supply-chain hardening.
|
|
2
2
|
# ignore-scripts: never run a dependency's install/postinstall scripts — the
|
|
3
|
-
# main supply-chain attack vector. TestGuard
|
|
4
|
-
# its
|
|
3
|
+
# main supply-chain attack vector. TestGuard's one runtime dependency (ajv)
|
|
4
|
+
# and its devDependency (vitest) work without install scripts.
|
|
5
5
|
# audit-level: surface moderate-and-above advisories.
|
|
6
6
|
ignore-scripts=true
|
|
7
7
|
audit-level=moderate
|
|
@@ -6,7 +6,8 @@ Read this before changing anything. It is short on purpose.
|
|
|
6
6
|
|
|
7
7
|
A CLI that proves a test suite defends the claims a project makes, by
|
|
8
8
|
injecting the faults those claims forbid and reporting every one the tests
|
|
9
|
-
miss. **Not a test generator.** Node ≥ 20, ESM,
|
|
9
|
+
miss. **Not a test generator.** Node ≥ 20, ESM, one exact-pinned runtime
|
|
10
|
+
dependency (`ajv`).
|
|
10
11
|
The shared formats under `spec/` are a contract other tools adopt; the CLI is
|
|
11
12
|
their first consumer.
|
|
12
13
|
|
|
@@ -16,6 +17,7 @@ their first consumer.
|
|
|
16
17
|
npm test # unit + fixture acceptance (~15s)
|
|
17
18
|
npm run test:spec # conformance suite only
|
|
18
19
|
npm run self:probe # TestGuard probes its own claims; must exit 0
|
|
20
|
+
npm run test:install # the packed tarball must run with production deps only
|
|
19
21
|
node cli/testguard.mjs claims fixtures/known-answer
|
|
20
22
|
```
|
|
21
23
|
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.2] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
From a field report on a real codebase (63 test files, 458 tests, 39 faults).
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- **A runner that cannot be resolved was reported as `FLAKY-DEFENDER`.** In
|
|
17
|
+
the scratch worktree, a symlinked `node_modules` (the sibling/auto-worktree
|
|
18
|
+
layout) was invisible, vitest failed to load, and the load error was
|
|
19
|
+
classified as flaky tests — blaming the wrong party. Symlinked
|
|
20
|
+
`node_modules` are now linked to their resolved target; the runner is
|
|
21
|
+
checked before any verdict and an unresolvable one is a precondition
|
|
22
|
+
failure (exit 2) with the fix in the message; defenders that fail to
|
|
23
|
+
*load* during a baseline are `UNVERIFIABLE` (`defenders-failed-to-load`),
|
|
24
|
+
never flaky.
|
|
25
|
+
- **Blast radius ignored path aliases.** `tsconfig`/`jsconfig` `paths` (with
|
|
26
|
+
`baseUrl` and relative `extends`) and `package.json#imports` are resolved,
|
|
27
|
+
so a module imported 83 times via `@/…` no longer ranks as if nothing
|
|
28
|
+
imported it. Bare package specifiers remain ignored, as documented.
|
|
29
|
+
- Summary said "N unproven claims" when N counted faults; it now reports
|
|
30
|
+
unproven faults *and* the distinct claims they belong to.
|
|
31
|
+
- `testguard claims` never showed which claims carry a `@claim` annotation;
|
|
32
|
+
it now reports the count and marks each annotated row.
|
|
33
|
+
- Piping output to a closed reader (`testguard claims | head`) no longer
|
|
34
|
+
crashes with an `EPIPE` stack trace.
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
|
|
38
|
+
- `--claim <ID,ID>` probes only the named claims and writes
|
|
39
|
+
`.testguard/evidence-partial.json`, keeping the canonical evidence intact —
|
|
40
|
+
turns a fix-loop iteration from minutes into seconds.
|
|
41
|
+
- `--runner-cmd "<cmd>"` with `{files}` and `{out}` placeholders for
|
|
42
|
+
monorepos, custom configs and other package managers.
|
|
43
|
+
- `--node-modules <dir>` (or `TESTGUARD_NODE_MODULES`) to link a specific
|
|
44
|
+
`node_modules` into the scratch worktree.
|
|
45
|
+
|
|
46
|
+
### Changed
|
|
47
|
+
|
|
48
|
+
- The `--in-place` precondition message says what it means: only fault
|
|
49
|
+
target files must be clean; test files may be dirty.
|
|
50
|
+
- `testguard baseline` prints the two `.gitignore` lines for the regenerated
|
|
51
|
+
files instead of leaving it to the README.
|
|
52
|
+
- With no baseline, the ranked block is not printed a second time under the
|
|
53
|
+
per-fault stream.
|
|
54
|
+
- `--quiet` is documented precisely.
|
|
55
|
+
- Staged Homebrew formula carries the sha256 of the published 0.1.1 tarball.
|
|
56
|
+
|
|
57
|
+
## [0.1.1] - 2026-09-17
|
|
58
|
+
|
|
59
|
+
### Fixed
|
|
60
|
+
|
|
61
|
+
- **0.1.0 did not run when installed.** `ajv`, which validates every document
|
|
62
|
+
against the spec, was declared as a devDependency, so `testguard` crashed on
|
|
63
|
+
startup from npm, `npx`, `pip` and Homebrew. It is now an exact-pinned
|
|
64
|
+
runtime dependency — the project's one dependency.
|
|
65
|
+
- CI and the release job now pack the tarball, install it into a scratch
|
|
66
|
+
project with production dependencies only, and run the CLI from there
|
|
67
|
+
(`npm run test:install`). The suite alone runs from the checkout and could
|
|
68
|
+
not see this class of defect.
|
|
69
|
+
|
|
70
|
+
### Changed
|
|
71
|
+
|
|
72
|
+
- Homebrew formula carries the sha256 of the published tarball; description
|
|
73
|
+
shortened to satisfy `brew audit --strict`.
|
|
74
|
+
|
|
75
|
+
## [0.1.0] - 2026-09-17
|
|
76
|
+
|
|
77
|
+
First release. A claim verifier, not a test generator.
|
|
78
|
+
|
|
79
|
+
### Added
|
|
80
|
+
|
|
81
|
+
- **Contract spine** (`spec/`): six shared JSON Schemas — claims, evidence,
|
|
82
|
+
baseline, ignore, calibration, brief — identified as `urn:guard-spec:v1:*`,
|
|
83
|
+
with a validator that enforces the semantic rules a schema cannot express,
|
|
84
|
+
a single fingerprint derivation, `GATE-SEMANTICS.md`, and a conformance
|
|
85
|
+
corpus (one valid document per kind, 22 must-reject documents).
|
|
86
|
+
- **`testguard claims`** — validates the claims file and reports drift
|
|
87
|
+
against `@claim <ID>` annotations in source.
|
|
88
|
+
- **`testguard probe`** — applies each fault in a scratch git worktree,
|
|
89
|
+
confirms the defenders green N times, runs them N times with the fault,
|
|
90
|
+
escalates survivors to the whole suite with N-run attribution, restores,
|
|
91
|
+
classifies into a closed verdict set (`killed`, `survived`, `nocover`,
|
|
92
|
+
`unverifiable`, `timeout`, `fault-invalid`, `flaky-defender`), ranks, and
|
|
93
|
+
writes evidence validated against the spec. `--ref` pins the probed commit.
|
|
94
|
+
Verdicts are reused when the target, defenders and N are unchanged.
|
|
95
|
+
- **`testguard baseline`** — freezes every non-passing fingerprint so only
|
|
96
|
+
new findings gate; severity floor via `--severity`.
|
|
97
|
+
- **`testguard brief`** — a ranked, capped `## TEST BLINDSPOT CONTEXT` block
|
|
98
|
+
for an agent's session start; `--text` is hook-safe.
|
|
99
|
+
- **Known-answer fixture** with a genuine `objectContaining` blind spot and a
|
|
100
|
+
case for every verdict; probed end to end in the test suite and in CI.
|
|
101
|
+
- **Self-verification**: `testguard.claims.json` states invariants of the
|
|
102
|
+
tool itself, probed by the tool in CI.
|
|
103
|
+
- Distribution: npm (`testguard-cli`), PyPI wrapper (`testguard-cli`),
|
|
104
|
+
GitHub Action, Homebrew formula (staged), pre-commit hooks.
|
|
@@ -8,9 +8,10 @@ follow from that.
|
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
git clone https://github.com/raccioly/testguard && cd testguard
|
|
11
|
-
npm install
|
|
11
|
+
npm install
|
|
12
12
|
npm test # ~15s: unit tests + probing the fixture end to end
|
|
13
13
|
npm run self:probe # TestGuard's own claims, probed by TestGuard
|
|
14
|
+
npm run test:install # pack → install with prod deps only → run; what 0.1.0 lacked
|
|
14
15
|
```
|
|
15
16
|
|
|
16
17
|
Node ≥ 20. Python ≥ 3.8 only if you touch `testguard_cli/`.
|
|
@@ -41,8 +42,9 @@ Node ≥ 20. Python ≥ 3.8 only if you touch `testguard_cli/`.
|
|
|
41
42
|
load failure, or an exit code count as detection.
|
|
42
43
|
5. **N-run confirmation everywhere** — baseline, probe, and escalation
|
|
43
44
|
attribution. A single run is never evidence.
|
|
44
|
-
6. **
|
|
45
|
-
discussion first.
|
|
45
|
+
6. **One runtime dependency** (`ajv`, exact-pinned). Adding another needs a
|
|
46
|
+
very good reason and a discussion first. `npm run test:install` proves
|
|
47
|
+
the package runs as installed with production dependencies only.
|
|
46
48
|
7. **No client-identifying material.** Fixtures and examples use neutral
|
|
47
49
|
domains. Benchmarks against real codebases live outside this repository
|
|
48
50
|
(`bench/README.md`).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -29,7 +29,7 @@ Description-Content-Type: text/markdown
|
|
|
29
29
|
[](https://pypi.org/project/testguard-cli/)
|
|
30
30
|
[](https://nodejs.org)
|
|
31
31
|
[](./LICENSE)
|
|
32
|
-
[](./package.json)
|
|
33
33
|
|
|
34
34
|
> Proves that a test suite actually defends the claims a project makes — by
|
|
35
35
|
> injecting the faults those claims say cannot happen, and reporting every
|
|
@@ -59,6 +59,15 @@ on known-broken code. The largest gap was a compliance-critical path with
|
|
|
59
59
|
100% coverage, where the one assertion that mattered used
|
|
60
60
|
`expect.objectContaining({...})` and omitted the field carrying the data.
|
|
61
61
|
|
|
62
|
+
A second, independent run on a different AI-authored codebase (63 test
|
|
63
|
+
files, 458 tests, 24 hand-written security claims, 39 faults): **21 of 39
|
|
64
|
+
faults survived a fully green suite — 9 of them critical.** Super-admin
|
|
65
|
+
gating, membership checks, cookie flags and the whole authorization callback
|
|
66
|
+
could be disabled without a single test noticing. One test file had
|
|
67
|
+
re-implemented the authorization logic *inside the test* and asserted
|
|
68
|
+
against the copy: fifteen green tests, zero detection. After wrapper-level
|
|
69
|
+
tests were written against the survivors, 39/39 were killed.
|
|
70
|
+
|
|
62
71
|
## Install
|
|
63
72
|
|
|
64
73
|
| How | Command |
|
|
@@ -67,7 +76,7 @@ on known-broken code. The largest gap was a compliance-critical path with
|
|
|
67
76
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
68
77
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
69
78
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
70
|
-
| GitHub Action | `uses: raccioly/testguard@v0.1.
|
|
79
|
+
| GitHub Action | `uses: raccioly/testguard@v0.1.2` — see [`action.yml`](./action.yml) |
|
|
71
80
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
72
81
|
|
|
73
82
|
## How it works
|
|
@@ -84,7 +93,7 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
84
93
|
deterministic source change that would make the statement false. Every
|
|
85
94
|
claim and every fault records who produced it. `testguard claims`
|
|
86
95
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
87
|
-
source.
|
|
96
|
+
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
88
97
|
2. **Probe** confirms the defenders are green N times unmodified, applies
|
|
89
98
|
each fault in a scratch git worktree (your tree is never touched), runs
|
|
90
99
|
the defenders N times, re-runs survivors against the whole suite with
|
|
@@ -101,8 +110,18 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
101
110
|
| `FLAKY-DEFENDER` | the defenders are not reliably green, or disagreed across runs |
|
|
102
111
|
|
|
103
112
|
Never a single score. Findings are ranked by severity, claim provenance
|
|
104
|
-
and blast radius
|
|
105
|
-
|
|
113
|
+
and blast radius (relative imports, `tsconfig` path aliases and
|
|
114
|
+
`package.json#imports` are resolved; bare package names are not), and
|
|
115
|
+
written to `.testguard/evidence.json` — validated against the spec before
|
|
116
|
+
it is written.
|
|
117
|
+
|
|
118
|
+
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
119
|
+
suite N times per survivor); iterate on one claim with
|
|
120
|
+
`--claim <ID> --in-place` — only the files faults are applied to must be
|
|
121
|
+
committed, test files may be dirty; final pass with defaults. A custom
|
|
122
|
+
runner (`pnpm --filter`, a specific config) goes in
|
|
123
|
+
`--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
|
|
124
|
+
see your `node_modules`, pass `--node-modules <dir>`.
|
|
106
125
|
3. **Baseline** freezes every non-passing fingerprint. Later probes suppress
|
|
107
126
|
what was already known and exit non-zero only on what is new. Claims whose
|
|
108
127
|
source and defenders are unchanged reuse their prior verdict, so a probe
|
|
@@ -147,7 +166,7 @@ through every verdict.
|
|
|
147
166
|
|
|
148
167
|
**v0.1.** Four commands, vitest runner, hand-authored faults. The contract
|
|
149
168
|
spine — six JSON Schemas shared with the other Guard tools — is under
|
|
150
|
-
[`spec/`](spec/).
|
|
169
|
+
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
151
170
|
|
|
152
171
|
Not yet: test generation (the two-gate acceptance loop), other runners,
|
|
153
172
|
mechanical fault producers, and calibration of fault classes against real
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://pypi.org/project/testguard-cli/)
|
|
6
6
|
[](https://nodejs.org)
|
|
7
7
|
[](./LICENSE)
|
|
8
|
-
[](./package.json)
|
|
9
9
|
|
|
10
10
|
> Proves that a test suite actually defends the claims a project makes — by
|
|
11
11
|
> injecting the faults those claims say cannot happen, and reporting every
|
|
@@ -35,6 +35,15 @@ on known-broken code. The largest gap was a compliance-critical path with
|
|
|
35
35
|
100% coverage, where the one assertion that mattered used
|
|
36
36
|
`expect.objectContaining({...})` and omitted the field carrying the data.
|
|
37
37
|
|
|
38
|
+
A second, independent run on a different AI-authored codebase (63 test
|
|
39
|
+
files, 458 tests, 24 hand-written security claims, 39 faults): **21 of 39
|
|
40
|
+
faults survived a fully green suite — 9 of them critical.** Super-admin
|
|
41
|
+
gating, membership checks, cookie flags and the whole authorization callback
|
|
42
|
+
could be disabled without a single test noticing. One test file had
|
|
43
|
+
re-implemented the authorization logic *inside the test* and asserted
|
|
44
|
+
against the copy: fifteen green tests, zero detection. After wrapper-level
|
|
45
|
+
tests were written against the survivors, 39/39 were killed.
|
|
46
|
+
|
|
38
47
|
## Install
|
|
39
48
|
|
|
40
49
|
| How | Command |
|
|
@@ -43,7 +52,7 @@ on known-broken code. The largest gap was a compliance-critical path with
|
|
|
43
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
44
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
45
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
46
|
-
| GitHub Action | `uses: raccioly/testguard@v0.1.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.1.2` — see [`action.yml`](./action.yml) |
|
|
47
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
48
57
|
|
|
49
58
|
## How it works
|
|
@@ -60,7 +69,7 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
60
69
|
deterministic source change that would make the statement false. Every
|
|
61
70
|
claim and every fault records who produced it. `testguard claims`
|
|
62
71
|
validates the file and reports drift against `@claim <ID>` annotations in
|
|
63
|
-
source.
|
|
72
|
+
source. Test files are deliberately not scanned — a claim asserted by a test is the authorship trap the tool exists for — and annotation ids must contain a hyphen so prose is never mistaken for one.
|
|
64
73
|
2. **Probe** confirms the defenders are green N times unmodified, applies
|
|
65
74
|
each fault in a scratch git worktree (your tree is never touched), runs
|
|
66
75
|
the defenders N times, re-runs survivors against the whole suite with
|
|
@@ -77,8 +86,18 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
77
86
|
| `FLAKY-DEFENDER` | the defenders are not reliably green, or disagreed across runs |
|
|
78
87
|
|
|
79
88
|
Never a single score. Findings are ranked by severity, claim provenance
|
|
80
|
-
and blast radius
|
|
81
|
-
|
|
89
|
+
and blast radius (relative imports, `tsconfig` path aliases and
|
|
90
|
+
`package.json#imports` are resolved; bare package names are not), and
|
|
91
|
+
written to `.testguard/evidence.json` — validated against the spec before
|
|
92
|
+
it is written.
|
|
93
|
+
|
|
94
|
+
Practical loop: first pass `--no-escalate` (escalation re-runs the whole
|
|
95
|
+
suite N times per survivor); iterate on one claim with
|
|
96
|
+
`--claim <ID> --in-place` — only the files faults are applied to must be
|
|
97
|
+
committed, test files may be dirty; final pass with defaults. A custom
|
|
98
|
+
runner (`pnpm --filter`, a specific config) goes in
|
|
99
|
+
`--runner-cmd "<cmd> {files} … {out}"`; if the scratch worktree cannot
|
|
100
|
+
see your `node_modules`, pass `--node-modules <dir>`.
|
|
82
101
|
3. **Baseline** freezes every non-passing fingerprint. Later probes suppress
|
|
83
102
|
what was already known and exit non-zero only on what is new. Claims whose
|
|
84
103
|
source and defenders are unchanged reuse their prior verdict, so a probe
|
|
@@ -123,7 +142,7 @@ through every verdict.
|
|
|
123
142
|
|
|
124
143
|
**v0.1.** Four commands, vitest runner, hand-authored faults. The contract
|
|
125
144
|
spine — six JSON Schemas shared with the other Guard tools — is under
|
|
126
|
-
[`spec/`](spec/).
|
|
145
|
+
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
127
146
|
|
|
128
147
|
Not yet: test generation (the two-gate acceptance loop), other runners,
|
|
129
148
|
mechanical fault producers, and calibration of fault classes against real
|
|
@@ -18,7 +18,9 @@ You will get an acknowledgement within 48 hours and a timeline for a fix.
|
|
|
18
18
|
|
|
19
19
|
TestGuard is a local CLI with a deliberately small surface:
|
|
20
20
|
|
|
21
|
-
- **
|
|
21
|
+
- **One runtime dependency, exact-pinned:** `ajv`, which validates every
|
|
22
|
+
document the tool reads or writes against the spec. Install scripts are
|
|
23
|
+
disabled (`.npmrc`). `vitest` is development-only.
|
|
22
24
|
- **No network access.** Nothing is uploaded, fetched, or reported anywhere.
|
|
23
25
|
- **No credentials.** There is nothing to authenticate to.
|
|
24
26
|
- **Writes are confined** to `.testguard/` in the probed project and to
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { main } from '../src/cli.mjs';
|
|
3
|
+
|
|
4
|
+
// `testguard … | head` closes stdout early; that is not an error worth a stack trace.
|
|
5
|
+
for (const stream of [process.stdout, process.stderr]) {
|
|
6
|
+
stream.on('error', (err) => {
|
|
7
|
+
if (err.code === 'EPIPE') process.exit(0);
|
|
8
|
+
throw err;
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
main(process.argv.slice(2)).then((code) => {
|
|
13
|
+
process.exitCode = code;
|
|
14
|
+
});
|
|
@@ -1,15 +1,20 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "testguard-cli",
|
|
9
|
-
"version": "0.1.
|
|
9
|
+
"version": "0.1.2",
|
|
10
10
|
"license": "MIT",
|
|
11
|
+
"dependencies": {
|
|
12
|
+
"ajv": "8.20.0"
|
|
13
|
+
},
|
|
14
|
+
"bin": {
|
|
15
|
+
"testguard": "cli/testguard.mjs"
|
|
16
|
+
},
|
|
11
17
|
"devDependencies": {
|
|
12
|
-
"ajv": "^8.20.0",
|
|
13
18
|
"vitest": "^5.0.1"
|
|
14
19
|
},
|
|
15
20
|
"engines": {
|
|
@@ -418,7 +423,6 @@
|
|
|
418
423
|
"version": "8.20.0",
|
|
419
424
|
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
|
|
420
425
|
"integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==",
|
|
421
|
-
"dev": true,
|
|
422
426
|
"license": "MIT",
|
|
423
427
|
"dependencies": {
|
|
424
428
|
"fast-deep-equal": "^3.1.3",
|
|
@@ -493,14 +497,12 @@
|
|
|
493
497
|
"version": "3.1.3",
|
|
494
498
|
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
|
|
495
499
|
"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
|
|
496
|
-
"dev": true,
|
|
497
500
|
"license": "MIT"
|
|
498
501
|
},
|
|
499
502
|
"node_modules/fast-uri": {
|
|
500
503
|
"version": "3.1.8",
|
|
501
504
|
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz",
|
|
502
505
|
"integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==",
|
|
503
|
-
"dev": true,
|
|
504
506
|
"funding": [
|
|
505
507
|
{
|
|
506
508
|
"type": "github",
|
|
@@ -551,7 +553,6 @@
|
|
|
551
553
|
"version": "1.0.0",
|
|
552
554
|
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
|
|
553
555
|
"integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
|
|
554
|
-
"dev": true,
|
|
555
556
|
"license": "MIT"
|
|
556
557
|
},
|
|
557
558
|
"node_modules/lightningcss": {
|
|
@@ -938,7 +939,6 @@
|
|
|
938
939
|
"version": "2.0.2",
|
|
939
940
|
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
|
|
940
941
|
"integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==",
|
|
941
|
-
"dev": true,
|
|
942
942
|
"license": "MIT",
|
|
943
943
|
"engines": {
|
|
944
944
|
"node": ">=0.10.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -28,7 +28,8 @@
|
|
|
28
28
|
"test:spec": "vitest run spec/conformance",
|
|
29
29
|
"self:probe": "node cli/testguard.mjs probe . --budget 180000",
|
|
30
30
|
"release:sync": "node .github/scripts/sync-release-version.mjs",
|
|
31
|
-
"prepublishOnly": "node cli/testguard.mjs --version && node .github/scripts/sync-release-version.mjs --check"
|
|
31
|
+
"prepublishOnly": "node cli/testguard.mjs --version && node .github/scripts/sync-release-version.mjs --check",
|
|
32
|
+
"test:install": "node .github/scripts/install-smoke.mjs"
|
|
32
33
|
},
|
|
33
34
|
"keywords": [
|
|
34
35
|
"testing",
|
|
@@ -45,7 +46,6 @@
|
|
|
45
46
|
"quality-gate"
|
|
46
47
|
],
|
|
47
48
|
"devDependencies": {
|
|
48
|
-
"ajv": "^8.20.0",
|
|
49
49
|
"vitest": "^5.0.1"
|
|
50
50
|
},
|
|
51
51
|
"bin": {
|
|
@@ -62,5 +62,8 @@
|
|
|
62
62
|
},
|
|
63
63
|
"publishConfig": {
|
|
64
64
|
"access": "public"
|
|
65
|
+
},
|
|
66
|
+
"dependencies": {
|
|
67
|
+
"ajv": "8.20.0"
|
|
65
68
|
}
|
|
66
69
|
}
|
|
@@ -11,10 +11,10 @@
|
|
|
11
11
|
# and set `sha256` from the published tarball:
|
|
12
12
|
# curl -sL https://registry.npmjs.org/testguard-cli/-/testguard-cli-<VER>.tgz | shasum -a 256
|
|
13
13
|
class Testguard < Formula
|
|
14
|
-
desc "Proves a test suite defends the claims a project makes
|
|
14
|
+
desc "Proves a test suite defends the claims a project makes"
|
|
15
15
|
homepage "https://github.com/raccioly/testguard"
|
|
16
|
-
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.
|
|
17
|
-
sha256 "
|
|
16
|
+
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.1.2.tgz"
|
|
17
|
+
sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
|
|
18
18
|
license "MIT"
|
|
19
19
|
|
|
20
20
|
depends_on "node"
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "testguard-cli"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.2"
|
|
8
8
|
description = "Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -12,7 +12,7 @@ The verdict set is closed. Only one value is a pass.
|
|
|
12
12
|
| `killed` | Fault applied; every one of N probe runs failed with a genuine assertion failure, on a defender set that was green N/N unmodified. | no |
|
|
13
13
|
| `survived` | Fault applied; every one of N probe runs passed. The claim is **unproven**. | **yes** |
|
|
14
14
|
| `nocover` | No defending test exists: the declared globs resolve to nothing, or no test imports the subject. Worse than `survived` — nothing was even tried. | **yes** |
|
|
15
|
-
| `unverifiable` | The
|
|
15
|
+
| `unverifiable` | The claim could not be probed: the fault's anchor is missing or ambiguous, or its defenders failed to load (`defenders-failed-to-load`). Carries a `reason`. A loud, gating verdict — a claim that cannot be probed is not "skipped", it is undefended until someone fixes the fault or the defenders. Never confused with `flaky-defender`, which requires tests that *ran*. | **yes** |
|
|
16
16
|
| `timeout` | Probe run exceeded its budget. Not counted as a kill; the pessimistic reading is the safe one because flakiness biases the metric optimistically. | **yes** |
|
|
17
17
|
| `fault-invalid` | The replacement does not load or compile. A bad fault, not a detection. | **yes** |
|
|
18
18
|
| `flaky-defender` | Defenders were not green N/N on unmodified source (`defenders-not-green`), or the N probe runs disagreed with each other (`inconsistent-probe`). Either way no verdict about the fault can be trusted; fix the defenders first. | **yes** |
|
|
@@ -81,7 +81,7 @@ evidence; they simply do not turn CI red.
|
|
|
81
81
|
|---|---|
|
|
82
82
|
| `0` | No new gating findings at or above the severity floor. |
|
|
83
83
|
| `1` | At least one new gating finding. |
|
|
84
|
-
| `2` | Precondition failed:
|
|
84
|
+
| `2` | Precondition failed: test runner not resolvable (e.g. no `node_modules` linked into the scratch worktree), working tree dirty for a fault target file, claims file invalid, no commits. Nothing was probed. |
|
|
85
85
|
| `3` | Usage or configuration error. |
|
|
86
86
|
|
|
87
87
|
A tool must never exit `0` because it had nothing to check. If the claims
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"enum": ["critical", "high", "medium", "low"]
|
|
32
32
|
},
|
|
33
33
|
"verdict": {
|
|
34
|
-
"description": "Closed set. Only `killed` is a pass. Every other verdict gates by default (see GATE-SEMANTICS.md).",
|
|
34
|
+
"description": "Closed set. Only `killed` is a pass. Every other verdict gates by default (see GATE-SEMANTICS.md). `unverifiable` covers a fault that could not be applied (anchor missing/ambiguous) and a claim whose defenders failed to load (`defenders-failed-to-load`).",
|
|
35
35
|
"enum": [
|
|
36
36
|
"killed",
|
|
37
37
|
"survived",
|