testguard-cli 0.1.3__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/scripts/install-smoke.mjs +2 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/AGENTS.md +2 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/CHANGELOG.md +18 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/PKG-INFO +35 -5
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/README.md +34 -4
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/action.yml +1 -1
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/package-lock.json +2 -2
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/package.json +1 -1
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/packaging/homebrew/testguard.rb +1 -1
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/pyproject.toml +1 -1
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/cli.mjs +8 -2
- testguard_cli-0.2.0/src/commands/scaffold.mjs +34 -0
- testguard_cli-0.2.0/src/scaffold/producers.mjs +85 -0
- testguard_cli-0.2.0/src/scaffold/scaffold.mjs +118 -0
- testguard_cli-0.2.0/test/scaffold.test.mjs +133 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/testguard.claims.json +31 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.gitattributes +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/ISSUE_TEMPLATE/bug_report.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/ISSUE_TEMPLATE/feature_request.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/dependabot.yml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/scripts/sync-release-version.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/workflows/auto-merge.yml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/workflows/ci.yml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/workflows/release.yml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/workflows/scheduled-release.yml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.github/workflows/supply-chain.yml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.gitignore +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.npmignore +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.npmrc +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/.pre-commit-hooks.yaml +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/CLAUDE.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/CODE_OF_CONDUCT.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/CONTRIBUTING.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/LICENSE +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/PRIVACY.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/SECURITY.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/SUPPORT.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/bench/README.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/cli/testguard.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/.gitignore +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/README.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/expected.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/package.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/src/export.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/src/redact.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/test/flaky.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/test/redact.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/testguard.claims.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/fixtures/known-answer/vitest.config.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/GATE-SEMANTICS.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/README.md +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/examples/baseline.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/examples/brief.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/examples/calibration.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/examples/claims.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/examples/evidence.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/examples/ignore.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/baseline.bad-fingerprint-key.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/baseline.zero-count.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/brief.text-without-heading.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/brief.unknown-verdict-key.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/calibration.p-outside-ci.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/calibration.positives-exceed-n.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.absolute-path.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.duplicate-fault-id.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.missing-provenance.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.noop-fault.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.occurrence-exceeds-hits.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.parent-traversal.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.flaky-defender-without-flakiness.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.killed-by-non-assertion.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.killed-on-red-baseline.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.killed-without-n-runs.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.rank-extra-property.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.undeclared-killers-without-reason.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.unknown-verdict.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.unverifiable-without-reason.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.wrong-fingerprint.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/ignore.missing-reason.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/ignore.short-reason.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/schemas.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/lib/fingerprint.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/lib/validate.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/baseline.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/brief.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/calibration.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/claims.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/common.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/evidence.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/schemas/ignore.schema.json +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/baseline/baseline.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/brief/brief.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/claims/annotations.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/claims/load.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/commands/baseline.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/commands/brief.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/commands/claims.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/commands/probe.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/evidence/writer.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/git.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/classify.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/discover.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/inject.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/probe.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/rank.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/runner-vitest.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/probe/worktree.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/render.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/util/glob.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/src/util/hash.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/annotations.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/baseline.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/brief.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/classify.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/discover.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/glob.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/inject.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/probe-preconditions.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/probe.fixture.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/rank-aliases.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/runner-command.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/runner-vitest.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/snapshot.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/worktree.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/test/writer.test.mjs +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/testguard_cli/__init__.py +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/testguard_cli/wrapper.py +0 -0
- {testguard_cli-0.1.3 → testguard_cli-0.2.0}/vitest.config.mjs +0 -0
|
@@ -30,6 +30,8 @@ try {
|
|
|
30
30
|
const bin = join(consumer, 'node_modules', '.bin', process.platform === 'win32' ? 'testguard.cmd' : 'testguard');
|
|
31
31
|
const version = run(bin, ['--version'], consumer).trim();
|
|
32
32
|
const claims = run(bin, ['claims', '.'], consumer);
|
|
33
|
+
const draft = JSON.parse(run(bin, ['scaffold', 'src/redact.mjs', '--json'], consumer));
|
|
34
|
+
if (!draft.claims?.length) throw new Error('scaffold produced no claims from the installed tarball');
|
|
33
35
|
if (!/^\d+\.\d+\.\d+/.test(version)) throw new Error(`unexpected --version output: ${version}`);
|
|
34
36
|
if (!/^\d+ claims in /.test(claims)) throw new Error(`claims did not list the fixture:\n${claims}`);
|
|
35
37
|
const deps = Object.keys(JSON.parse(run(npm, ['ls', '--omit=dev', '--json', '--depth=0'], consumer)).dependencies ?? {});
|
|
@@ -19,6 +19,7 @@ npm run test:spec # conformance suite only
|
|
|
19
19
|
npm run self:probe # TestGuard probes its own claims; must exit 0
|
|
20
20
|
npm run test:install # the packed tarball must run with production deps only
|
|
21
21
|
node cli/testguard.mjs claims fixtures/known-answer
|
|
22
|
+
node cli/testguard.mjs scaffold src/probe/classify.mjs --json # what the producers propose for a file
|
|
22
23
|
```
|
|
23
24
|
|
|
24
25
|
## Key files
|
|
@@ -29,6 +30,7 @@ node cli/testguard.mjs claims fixtures/known-answer
|
|
|
29
30
|
- `src/probe/probe.mjs` — orchestrator: isolation → baseline (cached per defender set) → apply → probe → escalate → restore → classify → rank.
|
|
30
31
|
- `fixtures/known-answer/expected.json` — the oracle. Never edit it to match output.
|
|
31
32
|
- `testguard.claims.json` — claims about this codebase, probed in CI.
|
|
33
|
+
- `src/scaffold/producers.mjs` — the five fault shapes. Deterministic line heuristics; no AST, no LLM. A new shape needs a synthetic-file test and a README row.
|
|
32
34
|
|
|
33
35
|
## Rules
|
|
34
36
|
|
|
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.2.0] - 2026-09-17
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **`testguard scaffold <file>`** — mechanical fault producer (#6). Proposes
|
|
15
|
+
the five shapes both field reports found behind ~80% of hand-written
|
|
16
|
+
faults: guard forced false, single-line guard or state change removed,
|
|
17
|
+
`return <check>` → `return true`, security literal weakened, check call
|
|
18
|
+
removed. Every proposal is an exact-line anchor with `expectHits` and
|
|
19
|
+
`occurrence` computed from the file (verifiable by construction; anything
|
|
20
|
+
`locate()` would reject is never emitted), `producedBy: derived`,
|
|
21
|
+
`defendedBy` prefilled from the tests that import the module, grouped
|
|
22
|
+
under a preceding `@claim` annotation or by enclosing function.
|
|
23
|
+
Statements are `TODO:` placeholders; the output is a draft under
|
|
24
|
+
`.testguard/`, never the claims file. `--claim <ID>` puts everything under
|
|
25
|
+
one claim and copies it if it exists; `--json` prints instead.
|
|
26
|
+
- Self-claim `TG-SCAFFOLD-ANCHORS-HIT`; install smoke exercises `scaffold`.
|
|
27
|
+
|
|
10
28
|
## [0.1.3] - 2026-09-17
|
|
11
29
|
|
|
12
30
|
From a second field report on a real codebase (456 tests, 27 claims, 35
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: testguard-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+).
|
|
5
5
|
Project-URL: Homepage, https://github.com/raccioly/testguard
|
|
6
6
|
Project-URL: Documentation, https://github.com/raccioly/testguard#readme
|
|
@@ -76,7 +76,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
76
76
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
77
77
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
78
78
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
79
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
79
|
+
| GitHub Action | `uses: raccioly/testguard@v0.2.0` — see [`action.yml`](./action.yml) |
|
|
80
80
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
81
81
|
|
|
82
82
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -90,6 +90,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
90
90
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
91
91
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
92
92
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
93
|
+
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
93
94
|
```
|
|
94
95
|
|
|
95
96
|
1. **Claims** live in `testguard.claims.json` (editors validate it against
|
|
@@ -159,6 +160,34 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
159
160
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
160
161
|
so the hook can never break a session.
|
|
161
162
|
|
|
163
|
+
### Authoring faults mechanically
|
|
164
|
+
|
|
165
|
+
Writing faults by hand means reading the code to find exact anchors. Two
|
|
166
|
+
field reports found that ~80% of hand-written faults are one of five shapes,
|
|
167
|
+
so `scaffold` proposes them for you:
|
|
168
|
+
|
|
169
|
+
```bash
|
|
170
|
+
npx testguard-cli scaffold src/auth.ts # → .testguard/scaffold-auth.json (a draft, never your claims file)
|
|
171
|
+
npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN # every proposal under one claim; copies it if it exists
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
| Shape | What it proposes |
|
|
175
|
+
|---|---|
|
|
176
|
+
| `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
|
|
177
|
+
| `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
|
|
178
|
+
| `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
|
|
179
|
+
| `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
|
|
180
|
+
| `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
|
|
181
|
+
|
|
182
|
+
Every proposal's `find` is the exact line with `expectHits`/`occurrence`
|
|
183
|
+
computed from the file, so it is verifiable by construction; provenance is
|
|
184
|
+
`producer: derived`; `defendedBy` is prefilled from the tests that import
|
|
185
|
+
the module; proposals are grouped under a preceding `@claim <ID>` annotation
|
|
186
|
+
or by enclosing function. Statements are `TODO:` placeholders — a proposal
|
|
187
|
+
becomes a claim only when a human states what it defends. Deterministic
|
|
188
|
+
heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
|
|
189
|
+
emitted.
|
|
190
|
+
|
|
162
191
|
**Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
|
|
163
192
|
The baseline is the frozen contract; the other two are regenerated per run.
|
|
164
193
|
|
|
@@ -184,13 +213,14 @@ through every verdict.
|
|
|
184
213
|
|
|
185
214
|
## Status
|
|
186
215
|
|
|
187
|
-
**v0.
|
|
216
|
+
**v0.2.** Five commands, vitest runner, hand-authored faults plus a
|
|
217
|
+
mechanical scaffold for the five common shapes. The contract
|
|
188
218
|
spine — six JSON Schemas shared with the other Guard tools — is under
|
|
189
219
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
190
220
|
|
|
191
221
|
Not yet: test generation (the two-gate acceptance loop), other runners,
|
|
192
|
-
|
|
193
|
-
|
|
222
|
+
AST-aware producers, and calibration of fault classes against real escaped
|
|
223
|
+
bugs. Each is designed for; none is claimed.
|
|
194
224
|
|
|
195
225
|
## Licence
|
|
196
226
|
|
|
@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
|
|
|
52
52
|
| npm | `npm i -D testguard-cli` then `npx testguard probe` |
|
|
53
53
|
| pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
|
|
54
54
|
| Homebrew | `brew tap raccioly/tap && brew install testguard` |
|
|
55
|
-
| GitHub Action | `uses: raccioly/testguard@v0.
|
|
55
|
+
| GitHub Action | `uses: raccioly/testguard@v0.2.0` — see [`action.yml`](./action.yml) |
|
|
56
56
|
| pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
|
|
57
57
|
|
|
58
58
|
Projects that set `min-release-age` in `.npmrc` cannot see a version published
|
|
@@ -66,6 +66,7 @@ npx testguard-cli claims # what does this project claim, and is every claim
|
|
|
66
66
|
npx testguard-cli probe # try to falsify each claim; report what the tests missed
|
|
67
67
|
npx testguard-cli baseline # freeze today's unproven findings; from now on only new ones gate
|
|
68
68
|
npx testguard-cli brief # tell the agent where the suite is blind, before it writes
|
|
69
|
+
npx testguard-cli scaffold src/x.ts # propose faults for a file, as a draft to keep or drop
|
|
69
70
|
```
|
|
70
71
|
|
|
71
72
|
1. **Claims** live in `testguard.claims.json` (editors validate it against
|
|
@@ -135,6 +136,34 @@ npx testguard-cli brief # tell the agent where the suite is blind, before
|
|
|
135
136
|
`--text` prints only, and exits 0 silently when there is no evidence yet,
|
|
136
137
|
so the hook can never break a session.
|
|
137
138
|
|
|
139
|
+
### Authoring faults mechanically
|
|
140
|
+
|
|
141
|
+
Writing faults by hand means reading the code to find exact anchors. Two
|
|
142
|
+
field reports found that ~80% of hand-written faults are one of five shapes,
|
|
143
|
+
so `scaffold` proposes them for you:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
npx testguard-cli scaffold src/auth.ts # → .testguard/scaffold-auth.json (a draft, never your claims file)
|
|
147
|
+
npx testguard-cli scaffold src/auth.ts --claim AUTH-ADMIN # every proposal under one claim; copies it if it exists
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
| Shape | What it proposes |
|
|
151
|
+
|---|---|
|
|
152
|
+
| `condition-forced` | `if (<guard>) {` → `if (false) {` — a guard is a `!…` condition or one whose body returns, throws or 4xx-es |
|
|
153
|
+
| `statement-deleted` | a single-line guard (`if (…) return …;`) or a state change (`x = …;`) removed |
|
|
154
|
+
| `return-altered` | `return <check>;` (`===`, `.includes(`, `&&`, …) → `return true;` |
|
|
155
|
+
| `literal-changed` | `httpOnly`/`secure` flipped, `sameSite` → `none`, a cost/rounds → `1`, a ttl/tolerance/window/limit ×1000 |
|
|
156
|
+
| `call-removed` | a bare `verify…()` / `validate…()` / `check…()` / `authorize…()` call removed |
|
|
157
|
+
|
|
158
|
+
Every proposal's `find` is the exact line with `expectHits`/`occurrence`
|
|
159
|
+
computed from the file, so it is verifiable by construction; provenance is
|
|
160
|
+
`producer: derived`; `defendedBy` is prefilled from the tests that import
|
|
161
|
+
the module; proposals are grouped under a preceding `@claim <ID>` annotation
|
|
162
|
+
or by enclosing function. Statements are `TODO:` placeholders — a proposal
|
|
163
|
+
becomes a claim only when a human states what it defends. Deterministic
|
|
164
|
+
heuristics, no AST, no LLM; a proposal the tool cannot anchor is never
|
|
165
|
+
emitted.
|
|
166
|
+
|
|
138
167
|
**Commit `.testguard/baseline.json`; ignore `evidence.json` and `brief.json`.**
|
|
139
168
|
The baseline is the frozen contract; the other two are regenerated per run.
|
|
140
169
|
|
|
@@ -160,13 +189,14 @@ through every verdict.
|
|
|
160
189
|
|
|
161
190
|
## Status
|
|
162
191
|
|
|
163
|
-
**v0.
|
|
192
|
+
**v0.2.** Five commands, vitest runner, hand-authored faults plus a
|
|
193
|
+
mechanical scaffold for the five common shapes. The contract
|
|
164
194
|
spine — six JSON Schemas shared with the other Guard tools — is under
|
|
165
195
|
[`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
|
|
166
196
|
|
|
167
197
|
Not yet: test generation (the two-gate acceptance loop), other runners,
|
|
168
|
-
|
|
169
|
-
|
|
198
|
+
AST-aware producers, and calibration of fault classes against real escaped
|
|
199
|
+
bugs. Each is designed for; none is claimed.
|
|
170
200
|
|
|
171
201
|
## Licence
|
|
172
202
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "testguard-cli",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.2.0",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"ajv": "8.20.0"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "testguard-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
class Testguard < Formula
|
|
14
14
|
desc "Proves a test suite defends the claims a project makes"
|
|
15
15
|
homepage "https://github.com/raccioly/testguard"
|
|
16
|
-
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.
|
|
16
|
+
url "https://registry.npmjs.org/testguard-cli/-/testguard-cli-0.2.0.tgz"
|
|
17
17
|
sha256 "385d69f9d3c153b934d9c1cb6a2c754eb9b221b0a8c0ba8b805858384d2d4678"
|
|
18
18
|
license "MIT"
|
|
19
19
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "testguard-cli"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "Proves a test suite actually defends the claims a project makes: injects the faults those claims forbid and reports every one the tests miss. Python wrapper for the Node.js CLI (requires Node.js 20+)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { text = "MIT" }
|
|
@@ -10,6 +10,7 @@ import { probeCommand } from './commands/probe.mjs';
|
|
|
10
10
|
import { claimsCommand } from './commands/claims.mjs';
|
|
11
11
|
import { baselineCommand } from './commands/baseline.mjs';
|
|
12
12
|
import { briefCommand } from './commands/brief.mjs';
|
|
13
|
+
import { scaffoldCommand } from './commands/scaffold.mjs';
|
|
13
14
|
|
|
14
15
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
15
16
|
|
|
@@ -19,6 +20,7 @@ const USAGE = `testguard ${VERSION} — proves a test suite defends the claims a
|
|
|
19
20
|
testguard probe [dir] inject each claim's faults, run its defenders, report what survived
|
|
20
21
|
testguard baseline [dir] freeze today's unproven findings so only new ones gate
|
|
21
22
|
testguard brief [dir] emit the blind-spot block for an agent's session-start context
|
|
23
|
+
testguard scaffold <file> propose faults mechanically for one source file, as a draft claims document
|
|
22
24
|
|
|
23
25
|
probe
|
|
24
26
|
--claims <path> claims file (default: <dir>/testguard.claims.json)
|
|
@@ -38,6 +40,9 @@ probe
|
|
|
38
40
|
--no-reuse re-probe claims whose inputs have not changed
|
|
39
41
|
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
40
42
|
|
|
43
|
+
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
44
|
+
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
45
|
+
· security flag/window/cost literal weakened · verify/validate/check call removed
|
|
41
46
|
claims --json
|
|
42
47
|
baseline --evidence <path> --out <path>
|
|
43
48
|
brief --evidence <path> --baseline <path> --max <n> --text (print only; safe for hooks)
|
|
@@ -45,7 +50,7 @@ brief --evidence <path> --baseline <path> --max <n> --text (print only;
|
|
|
45
50
|
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift) · 2 precondition failed · 3 usage
|
|
46
51
|
`;
|
|
47
52
|
|
|
48
|
-
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand };
|
|
53
|
+
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand };
|
|
49
54
|
|
|
50
55
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
51
56
|
let parsed;
|
|
@@ -104,7 +109,8 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
104
109
|
return 3;
|
|
105
110
|
}
|
|
106
111
|
try {
|
|
107
|
-
|
|
112
|
+
const projectDir = command === 'scaffold' ? resolve('.') : resolve(dirArg ?? '.');
|
|
113
|
+
return await handler({ projectDir, file: dirArg, values, version: VERSION }, io);
|
|
108
114
|
} catch (e) {
|
|
109
115
|
if (e instanceof ClaimsError || e instanceof PreconditionError || e instanceof GitError || e instanceof SpecDocError) {
|
|
110
116
|
io.err(`error: ${e.message}`);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { join, resolve, relative, basename, extname } from 'node:path';
|
|
3
|
+
import { scaffoldFile } from '../scaffold/scaffold.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
|
+
import { PreconditionError } from '../probe/worktree.mjs';
|
|
7
|
+
|
|
8
|
+
export async function scaffoldCommand({ projectDir, file, values, version }, io) {
|
|
9
|
+
if (!file) {
|
|
10
|
+
io.err('usage: testguard scaffold <source-file> [--claim <ID>] [--out <path>] [--json]');
|
|
11
|
+
return 3;
|
|
12
|
+
}
|
|
13
|
+
const abs = resolve(file);
|
|
14
|
+
if (!existsSync(abs)) throw new PreconditionError(`no such file: ${file}`);
|
|
15
|
+
const rel = relative(projectDir, abs);
|
|
16
|
+
if (rel.startsWith('..')) throw new PreconditionError(`${file} is outside the project directory ${projectDir}`);
|
|
17
|
+
|
|
18
|
+
const claimsPath = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
19
|
+
const existingClaims = existsSync(claimsPath) ? loadClaims(claimsPath) : undefined;
|
|
20
|
+
const { doc, stats } = scaffoldFile({ projectDir, file: rel, claimId: values.claim, existingClaims, toolVersion: version });
|
|
21
|
+
|
|
22
|
+
if (values.json) {
|
|
23
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
24
|
+
return 0;
|
|
25
|
+
}
|
|
26
|
+
const outPath = values.out ? resolve(values.out) : join(projectDir, '.testguard', `scaffold-${basename(rel, extname(rel))}.json`);
|
|
27
|
+
writeSpecDoc('claims', outPath, doc);
|
|
28
|
+
const shapes = Object.entries(stats.byClass).map(([k, v]) => `${v} ${k}`).join(', ');
|
|
29
|
+
io.out(`${stats.proposals} proposed fault${stats.proposals === 1 ? '' : 's'} in ${stats.claims} draft claim${stats.claims === 1 ? '' : 's'} for ${rel}${shapes ? ` — ${shapes}` : ''}`);
|
|
30
|
+
io.out(stats.defendedBy.length ? `defendedBy prefilled from imports: ${stats.defendedBy.join(', ')}` : 'no test file imports this module; probing the draft as-is will report NOCOVER');
|
|
31
|
+
io.out(`draft: ${outPath}`);
|
|
32
|
+
io.out('Next: replace each TODO statement, drop proposals that are not claims, then move the claims into testguard.claims.json.');
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mechanical fault producers. Deterministic, line-oriented, no AST, no LLM.
|
|
3
|
+
*
|
|
4
|
+
* Two independent field reports found that ~80% of hand-written faults are
|
|
5
|
+
* one of these shapes. Each producer looks at one line (plus a little
|
|
6
|
+
* context) and proposes a fault whose `find` is the exact line, so the anchor
|
|
7
|
+
* hits by construction. A human keeps or drops every proposal.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const COMMENT = /^\s*(\/\/|\*|\/\*)/;
|
|
11
|
+
const GUARD_BODY = /\b(return|throw)\b|\.status\(\s*4\d\d|\bredirect\(|\bfalse\b|\bnull\b/;
|
|
12
|
+
const IF_LINE = /^(\s*)(?:\}\s*)?(?:else\s+)?if\s*\((.+)\)\s*(\{\s*|(?:return|throw)\b.*;\s*)?$/;
|
|
13
|
+
const RETURN_CHECK = /^\s*return\s+(.+);\s*$/;
|
|
14
|
+
const CHECK_EXPR = /(===|!==|\.includes\(|\.has\(|\.some\(|\.every\(|\.test\(|\.startsWith\(|\.endsWith\(|\binstanceof\b|&&|\|\||^!)/;
|
|
15
|
+
const CHECK_CALL = /^\s*(?:await\s+)?(?:[\w$]+\.)*(verify|validate|assert|check|require|ensure|authoriz|authentic|rateLimit|throttle|enforce|guard)\w*\s*\(.*\)\s*;\s*$/i;
|
|
16
|
+
const MUTATION = /^\s*(?!(?:const|let|var|return|if|for|while|else|switch|case|import|export|throw)\b)[\w$]+(?:[.\[][\w$'"\]]+)*\s*(=|\+=|-=|\|\|=|&&=|\?\?=)\s*(?!=)[^;]*;\s*$/;
|
|
17
|
+
const FLAG_TRUE = /\b(httpOnly|secure|signed|requireTLS|rejectUnauthorized|strict)\s*:\s*true\b/;
|
|
18
|
+
const SAME_SITE = /\bsameSite\s*:\s*(['"])(strict|lax)\1/i;
|
|
19
|
+
const COST_NUM = /\b(\w*(?:cost|rounds|iterations)\w*)\s*([:=])\s*(\d+)\b/i;
|
|
20
|
+
const WINDOW_NUM = /\b(\w*(?:ttl|expir|tolerance|window|maxAge|max_age|timeout|limit|attempts|length|minLength|clockSkew|skew)\w*)\s*([:=])\s*(\d+)\b/i;
|
|
21
|
+
const FUNCTION_HEAD = [
|
|
22
|
+
/^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?function\s*\*?\s*([\w$]+)\s*\(/,
|
|
23
|
+
/^\s*(?:export\s+)?(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s*)?(?:\([^)]*\)|[\w$]+)\s*=>/,
|
|
24
|
+
/^\s*(?:export\s+)?(?:const|let|var)\s+([\w$]+)\s*=\s*(?:async\s+)?function\b/,
|
|
25
|
+
/^\s*(?:public\s+|private\s+|protected\s+)?(?:static\s+)?(?:async\s+)?([\w$]+)\s*\([^)]*\)\s*(?::\s*[\w<>\[\]| ]+)?\s*\{\s*$/,
|
|
26
|
+
];
|
|
27
|
+
const NOT_A_FUNCTION = new Set(['if', 'for', 'while', 'switch', 'catch', 'else', 'return', 'function', 'constructor']);
|
|
28
|
+
|
|
29
|
+
/** Name of the function whose head is on this line, or null. */
|
|
30
|
+
export function functionHead(line) {
|
|
31
|
+
for (const re of FUNCTION_HEAD) {
|
|
32
|
+
const m = re.exec(line);
|
|
33
|
+
if (m && !NOT_A_FUNCTION.has(m[1])) return m[1];
|
|
34
|
+
}
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function isGuard(cond, lines, i) {
|
|
39
|
+
if (/^\s*!/.test(cond)) return true;
|
|
40
|
+
const tail = lines.slice(i, i + 4).join('\n');
|
|
41
|
+
return GUARD_BODY.test(tail);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** Proposals for one line. Each: { faultClass, description, replace } where find is the line itself. */
|
|
45
|
+
export function proposalsForLine(lines, i) {
|
|
46
|
+
const line = lines[i];
|
|
47
|
+
const out = [];
|
|
48
|
+
if (!line.trim() || COMMENT.test(line)) return out;
|
|
49
|
+
|
|
50
|
+
const ifm = IF_LINE.exec(line);
|
|
51
|
+
if (ifm && isGuard(ifm[2], lines, i)) {
|
|
52
|
+
const singleLine = ifm[3] && /^(return|throw)\b/.test(ifm[3].trim());
|
|
53
|
+
if (singleLine) {
|
|
54
|
+
out.push({ faultClass: 'statement-deleted', description: `Guard removed: \`${line.trim()}\` no longer runs.`, replace: '' });
|
|
55
|
+
} else {
|
|
56
|
+
out.push({ faultClass: 'condition-forced', description: `Guard never triggers: \`if (${ifm[2].trim()})\` becomes \`if (false)\`.`, replace: line.replace(`(${ifm[2]})`, '(false)') });
|
|
57
|
+
}
|
|
58
|
+
return out; // an `if` line is not also a mutation/return/call
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const rm = RETURN_CHECK.exec(line);
|
|
62
|
+
if (rm && CHECK_EXPR.test(rm[1]) && !/^(new|await)\b/.test(rm[1].trim())) {
|
|
63
|
+
out.push({ faultClass: 'return-altered', description: `Check always passes: \`return ${rm[1].trim()}\` becomes \`return true\`.`, replace: line.replace(/return\s+.+;/, 'return true;') });
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (CHECK_CALL.test(line)) {
|
|
67
|
+
out.push({ faultClass: 'call-removed', description: `Check call removed: \`${line.trim()}\` no longer runs.`, replace: '' });
|
|
68
|
+
} else if (MUTATION.test(line)) {
|
|
69
|
+
out.push({ faultClass: 'statement-deleted', description: `State change removed: \`${line.trim()}\` no longer runs.`, replace: '' });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
let m;
|
|
73
|
+
if ((m = FLAG_TRUE.exec(line))) {
|
|
74
|
+
out.push({ faultClass: 'literal-changed', description: `Security flag flipped: \`${m[1]}: true\` becomes \`${m[1]}: false\`.`, replace: line.replace(m[0], `${m[1]}: false`) });
|
|
75
|
+
}
|
|
76
|
+
if ((m = SAME_SITE.exec(line))) {
|
|
77
|
+
out.push({ faultClass: 'literal-changed', description: `sameSite weakened: \`${m[2]}\` becomes \`none\`.`, replace: line.replace(m[0], `sameSite: ${m[1]}none${m[1]}`) });
|
|
78
|
+
}
|
|
79
|
+
if ((m = COST_NUM.exec(line))) {
|
|
80
|
+
out.push({ faultClass: 'literal-changed', description: `Work factor collapsed: \`${m[1]}\` ${m[3]} becomes 1.`, replace: line.replace(m[0], `${m[1]}${m[2] === ':' ? ': ' : ' = '}1`) });
|
|
81
|
+
} else if ((m = WINDOW_NUM.exec(line))) {
|
|
82
|
+
out.push({ faultClass: 'literal-changed', description: `Window widened ×1000: \`${m[1]}\` ${m[3]} becomes ${Number(m[3]) * 1000}.`, replace: line.replace(m[0], `${m[1]}${m[2] === ':' ? ': ' : ' = '}${Number(m[3]) * 1000}`) });
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { basename, extname, join } from 'node:path';
|
|
3
|
+
import { proposalsForLine, functionHead } from './producers.mjs';
|
|
4
|
+
import { locate } from '../probe/inject.mjs';
|
|
5
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
6
|
+
import { validate } from '../../spec/lib/validate.mjs';
|
|
7
|
+
|
|
8
|
+
const ANNOTATION = /@claim\s+([A-Za-z0-9]+(?:[._]?[A-Za-z0-9]+)*-[A-Za-z0-9._-]*[A-Za-z0-9])\b/;
|
|
9
|
+
|
|
10
|
+
const idPart = (s) => s.replace(/[^A-Za-z0-9]+/g, '-').replace(/^-+|-+$/g, '').toUpperCase();
|
|
11
|
+
|
|
12
|
+
/** How many times `find` occurs, and which occurrence the line at `offset` is. */
|
|
13
|
+
function anchorFor(source, find, offset) {
|
|
14
|
+
let hits = 0;
|
|
15
|
+
let occurrence = 0;
|
|
16
|
+
let i = -1;
|
|
17
|
+
while ((i = source.indexOf(find, i + 1)) !== -1) {
|
|
18
|
+
hits++;
|
|
19
|
+
if (i === offset) occurrence = hits;
|
|
20
|
+
}
|
|
21
|
+
return { hits, occurrence };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Scan one source file and produce a DRAFT claims document: every proposal is
|
|
26
|
+
* an exact-line anchor that `locate()` accepts, grouped by enclosing function
|
|
27
|
+
* (or by a preceding `@claim <ID>` annotation, or entirely under `claimId`),
|
|
28
|
+
* with `producedBy: { producer: "derived" }` and TODO statements a human must
|
|
29
|
+
* replace. Never touches the real claims file.
|
|
30
|
+
*/
|
|
31
|
+
export function scaffoldFile({ projectDir, file, claimId, existingClaims, toolVersion = '0.0.0' }) {
|
|
32
|
+
const source = readFileSync(join(projectDir, file), 'utf8');
|
|
33
|
+
const lines = source.split('\n');
|
|
34
|
+
const stem = idPart(basename(file, extname(file)));
|
|
35
|
+
const producedBy = { producer: 'derived', by: `testguard scaffold ${toolVersion}` };
|
|
36
|
+
|
|
37
|
+
const groups = new Map(); // key → { id, proposals[] , annotated }
|
|
38
|
+
const groupFor = (key, id) => {
|
|
39
|
+
if (!groups.has(key)) groups.set(key, { id, proposals: [] });
|
|
40
|
+
return groups.get(key);
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
let offset = 0;
|
|
44
|
+
let fn = null;
|
|
45
|
+
let fnDepth = 0;
|
|
46
|
+
let depth = 0;
|
|
47
|
+
let pendingAnnotation = null;
|
|
48
|
+
const proposals = [];
|
|
49
|
+
lines.forEach((line, i) => {
|
|
50
|
+
const ann = ANNOTATION.exec(line);
|
|
51
|
+
if (ann && /^\s*(\/\/|\*|\/\*)/.test(line)) pendingAnnotation = ann[1];
|
|
52
|
+
|
|
53
|
+
const head = functionHead(line);
|
|
54
|
+
if (head) {
|
|
55
|
+
fn = head;
|
|
56
|
+
fnDepth = depth;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
for (const p of proposalsForLine(lines, i)) {
|
|
60
|
+
const find = line;
|
|
61
|
+
const { hits, occurrence } = anchorFor(source, find, offset);
|
|
62
|
+
const fault = { ...p, find, replace: p.replace, expectHits: hits, occurrence, line: i + 1, fn, annotation: pendingAnnotation };
|
|
63
|
+
if (locate(source, fault).status !== 'ok') continue; // never propose an anchor that would be unverifiable
|
|
64
|
+
proposals.push(fault);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
depth += (line.match(/\{/g) ?? []).length - (line.match(/\}/g) ?? []).length;
|
|
68
|
+
if (fn && depth <= fnDepth && !head) fn = null;
|
|
69
|
+
if (pendingAnnotation && !ann && !head && line.trim() && !/^\s*(\/\/|\*|\/\*)/.test(line)) pendingAnnotation = null;
|
|
70
|
+
offset += line.length + 1;
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const existing = new Map((existingClaims?.claims ?? []).map((c) => [c.id, c]));
|
|
74
|
+
const usedIds = new Set();
|
|
75
|
+
for (const p of proposals) {
|
|
76
|
+
const key = claimId ? '__all__' : p.annotation ?? p.fn ?? '__file__';
|
|
77
|
+
let id = claimId ?? p.annotation ?? `${stem}-${p.fn ? idPart(p.fn) : 'FILE'}`;
|
|
78
|
+
groupFor(key, id).proposals.push(p);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const defendedBy = discoverDefenders(projectDir, file);
|
|
82
|
+
const claims = [];
|
|
83
|
+
for (const g of groups.values()) {
|
|
84
|
+
let id = g.id;
|
|
85
|
+
while (usedIds.has(id)) id += '-2';
|
|
86
|
+
usedIds.add(id);
|
|
87
|
+
const base = existing.get(id);
|
|
88
|
+
const fnLabel = g.proposals[0].fn ? `\`${g.proposals[0].fn}\`` : 'this file';
|
|
89
|
+
const claim = {
|
|
90
|
+
id,
|
|
91
|
+
statement: base?.statement ?? `TODO: state what ${fnLabel} in ${file} guarantees (${g.proposals.length} proposed fault${g.proposals.length === 1 ? '' : 's'}; keep or drop each)`,
|
|
92
|
+
source: base?.source ?? { kind: 'manual', ref: `testguard scaffold ${file}` },
|
|
93
|
+
severity: base?.severity ?? 'medium',
|
|
94
|
+
producedBy: base?.producedBy ?? producedBy,
|
|
95
|
+
...(base?.defendedBy?.length ? { defendedBy: base.defendedBy } : defendedBy.length ? { defendedBy } : {}),
|
|
96
|
+
faults: g.proposals.map((p, n) => ({
|
|
97
|
+
id: `S${n + 1}`,
|
|
98
|
+
description: `[line ${p.line}] ${p.description}`,
|
|
99
|
+
faultClass: p.faultClass,
|
|
100
|
+
file,
|
|
101
|
+
find: p.find,
|
|
102
|
+
replace: p.replace,
|
|
103
|
+
...(p.expectHits > 1 ? { expectHits: p.expectHits, occurrence: p.occurrence } : {}),
|
|
104
|
+
producedBy,
|
|
105
|
+
})),
|
|
106
|
+
...(base?.tags ? { tags: base.tags } : {}),
|
|
107
|
+
};
|
|
108
|
+
claims.push(claim);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const doc = { $schema: './node_modules/testguard-cli/spec/schemas/claims.schema.json', schemaVersion: 1, claims };
|
|
112
|
+
const result = validate('claims', doc);
|
|
113
|
+
if (!result.ok) throw new Error(`scaffold produced a non-conforming draft:\n${result.errors.map((e) => ` ${e.path}: ${e.message}`).join('\n')}`);
|
|
114
|
+
|
|
115
|
+
const byClass = {};
|
|
116
|
+
for (const p of proposals) byClass[p.faultClass] = (byClass[p.faultClass] ?? 0) + 1;
|
|
117
|
+
return { doc, stats: { proposals: proposals.length, claims: claims.length, byClass, defendedBy } };
|
|
118
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { dirname, join } from 'node:path';
|
|
5
|
+
import { fileURLToPath } from 'node:url';
|
|
6
|
+
import { scaffoldFile } from '../src/scaffold/scaffold.mjs';
|
|
7
|
+
import { proposalsForLine, functionHead } from '../src/scaffold/producers.mjs';
|
|
8
|
+
import { locate } from '../src/probe/inject.mjs';
|
|
9
|
+
import { validate } from '../spec/lib/validate.mjs';
|
|
10
|
+
import { main } from '../src/cli.mjs';
|
|
11
|
+
|
|
12
|
+
const FIXTURE = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'known-answer');
|
|
13
|
+
const capture = () => { const lines = { out: [], err: [] }; return { lines, io: { out: (s) => lines.out.push(s), err: (s) => lines.err.push(s) } }; };
|
|
14
|
+
|
|
15
|
+
describe('scaffold on the known-answer fixture', () => {
|
|
16
|
+
const { doc, stats } = scaffoldFile({ projectDir: FIXTURE, file: 'src/redact.mjs', toolVersion: 'test' });
|
|
17
|
+
const all = doc.claims.flatMap((c) => c.faults.map((f) => ({ claim: c, ...f })));
|
|
18
|
+
|
|
19
|
+
it('produces a conforming draft whose every anchor locates exactly as declared', () => {
|
|
20
|
+
expect(validate('claims', doc).errors).toEqual([]);
|
|
21
|
+
const source = readFileSync(join(FIXTURE, 'src/redact.mjs'), 'utf8');
|
|
22
|
+
for (const f of all) expect(locate(source, f).status, f.description).toBe('ok');
|
|
23
|
+
expect(all.length).toBeGreaterThanOrEqual(3);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('proposes the shape-expressible fixture faults by anchor: the fail-closed guard and the masking mutation', () => {
|
|
27
|
+
const guard = all.find((f) => f.find.includes('if (!ctx || !ctx.scope) {'));
|
|
28
|
+
expect(guard).toMatchObject({ faultClass: 'condition-forced' });
|
|
29
|
+
expect(guard.replace.trim()).toBe('if (false) {');
|
|
30
|
+
const mutation = all.find((f) => f.find.includes("out = out.replace(rule.re, (m) => '*'.repeat(m.length));"));
|
|
31
|
+
expect(mutation).toMatchObject({ faultClass: 'statement-deleted', replace: '' });
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('groups the guard under its @claim annotation and everything else by enclosing function', () => {
|
|
35
|
+
expect(doc.claims.map((c) => c.id).sort()).toEqual(['REDACT-003', 'REDACT-FINDRULE', 'REDACT-MASK']);
|
|
36
|
+
expect(doc.claims.find((c) => c.id === 'REDACT-MASK').statement).toMatch(/^TODO: state what `mask`/);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('prefills defendedBy from discovery, marks provenance derived, never touches the claims file', () => {
|
|
40
|
+
for (const c of doc.claims) {
|
|
41
|
+
expect(c.defendedBy).toEqual(['test/redact.test.mjs']);
|
|
42
|
+
expect(c.producedBy.producer).toBe('derived');
|
|
43
|
+
for (const f of c.faults) expect(f.producedBy).toMatchObject({ producer: 'derived', by: 'testguard scaffold test' });
|
|
44
|
+
}
|
|
45
|
+
expect(stats.defendedBy).toEqual(['test/redact.test.mjs']);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
describe('scaffold producers on a synthetic security module', () => {
|
|
50
|
+
const dir = mkdtempSync(join(tmpdir(), 'tg-scaffold-'));
|
|
51
|
+
mkdirSync(join(dir, 'src'));
|
|
52
|
+
mkdirSync(join(dir, 'test'));
|
|
53
|
+
writeFileSync(join(dir, 'src', 'auth.ts'), `import bcrypt from 'bcrypt';
|
|
54
|
+
export const COOKIE = { httpOnly: true, secure: true, sameSite: 'strict', maxAge: 3600 };
|
|
55
|
+
const BCRYPT_COST = 12;
|
|
56
|
+
const tolerance = 300;
|
|
57
|
+
export function isAdmin(user) {
|
|
58
|
+
return user.roles.includes('admin');
|
|
59
|
+
}
|
|
60
|
+
export async function handle(req, res) {
|
|
61
|
+
if (!req.user) return res.status(401).end();
|
|
62
|
+
if (!isAdmin(req.user)) {
|
|
63
|
+
return res.status(403).end();
|
|
64
|
+
}
|
|
65
|
+
await verifySignature(req);
|
|
66
|
+
req.session.touched = Date.now();
|
|
67
|
+
req.session.touched = Date.now();
|
|
68
|
+
return next();
|
|
69
|
+
}
|
|
70
|
+
`);
|
|
71
|
+
writeFileSync(join(dir, 'test', 'auth.test.ts'), "import { handle } from '../src/auth';\n");
|
|
72
|
+
writeFileSync(join(dir, 'testguard.claims.json'), JSON.stringify({ schemaVersion: 1, claims: [{ id: 'AUTH-ADMIN', statement: 'Only admins pass.', source: { kind: 'spec', ref: 'x' }, severity: 'critical', producedBy: { producer: 'human' }, defendedBy: ['test/auth.test.ts'], faults: [{ id: 'F1', description: 'd', faultClass: 'other', file: 'src/auth.ts', find: 'next()', replace: 'nope()', producedBy: { producer: 'human' } }] }] }));
|
|
73
|
+
const { doc } = scaffoldFile({ projectDir: dir, file: 'src/auth.ts', toolVersion: 'test' });
|
|
74
|
+
const all = doc.claims.flatMap((c) => c.faults);
|
|
75
|
+
const byClass = (k) => all.filter((f) => f.faultClass === k);
|
|
76
|
+
|
|
77
|
+
it('covers all five shapes', () => {
|
|
78
|
+
expect(byClass('literal-changed').map((f) => f.replace.trim())).toEqual(expect.arrayContaining([
|
|
79
|
+
expect.stringContaining('httpOnly: false'), expect.stringContaining("sameSite: 'none'"), expect.stringContaining('BCRYPT_COST = 1;'), expect.stringContaining('tolerance = 300000;'),
|
|
80
|
+
]));
|
|
81
|
+
expect(byClass('return-altered')[0].replace.trim()).toBe('return true;');
|
|
82
|
+
expect(byClass('statement-deleted').map((f) => f.find.trim())).toEqual(expect.arrayContaining(['if (!req.user) return res.status(401).end();', 'req.session.touched = Date.now();']));
|
|
83
|
+
expect(byClass('condition-forced')[0].replace.trim()).toBe('if (false) {');
|
|
84
|
+
expect(byClass('call-removed')[0].find.trim()).toBe('await verifySignature(req);');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('computes expectHits/occurrence for a duplicated line so both anchors locate', () => {
|
|
88
|
+
const dupes = all.filter((f) => f.find.trim() === 'req.session.touched = Date.now();');
|
|
89
|
+
expect(dupes.map((f) => [f.expectHits, f.occurrence])).toEqual([[2, 1], [2, 2]]);
|
|
90
|
+
const source = readFileSync(join(dir, 'src', 'auth.ts'), 'utf8');
|
|
91
|
+
for (const f of all) expect(locate(source, f).status).toBe('ok');
|
|
92
|
+
expect(validate('claims', doc).ok).toBe(true);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('--claim puts everything under one claim and copies an existing claim\'s statement, severity and defenders', () => {
|
|
96
|
+
const existing = JSON.parse(readFileSync(join(dir, 'testguard.claims.json'), 'utf8'));
|
|
97
|
+
const { doc: one } = scaffoldFile({ projectDir: dir, file: 'src/auth.ts', claimId: 'AUTH-ADMIN', existingClaims: existing, toolVersion: 'test' });
|
|
98
|
+
expect(one.claims).toHaveLength(1);
|
|
99
|
+
expect(one.claims[0]).toMatchObject({ id: 'AUTH-ADMIN', statement: 'Only admins pass.', severity: 'critical', defendedBy: ['test/auth.test.ts'] });
|
|
100
|
+
expect(one.claims[0].faults.length).toBe(all.length);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('CLI writes the draft beside the evidence, prints the shapes, and refuses a missing file', async () => {
|
|
104
|
+
const cwd = process.cwd();
|
|
105
|
+
process.chdir(dir);
|
|
106
|
+
try {
|
|
107
|
+
const a = capture();
|
|
108
|
+
expect(await main(['scaffold', 'src/auth.ts'], a.io)).toBe(0);
|
|
109
|
+
expect(a.lines.out[0]).toMatch(/^\d+ proposed faults in \d+ draft claims for src\/auth\.ts — /);
|
|
110
|
+
expect(existsSync(join(dir, '.testguard', 'scaffold-auth.json'))).toBe(true);
|
|
111
|
+
expect(readFileSync(join(dir, 'testguard.claims.json'), 'utf8')).toContain('"AUTH-ADMIN"'); // untouched
|
|
112
|
+
const b = capture();
|
|
113
|
+
expect(await main(['scaffold', 'src/nope.ts'], b.io)).toBe(2);
|
|
114
|
+
} finally {
|
|
115
|
+
process.chdir(cwd);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
describe('producer edge cases', () => {
|
|
121
|
+
it('recognises function heads and ignores control keywords', () => {
|
|
122
|
+
expect(functionHead('export async function handle(req) {')).toBe('handle');
|
|
123
|
+
expect(functionHead('const isAdmin = (u) => u.admin;')).toBe('isAdmin');
|
|
124
|
+
expect(functionHead(' async verify(token) {')).toBe('verify');
|
|
125
|
+
expect(functionHead(' if (x) {')).toBeNull();
|
|
126
|
+
expect(functionHead(' } catch (e) {')).toBeNull();
|
|
127
|
+
});
|
|
128
|
+
it('does not treat a comparison or a declaration as a mutation, nor a non-guard if as a guard', () => {
|
|
129
|
+
expect(proposalsForLine([' const a = b;'], 0)).toEqual([]);
|
|
130
|
+
expect(proposalsForLine([' if (a === b) {', ' count++;', ' }'], 0)).toEqual([]);
|
|
131
|
+
expect(proposalsForLine([' // if (!x) return;'], 0)).toEqual([]);
|
|
132
|
+
});
|
|
133
|
+
});
|
|
@@ -311,6 +311,37 @@
|
|
|
311
311
|
}
|
|
312
312
|
}
|
|
313
313
|
]
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
"id": "TG-SCAFFOLD-ANCHORS-HIT",
|
|
317
|
+
"statement": "Every fault scaffold proposes carries an expectHits/occurrence computed from the real source, so the anchor is verifiable by construction even when the line is duplicated.",
|
|
318
|
+
"source": {
|
|
319
|
+
"kind": "spec",
|
|
320
|
+
"ref": "issue #6 acceptance"
|
|
321
|
+
},
|
|
322
|
+
"severity": "high",
|
|
323
|
+
"producedBy": {
|
|
324
|
+
"producer": "human",
|
|
325
|
+
"by": "maintainer",
|
|
326
|
+
"at": "2026-09-17T00:00:00Z"
|
|
327
|
+
},
|
|
328
|
+
"defendedBy": [
|
|
329
|
+
"test/scaffold.test.mjs"
|
|
330
|
+
],
|
|
331
|
+
"faults": [
|
|
332
|
+
{
|
|
333
|
+
"id": "F1",
|
|
334
|
+
"description": "expectHits is always declared as 1; a duplicated line becomes anchor-ambiguous.",
|
|
335
|
+
"faultClass": "literal-changed",
|
|
336
|
+
"file": "src/scaffold/scaffold.mjs",
|
|
337
|
+
"find": "...(p.expectHits > 1 ? { expectHits: p.expectHits, occurrence: p.occurrence } : {}),",
|
|
338
|
+
"replace": "...{},",
|
|
339
|
+
"producedBy": {
|
|
340
|
+
"producer": "human",
|
|
341
|
+
"by": "maintainer"
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
]
|
|
314
345
|
}
|
|
315
346
|
]
|
|
316
347
|
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/baseline.zero-count.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/brief.text-without-heading.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/brief.unknown-verdict-key.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/calibration.p-outside-ci.json
RENAMED
|
File without changes
|
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.absolute-path.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.duplicate-fault-id.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.missing-provenance.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/claims.parent-traversal.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.unknown-verdict.json
RENAMED
|
File without changes
|
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/evidence.wrong-fingerprint.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/ignore.missing-reason.json
RENAMED
|
File without changes
|
{testguard_cli-0.1.3 → testguard_cli-0.2.0}/spec/conformance/invalid/ignore.short-reason.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|