agentrust-trace-tests 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/codeql.yml +3 -3
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/contributor-check.yml +4 -1
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/release.yml +1 -1
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/scorecard.yml +2 -2
- agentrust_trace_tests-0.5.0/CHANGELOG.md +52 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/PKG-INFO +30 -3
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/README.md +29 -2
- agentrust_trace_tests-0.5.0/docs/assets/og.png +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/quickstart.md +4 -1
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/mkdocs.yml +27 -0
- agentrust_trace_tests-0.5.0/overrides/main.html +87 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/pyproject.toml +1 -1
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/requirements-docs.txt +1 -0
- agentrust_trace_tests-0.5.0/src/trace_tests/__init__.py +20 -0
- agentrust_trace_tests-0.5.0/src/trace_tests/cli.py +211 -0
- agentrust_trace_tests-0.5.0/src/trace_tests/report.py +364 -0
- agentrust_trace_tests-0.5.0/tests/test_report.py +217 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_cli.py +26 -0
- agentrust_trace_tests-0.4.0/CHANGELOG.md +0 -30
- agentrust_trace_tests-0.4.0/src/trace_tests/__init__.py +0 -3
- agentrust_trace_tests-0.4.0/src/trace_tests/cli.py +0 -98
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/CODEOWNERS +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/dependabot.yml +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/ci.yml +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/docs.yml +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/require-maintainer-approval.yml +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.gitignore +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/CNAME +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/CODE_OF_CONDUCT.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/CONTRIBUTING.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/LICENSE +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/PRIVACY.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/SECURITY.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/error-codes.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/levels.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-anc.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-env.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-pol.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-rte.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-sca.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-sig.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules/tr-txn.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/modules.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/stylesheets/extra.css +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/tutorials/ci-integration.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/docs/tutorials/writing-conformance-tests.md +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/schemas/trace-claim.json +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/loader.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/__init__.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_anc.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_env.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_pol.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_rte.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_sca.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_sig.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/modules/tr_txn.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/result.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/src/trace_tests/runner.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/__init__.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/conftest.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/test_level0.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/test_level0_negative.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/test_level1.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/test_level2.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/test_schema.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/test_software_only_platform.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/__init__.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_loader.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_runner.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_tr_anc.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_tr_env.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_tr_pol.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_tr_rte.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/unit/test_tr_sig.py +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/vectors/invalid_missing_runtime.json +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/vectors/invalid_wrong_profile.json +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/vectors/valid_cmcp_runtime.json +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/vectors/valid_level0.json +0 -0
- {agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/tests/vectors/valid_level0_with_transcript.json +0 -0
|
@@ -28,16 +28,16 @@ jobs:
|
|
|
28
28
|
uses: actions/checkout@v7
|
|
29
29
|
|
|
30
30
|
- name: Initialize CodeQL
|
|
31
|
-
uses: github/codeql-action/init@v4
|
|
31
|
+
uses: github/codeql-action/init@v4.37.4
|
|
32
32
|
with:
|
|
33
33
|
languages: python
|
|
34
34
|
queries: +security-extended
|
|
35
35
|
|
|
36
36
|
- name: Autobuild
|
|
37
|
-
uses: github/codeql-action/autobuild@v4
|
|
37
|
+
uses: github/codeql-action/autobuild@v4.37.4
|
|
38
38
|
|
|
39
39
|
- name: Perform CodeQL Analysis
|
|
40
|
-
uses: github/codeql-action/analyze@v4
|
|
40
|
+
uses: github/codeql-action/analyze@v4.37.4
|
|
41
41
|
continue-on-error: true
|
|
42
42
|
with:
|
|
43
43
|
category: /language:python
|
{agentrust_trace_tests-0.4.0 → agentrust_trace_tests-0.5.0}/.github/workflows/contributor-check.yml
RENAMED
|
@@ -30,7 +30,10 @@ jobs:
|
|
|
30
30
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
31
31
|
with:
|
|
32
32
|
repository: agentrust-io/.github
|
|
33
|
-
|
|
33
|
+
# Pinned deliberately: this workflow runs on pull_request_target
|
|
34
|
+
# with a write-capable token, so the code it checks out must not
|
|
35
|
+
# track a mutable ref. Bump this SHA after reviewing the diff.
|
|
36
|
+
ref: 0fe4520ee71a743fcd33f409fe366ee3e1da6f86 # main as of 2026-08-01
|
|
34
37
|
persist-credentials: false
|
|
35
38
|
sparse-checkout: |
|
|
36
39
|
scripts
|
|
@@ -26,7 +26,7 @@ jobs:
|
|
|
26
26
|
fetch-depth: 0
|
|
27
27
|
|
|
28
28
|
- name: Run Scorecard
|
|
29
|
-
uses: ossf/scorecard-action@
|
|
29
|
+
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
|
30
30
|
with:
|
|
31
31
|
results_file: scorecard-results.sarif
|
|
32
32
|
results_format: sarif
|
|
@@ -35,6 +35,6 @@ jobs:
|
|
|
35
35
|
publish_results: false
|
|
36
36
|
|
|
37
37
|
- name: Upload SARIF
|
|
38
|
-
uses: github/codeql-action/upload-sarif@v4
|
|
38
|
+
uses: github/codeql-action/upload-sarif@v4.37.4
|
|
39
39
|
with:
|
|
40
40
|
sarif_file: scorecard-results.sarif
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## v0.5.0 — 2026-08-09
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **`trace-tests report`: conformance results as an artifact somebody can forward.** `verify` answers a question for whoever ran it; a pass/fail in a terminal is useless to an auditor, a counterparty or an acquirer. The new command emits self-contained HTML, a machine-readable JSON document (`schema: agentrust-io/trace-tests/report/1`), and an SVG level badge.
|
|
10
|
+
|
|
11
|
+
It runs every level up to `--max-level` instead of one, because the answer a reader needs is the highest level the record reaches, not whether it cleared the level the person running the tool happened to choose. `--fail-under N` gates CI on a level; without it the command exits `0`, since producing an artifact and enforcing a threshold are different jobs.
|
|
12
|
+
|
|
13
|
+
**The report states that it is not evidence.** It is unsigned HTML describing one run of one suite version, and anyone can edit it, so every report carries the record's digest, the suite and authoring-library versions, and the command to reproduce the result — and tells a reader who does not trust the sender to go check the record instead. A conformance report that looks authoritative and cannot be checked is the same shape of thing as a control plane writing its own log, which is the problem this project exists to fix.
|
|
14
|
+
|
|
15
|
+
Self-contained by construction: no scripts, no external CSS, no fonts, no badge service. A badge served from someone else's infrastructure would add a dependency to an artifact whose whole point is needing none. A test asserts the HTML fetches nothing.
|
|
16
|
+
|
|
17
|
+
The HTML and the JSON are rendered from one assembled structure so they cannot disagree about the verdict, and unverified findings count as failures from Level 1 up exactly as they do in `verify`.
|
|
18
|
+
|
|
19
|
+
## v0.4.1 — 2026-08-03
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
|
|
23
|
+
- **`--version` reported the wrong version.** `__version__` was a second hardcoded literal alongside `pyproject.toml` and never moved, so it sat at `0.2.0` through both the 0.3.0 and 0.4.0 releases: `trace-tests --version` printed `0.2.0` from a 0.4.0 install while `importlib.metadata` correctly returned `0.4.0`. It is now read from installed distribution metadata, so there is one source of truth and the value cannot fall behind a release again.
|
|
24
|
+
|
|
25
|
+
This mattered more than a wrong string usually would. The v0.2 profile cutover shipped in 0.4.0, and a 0.2.x suite rejects every v0.2 record, so `--version` is exactly the command someone runs to work out whether their suite matches their producer. It was the one command that could not answer.
|
|
26
|
+
|
|
27
|
+
## v0.4.0 — 2026-07-28
|
|
28
|
+
|
|
29
|
+
### Changed
|
|
30
|
+
|
|
31
|
+
- **BREAKING: the suite now conforms to TRACE v0.2.** `TR-ENV` requires the profile `tag:agentrust-io.com,2026:trace-v0.2` and fails a record carrying the v0.1 identifier. The v0.1 URI named `agentrust.io`, a domain this project never controlled, which RFC 4151 does not permit for a tag URI; see agentrust-io/trace-spec#107. Nothing else about the record format changed, so a producer migrates by updating the profile string and bumping `agentrust-trace` to 0.5.0.
|
|
32
|
+
|
|
33
|
+
This is a deliberate cutover rather than dual acceptance: a conformance suite that passed both identifiers would certify records minted under a domain we do not own. A v0.1 record is checked with the 0.3.x releases of this suite, which stay published.
|
|
34
|
+
|
|
35
|
+
- Registry, verifier, and documentation hosts moved from `agentrust.io` to `agentrust-io.com`.
|
|
36
|
+
|
|
37
|
+
## v0.3.0 — 2026-07-21
|
|
38
|
+
|
|
39
|
+
- `azure-cvm-sev-snp` platform accepted (`runtime.platform`): Azure confidential VMs run SEV-SNP behind a Hyper-V paravisor (vTPM-rooted). Added to the bundled schema enum and the TR-RTE valid-platform set so Azure TRACE records pass conformance. Matches `agentrust-trace>=0.4`.
|
|
40
|
+
|
|
41
|
+
## v0.2.0 — 2026-06-19
|
|
42
|
+
|
|
43
|
+
- DID subject support: `subject` now accepts `did:` URIs in addition to `spiffe://`.
|
|
44
|
+
- Embedded signature verification: plain TRACE records signed with `agentrust-trace sign_record()` are now cryptographically verified at all levels.
|
|
45
|
+
- SLSA Level 0: `build_provenance.slsa_level: 0` is now valid for software-only / development records.
|
|
46
|
+
- Software-only platform: `runtime.platform: "software-only"` accepted at Level 0.
|
|
47
|
+
- Private key leak detection: TR-SIG now fails records that embed a private key (`d` member) in `cnf.jwk`.
|
|
48
|
+
|
|
49
|
+
## v0.1.0 — 2026-05-01
|
|
50
|
+
|
|
51
|
+
- Initial release with 7 test modules: TR-ENV, TR-SIG, TR-RTE, TR-POL, TR-TXN, TR-ANC, TR-SCA.
|
|
52
|
+
- Conformance levels 0, 1, 2.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentrust-trace-tests
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: TRACE conformance test suite
|
|
5
5
|
Project-URL: Homepage, https://github.com/agentrust-io/trace-tests
|
|
6
6
|
Project-URL: Repository, https://github.com/agentrust-io/trace-tests
|
|
@@ -64,17 +64,44 @@ Description-Content-Type: text/markdown
|
|
|
64
64
|
|
|
65
65
|
> **Test suite v0.2.** Tracks [TRACE Spec v0.2](https://github.com/agentrust-io/trace-spec).
|
|
66
66
|
|
|
67
|
-
Conformance tests for TRACE
|
|
67
|
+
Conformance tests for TRACE (Trust Runtime Attestation and Compliance Evidence). Run this suite against your implementation to verify it meets the spec before claiming TRACE compliance.
|
|
68
68
|
|
|
69
69
|
Seven test modules covering the full specification: envelope structure, signature algorithms, TEE runtime claims, policy binding, tool-call transcripts, SCITT transparency anchoring, and supply chain provenance.
|
|
70
70
|
|
|
71
71
|
## Quick start
|
|
72
72
|
|
|
73
73
|
```bash
|
|
74
|
-
pip install trace-tests
|
|
74
|
+
pip install agentrust-trace-tests
|
|
75
75
|
trace-tests verify --record path/to/trust-record.jwt --level 1
|
|
76
76
|
```
|
|
77
77
|
|
|
78
|
+
## A report you can hand to someone else
|
|
79
|
+
|
|
80
|
+
`verify` answers a question for the person running it. `report` produces an artifact for
|
|
81
|
+
somebody who was not there: an auditor, a counterparty, an acquirer.
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
trace-tests report --record trust-record.json --html report.html --json report.json --badge trace.svg
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
It runs **every** level up to `--max-level` rather than one, because the useful answer for
|
|
88
|
+
a reader is the highest level the record reaches, not whether it cleared the level someone
|
|
89
|
+
happened to pick. The HTML is self-contained: no scripts, no fonts, no external CSS, no
|
|
90
|
+
badge service, nothing fetched at open time.
|
|
91
|
+
|
|
92
|
+
Use `--fail-under 1` to gate CI on a level. Without it the command always exits `0`, which
|
|
93
|
+
is what you want when you are producing an artifact rather than enforcing a threshold.
|
|
94
|
+
|
|
95
|
+
**The report is not evidence, and it says so on its face.** It is unsigned HTML describing
|
|
96
|
+
one run of one suite version, and anybody can edit it. So it carries the record's digest,
|
|
97
|
+
the suite and library versions, and the exact command to reproduce the result. A reader who
|
|
98
|
+
does not trust the sender is told, in the artifact, to go check the record instead. A
|
|
99
|
+
conformance report that looks authoritative and cannot be checked is the same shape of
|
|
100
|
+
thing as a control plane writing its own log.
|
|
101
|
+
|
|
102
|
+
`report.json` is stable under `schema: agentrust-io/trace-tests/report/1` for dashboards
|
|
103
|
+
and CI.
|
|
104
|
+
|
|
78
105
|
## Test modules
|
|
79
106
|
|
|
80
107
|
| Module | ID | Tests |
|
|
@@ -27,17 +27,44 @@
|
|
|
27
27
|
|
|
28
28
|
> **Test suite v0.2.** Tracks [TRACE Spec v0.2](https://github.com/agentrust-io/trace-spec).
|
|
29
29
|
|
|
30
|
-
Conformance tests for TRACE
|
|
30
|
+
Conformance tests for TRACE (Trust Runtime Attestation and Compliance Evidence). Run this suite against your implementation to verify it meets the spec before claiming TRACE compliance.
|
|
31
31
|
|
|
32
32
|
Seven test modules covering the full specification: envelope structure, signature algorithms, TEE runtime claims, policy binding, tool-call transcripts, SCITT transparency anchoring, and supply chain provenance.
|
|
33
33
|
|
|
34
34
|
## Quick start
|
|
35
35
|
|
|
36
36
|
```bash
|
|
37
|
-
pip install trace-tests
|
|
37
|
+
pip install agentrust-trace-tests
|
|
38
38
|
trace-tests verify --record path/to/trust-record.jwt --level 1
|
|
39
39
|
```
|
|
40
40
|
|
|
41
|
+
## A report you can hand to someone else
|
|
42
|
+
|
|
43
|
+
`verify` answers a question for the person running it. `report` produces an artifact for
|
|
44
|
+
somebody who was not there: an auditor, a counterparty, an acquirer.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
trace-tests report --record trust-record.json --html report.html --json report.json --badge trace.svg
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
It runs **every** level up to `--max-level` rather than one, because the useful answer for
|
|
51
|
+
a reader is the highest level the record reaches, not whether it cleared the level someone
|
|
52
|
+
happened to pick. The HTML is self-contained: no scripts, no fonts, no external CSS, no
|
|
53
|
+
badge service, nothing fetched at open time.
|
|
54
|
+
|
|
55
|
+
Use `--fail-under 1` to gate CI on a level. Without it the command always exits `0`, which
|
|
56
|
+
is what you want when you are producing an artifact rather than enforcing a threshold.
|
|
57
|
+
|
|
58
|
+
**The report is not evidence, and it says so on its face.** It is unsigned HTML describing
|
|
59
|
+
one run of one suite version, and anybody can edit it. So it carries the record's digest,
|
|
60
|
+
the suite and library versions, and the exact command to reproduce the result. A reader who
|
|
61
|
+
does not trust the sender is told, in the artifact, to go check the record instead. A
|
|
62
|
+
conformance report that looks authoritative and cannot be checked is the same shape of
|
|
63
|
+
thing as a control plane writing its own log.
|
|
64
|
+
|
|
65
|
+
`report.json` is stable under `schema: agentrust-io/trace-tests/report/1` for dashboards
|
|
66
|
+
and CI.
|
|
67
|
+
|
|
41
68
|
## Test modules
|
|
42
69
|
|
|
43
70
|
| Module | ID | Tests |
|
|
Binary file
|
|
@@ -3,9 +3,12 @@
|
|
|
3
3
|
## Install
|
|
4
4
|
|
|
5
5
|
```bash
|
|
6
|
-
pip install trace-tests
|
|
6
|
+
pip install agentrust-trace-tests
|
|
7
7
|
```
|
|
8
8
|
|
|
9
|
+
The distribution is `agentrust-trace-tests`; `trace-tests` is the command it
|
|
10
|
+
installs. `pip install trace-tests` returns 404.
|
|
11
|
+
|
|
9
12
|
## Create a sample fixture
|
|
10
13
|
|
|
11
14
|
The test suite runs against a signed TRACE Trust Record. Generate a Level 0 development record with the `agentrust-trace` library:
|
|
@@ -25,6 +25,7 @@ exclude_docs: |
|
|
|
25
25
|
|
|
26
26
|
theme:
|
|
27
27
|
name: material
|
|
28
|
+
custom_dir: overrides
|
|
28
29
|
font: false
|
|
29
30
|
palette:
|
|
30
31
|
- scheme: slate
|
|
@@ -59,6 +60,32 @@ theme:
|
|
|
59
60
|
plugins:
|
|
60
61
|
- search:
|
|
61
62
|
lang: en
|
|
63
|
+
- llmstxt:
|
|
64
|
+
full_output: llms-full.txt
|
|
65
|
+
markdown_description: >-
|
|
66
|
+
TRACE Tests is the conformance suite for TRACE (Trust Runtime
|
|
67
|
+
Attestation and Compliance Evidence), the AgenTrust open standard for
|
|
68
|
+
signed, hardware-attested AI agent governance records. Use it to check
|
|
69
|
+
that your attestation receipts satisfy the specification, at each
|
|
70
|
+
conformance level, before shipping them to production.
|
|
71
|
+
sections:
|
|
72
|
+
Getting started:
|
|
73
|
+
- README.md
|
|
74
|
+
- docs/quickstart.md
|
|
75
|
+
- docs/levels.md
|
|
76
|
+
Test modules:
|
|
77
|
+
- docs/modules.md
|
|
78
|
+
- docs/modules/tr-env.md
|
|
79
|
+
- docs/modules/tr-sig.md
|
|
80
|
+
- docs/modules/tr-rte.md
|
|
81
|
+
- docs/modules/tr-pol.md
|
|
82
|
+
- docs/modules/tr-txn.md
|
|
83
|
+
- docs/modules/tr-anc.md
|
|
84
|
+
- docs/modules/tr-sca.md
|
|
85
|
+
Reference:
|
|
86
|
+
- docs/error-codes.md
|
|
87
|
+
- docs/tutorials/writing-conformance-tests.md
|
|
88
|
+
- docs/tutorials/ci-integration.md
|
|
62
89
|
- minify:
|
|
63
90
|
minify_html: true
|
|
64
91
|
- mkdocstrings:
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
{% extends "base.html" %}
|
|
2
|
+
|
|
3
|
+
{#
|
|
4
|
+
SEO and AEO head additions for the TRACE conformance suite.
|
|
5
|
+
|
|
6
|
+
1. The homepage <title> is overridden. Material falls back to site_name when a
|
|
7
|
+
page has no front-matter title, which left the homepage titled "TRACE
|
|
8
|
+
Tests". Front matter is not an option: docs_dir is the repository root, so
|
|
9
|
+
README.md is also the GitHub landing page and YAML would render as noise
|
|
10
|
+
there.
|
|
11
|
+
|
|
12
|
+
2. Open Graph and Twitter meta, so links produce a card instead of a bare URL,
|
|
13
|
+
plus JSON-LD so answer engines can model what this suite is.
|
|
14
|
+
|
|
15
|
+
Asset paths derive from config.site_url. Do not hardcode them: assets are
|
|
16
|
+
served under /docs/assets/ here because docs_dir is the repository root and
|
|
17
|
+
CI copies docs/ into the build directory. That differs between sibling
|
|
18
|
+
repositories, and a hardcoded path is how cMCP shipped a broken card.
|
|
19
|
+
#}
|
|
20
|
+
|
|
21
|
+
{% block htmltitle %}
|
|
22
|
+
{% if page and page.is_homepage %}
|
|
23
|
+
<title>Verify your TRACE implementation - {{ config.site_name }}</title>
|
|
24
|
+
{% else %}
|
|
25
|
+
{{ super() }}
|
|
26
|
+
{% endif %}
|
|
27
|
+
{% endblock %}
|
|
28
|
+
|
|
29
|
+
{% block extrahead %}
|
|
30
|
+
{{ super() }}
|
|
31
|
+
{% set page_desc = page.meta.description if page and page.meta and page.meta.description else config.site_description %}
|
|
32
|
+
{% set page_url = page.canonical_url if page and page.canonical_url else config.site_url %}
|
|
33
|
+
{% set og_image = config.site_url ~ 'docs/assets/og.png' %}
|
|
34
|
+
{% set social_title = 'Verify your TRACE implementation' %}
|
|
35
|
+
|
|
36
|
+
<meta property="og:type" content="website">
|
|
37
|
+
<meta property="og:site_name" content="{{ config.site_name }}">
|
|
38
|
+
<meta property="og:title" content="{% if page and page.title and not page.is_homepage %}{{ page.title }} - {{ config.site_name }}{% else %}{{ config.site_name }}: {{ social_title }}{% endif %}">
|
|
39
|
+
<meta property="og:description" content="{{ page_desc }}">
|
|
40
|
+
<meta property="og:url" content="{{ page_url }}">
|
|
41
|
+
<meta property="og:image" content="{{ og_image }}">
|
|
42
|
+
<meta property="og:image:width" content="1200">
|
|
43
|
+
<meta property="og:image:height" content="630">
|
|
44
|
+
<meta property="og:image:alt" content="TRACE Tests: the AgenTrust conformance suite">
|
|
45
|
+
<meta property="og:locale" content="en_US">
|
|
46
|
+
|
|
47
|
+
<meta name="twitter:card" content="summary_large_image">
|
|
48
|
+
<meta name="twitter:title" content="{% if page and page.title and not page.is_homepage %}{{ page.title }} - {{ config.site_name }}{% else %}{{ config.site_name }}: {{ social_title }}{% endif %}">
|
|
49
|
+
<meta name="twitter:description" content="{{ page_desc }}">
|
|
50
|
+
<meta name="twitter:image" content="{{ og_image }}">
|
|
51
|
+
|
|
52
|
+
<script type="application/ld+json">
|
|
53
|
+
{
|
|
54
|
+
"@context": "https://schema.org",
|
|
55
|
+
"@graph": [
|
|
56
|
+
{
|
|
57
|
+
"@type": "SoftwareApplication",
|
|
58
|
+
"name": "TRACE Tests",
|
|
59
|
+
"applicationCategory": "DeveloperApplication",
|
|
60
|
+
"description": "Conformance tests and an integration harness for TRACE implementations. Verify that attestation receipts meet the specification before shipping to production.",
|
|
61
|
+
"url": "{{ config.site_url }}",
|
|
62
|
+
"offers": { "@type": "Offer", "price": "0", "priceCurrency": "USD" },
|
|
63
|
+
"author": { "@type": "Organization", "name": "AgenTrust", "url": "https://agentrust-io.com" },
|
|
64
|
+
"isPartOf": { "@type": "WebSite", "name": "{{ config.site_name }}", "url": "{{ config.site_url }}" }
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"@type": "Organization",
|
|
68
|
+
"name": "AgenTrust",
|
|
69
|
+
"url": "https://agentrust-io.com",
|
|
70
|
+
"sameAs": [
|
|
71
|
+
"https://github.com/agentrust-io",
|
|
72
|
+
"https://agentrust-io.com/demos/",
|
|
73
|
+
"https://trace.agentrust-io.com",
|
|
74
|
+
"https://cmcp.agentrust-io.com",
|
|
75
|
+
"https://manifest.agentrust-io.com",
|
|
76
|
+
"https://ca2a.agentrust-io.com"
|
|
77
|
+
]
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
"@type": "WebSite",
|
|
81
|
+
"name": "{{ config.site_name }}",
|
|
82
|
+
"url": "{{ config.site_url }}"
|
|
83
|
+
}
|
|
84
|
+
]
|
|
85
|
+
}
|
|
86
|
+
</script>
|
|
87
|
+
{% endblock %}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""TRACE conformance test suite."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
# Read the version from installed distribution metadata rather than restating it
|
|
6
|
+
# here. `pyproject.toml` is what the build and PyPI publish, so a second literal
|
|
7
|
+
# in this file is a copy that can fall behind silently: it sat at "0.2.0" through
|
|
8
|
+
# both the 0.3.0 and 0.4.0 releases, so `trace-tests --version` reported 0.2.0
|
|
9
|
+
# from a 0.4.0 install. That is worse than cosmetic, because the v0.2 profile
|
|
10
|
+
# cutover landed in 0.4.0 and `--version` is the command someone runs to find out
|
|
11
|
+
# whether they have a suite that accepts v0.2 records.
|
|
12
|
+
try:
|
|
13
|
+
__version__ = version("agentrust-trace-tests")
|
|
14
|
+
except PackageNotFoundError: # pragma: no cover - source tree with no install
|
|
15
|
+
# Importable without being installed (e.g. PYTHONPATH against a checkout).
|
|
16
|
+
# There is no metadata to read here, and guessing a number would reintroduce
|
|
17
|
+
# the drift this exists to prevent.
|
|
18
|
+
__version__ = "0.0.0+unknown"
|
|
19
|
+
|
|
20
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""trace-tests CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as _dt
|
|
6
|
+
import importlib.metadata
|
|
7
|
+
import pathlib
|
|
8
|
+
import sys
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import click
|
|
12
|
+
|
|
13
|
+
from trace_tests import __version__
|
|
14
|
+
from trace_tests import report as report_mod
|
|
15
|
+
from trace_tests.loader import LoadError, load_record
|
|
16
|
+
from trace_tests.modules.tr_env import DEFAULT_MAX_AGE_SECONDS
|
|
17
|
+
from trace_tests.result import Status
|
|
18
|
+
from trace_tests.runner import run
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _library_version() -> str | None:
|
|
22
|
+
"""Version of the authoring library, when it is installed alongside the suite.
|
|
23
|
+
|
|
24
|
+
Recorded on the report because the two cut over to the v0.2 profile together
|
|
25
|
+
and neither dual-accepts, so a mismatched pair produces a confident failure on
|
|
26
|
+
a record that is fine. A reader of the report should be able to see the pair.
|
|
27
|
+
"""
|
|
28
|
+
try:
|
|
29
|
+
return importlib.metadata.version("agentrust-trace")
|
|
30
|
+
except importlib.metadata.PackageNotFoundError:
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _fmt_status(status: Status) -> str:
|
|
35
|
+
return status.value.upper().ljust(10)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _print_report(path: str, fmt: str, level: int, results: dict[str, list[Any]]) -> int:
|
|
39
|
+
"""Print the conformance report and return exit code (0=pass, 1=fail)."""
|
|
40
|
+
click.echo(f"\nTRACE Conformance Report -- Level {level}")
|
|
41
|
+
click.echo(f"Format : {fmt}")
|
|
42
|
+
click.echo(f"Record : {path}")
|
|
43
|
+
click.echo("")
|
|
44
|
+
|
|
45
|
+
failures = 0
|
|
46
|
+
skips = 0
|
|
47
|
+
passes = 0
|
|
48
|
+
unverified = 0
|
|
49
|
+
|
|
50
|
+
for module, findings in results.items():
|
|
51
|
+
for f in findings:
|
|
52
|
+
prefix = _fmt_status(f.status)
|
|
53
|
+
click.echo(f" {module} {prefix} {f.message}")
|
|
54
|
+
if f.failed():
|
|
55
|
+
failures += 1
|
|
56
|
+
elif f.passed():
|
|
57
|
+
passes += 1
|
|
58
|
+
elif f.unverified():
|
|
59
|
+
unverified += 1
|
|
60
|
+
else:
|
|
61
|
+
skips += 1
|
|
62
|
+
|
|
63
|
+
# Defense in depth: unverified findings must fail the run at any level that
|
|
64
|
+
# requires signatures, even if a module forgot to emit a hard FAIL.
|
|
65
|
+
if unverified and level >= 1:
|
|
66
|
+
failures += unverified
|
|
67
|
+
|
|
68
|
+
total = passes + failures + skips + (unverified if level == 0 else 0)
|
|
69
|
+
click.echo("")
|
|
70
|
+
if failures == 0:
|
|
71
|
+
if unverified:
|
|
72
|
+
click.echo(
|
|
73
|
+
f"Result: PASS ({total} checks, {skips} skipped, {unverified} UNVERIFIED "
|
|
74
|
+
f"-- record is NOT cryptographically verified)"
|
|
75
|
+
)
|
|
76
|
+
else:
|
|
77
|
+
click.echo(f"Result: PASS ({total} checks, {skips} skipped)")
|
|
78
|
+
return 0
|
|
79
|
+
else:
|
|
80
|
+
click.echo(f"Result: FAIL ({total} checks, {failures} failure(s), {skips} skipped)")
|
|
81
|
+
return 1
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@click.group()
|
|
85
|
+
@click.version_option(__version__)
|
|
86
|
+
def main() -> None:
|
|
87
|
+
"""TRACE conformance test suite."""
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@main.command()
|
|
91
|
+
@click.option("--record", required=True, type=click.Path(), help="Path to the trust record (JSON)")
|
|
92
|
+
@click.option("--level", default=0, type=click.IntRange(0, 2), show_default=True, help="Conformance level to check (0, 1, or 2)")
|
|
93
|
+
@click.option(
|
|
94
|
+
"--max-age",
|
|
95
|
+
"max_age",
|
|
96
|
+
default=DEFAULT_MAX_AGE_SECONDS,
|
|
97
|
+
type=click.IntRange(min=1),
|
|
98
|
+
show_default=True,
|
|
99
|
+
help="Maximum allowed record age in seconds (iat freshness window)",
|
|
100
|
+
)
|
|
101
|
+
def verify(record: str, level: int, max_age: int) -> None:
|
|
102
|
+
"""Verify a TRACE trust record against the conformance suite."""
|
|
103
|
+
try:
|
|
104
|
+
data, fmt = load_record(record)
|
|
105
|
+
except LoadError as exc:
|
|
106
|
+
click.echo(f"Error: {exc}", err=True)
|
|
107
|
+
sys.exit(2)
|
|
108
|
+
|
|
109
|
+
results = run(data, fmt, level, max_age_seconds=max_age)
|
|
110
|
+
exit_code = _print_report(record, fmt, level, results)
|
|
111
|
+
sys.exit(exit_code)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@main.command()
|
|
115
|
+
@click.option("--record", required=True, type=click.Path(), help="Path to the trust record (JSON)")
|
|
116
|
+
@click.option(
|
|
117
|
+
"--max-level",
|
|
118
|
+
default=2,
|
|
119
|
+
type=click.IntRange(0, 2),
|
|
120
|
+
show_default=True,
|
|
121
|
+
help="Highest level to attempt; every level up to it is run",
|
|
122
|
+
)
|
|
123
|
+
@click.option(
|
|
124
|
+
"--max-age",
|
|
125
|
+
"max_age",
|
|
126
|
+
default=DEFAULT_MAX_AGE_SECONDS,
|
|
127
|
+
type=click.IntRange(min=1),
|
|
128
|
+
show_default=True,
|
|
129
|
+
)
|
|
130
|
+
@click.option(
|
|
131
|
+
"--html", "html_out", type=click.Path(), help="Write a self-contained HTML report here"
|
|
132
|
+
)
|
|
133
|
+
@click.option(
|
|
134
|
+
"--json", "json_out", type=click.Path(), help="Write the machine-readable report here"
|
|
135
|
+
)
|
|
136
|
+
@click.option("--badge", "badge_out", type=click.Path(), help="Write a level badge (SVG) here")
|
|
137
|
+
@click.option(
|
|
138
|
+
"--fail-under",
|
|
139
|
+
type=click.IntRange(0, 2),
|
|
140
|
+
help="Exit non-zero unless the record reaches this level. Omit to always exit 0, "
|
|
141
|
+
"which is what you want when generating an artifact rather than gating on one.",
|
|
142
|
+
)
|
|
143
|
+
def report(
|
|
144
|
+
record: str,
|
|
145
|
+
max_level: int,
|
|
146
|
+
max_age: int,
|
|
147
|
+
html_out: str | None,
|
|
148
|
+
json_out: str | None,
|
|
149
|
+
badge_out: str | None,
|
|
150
|
+
fail_under: int | None,
|
|
151
|
+
) -> None:
|
|
152
|
+
"""Produce a conformance report you can hand to someone else.
|
|
153
|
+
|
|
154
|
+
Runs every level up to --max-level rather than one, because the useful answer
|
|
155
|
+
for a reader is the highest level the record reaches, not whether it cleared
|
|
156
|
+
the single level the person running the tool happened to pick.
|
|
157
|
+
"""
|
|
158
|
+
try:
|
|
159
|
+
data, fmt = load_record(record)
|
|
160
|
+
except LoadError as exc:
|
|
161
|
+
click.echo(f"Error: {exc}", err=True)
|
|
162
|
+
sys.exit(2)
|
|
163
|
+
|
|
164
|
+
results_by_level = {
|
|
165
|
+
level: run(data, fmt, level, max_age_seconds=max_age)
|
|
166
|
+
for level in range(max_level + 1)
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
built = report_mod.build(
|
|
170
|
+
record=data,
|
|
171
|
+
record_path=record,
|
|
172
|
+
record_format=fmt,
|
|
173
|
+
results_by_level=results_by_level,
|
|
174
|
+
suite_version=__version__,
|
|
175
|
+
library_version=_library_version(),
|
|
176
|
+
generated_at=_dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
wrote = []
|
|
180
|
+
for path, render in (
|
|
181
|
+
(html_out, report_mod.to_html),
|
|
182
|
+
(json_out, report_mod.to_json),
|
|
183
|
+
(badge_out, report_mod.badge_svg),
|
|
184
|
+
):
|
|
185
|
+
if path:
|
|
186
|
+
pathlib.Path(path).write_text(render(built), encoding="utf-8")
|
|
187
|
+
wrote.append(path)
|
|
188
|
+
|
|
189
|
+
click.echo(f"TRACE Conformance Report -- {built.verdict}")
|
|
190
|
+
click.echo(f"Record : {record}")
|
|
191
|
+
click.echo(f"Digest : {built.digest}")
|
|
192
|
+
for lv in built.levels:
|
|
193
|
+
mark = "PASS" if lv.passed else "FAIL"
|
|
194
|
+
click.echo(f" Level {lv.level} {mark}")
|
|
195
|
+
for path in wrote:
|
|
196
|
+
click.echo(f"Wrote : {path}")
|
|
197
|
+
if not wrote:
|
|
198
|
+
click.echo("")
|
|
199
|
+
click.echo(report_mod.to_json(built))
|
|
200
|
+
|
|
201
|
+
if fail_under is not None:
|
|
202
|
+
top = built.highest_level
|
|
203
|
+
if top is None or top < fail_under:
|
|
204
|
+
click.echo(
|
|
205
|
+
f"Result: below the required Level {fail_under}", err=True
|
|
206
|
+
)
|
|
207
|
+
sys.exit(1)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
if __name__ == "__main__":
|
|
211
|
+
main()
|