agentmandate 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentmandate-0.2.0 → agentmandate-0.3.0}/CHANGELOG.md +42 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/DESIGN.md +19 -1
- {agentmandate-0.2.0 → agentmandate-0.3.0}/PKG-INFO +23 -4
- {agentmandate-0.2.0 → agentmandate-0.3.0}/README.md +22 -3
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/__init__.py +17 -1
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/cli.py +47 -0
- agentmandate-0.3.0/agentmandate/obligations.py +360 -0
- agentmandate-0.3.0/docs/test-obligations.md +117 -0
- agentmandate-0.3.0/examples/reviewed-obligations.json +36 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/pyproject.toml +1 -1
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_cli.py +55 -0
- agentmandate-0.3.0/tests/test_obligations.py +356 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/.github/workflows/ci.yml +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/.github/workflows/release.yml +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/.gitignore +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/CONTRIBUTING.md +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/LICENSE +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/RELEASING.md +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/ROADMAP.md +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/SECURITY.md +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/STABILITY.md +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/diff.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/lint.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/manifest.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/py.typed +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/reach.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/scan.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/agentmandate/verify.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/docs/assets/authority-path.svg +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/docs/manifest.md +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/examples/dispute-resolver-sod.yaml +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/examples/dispute-resolver-v2.yaml +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/examples/dispute-resolver.yaml +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/examples/mcp-tools.json +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/examples/observed-calls.jsonl +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_diff.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_lint.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_manifest.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_reach.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_scan.py +0 -0
- {agentmandate-0.2.0 → agentmandate-0.3.0}/tests/test_verify.py +0 -0
|
@@ -6,6 +6,48 @@ All notable changes to this project are documented here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## Unreleased
|
|
8
8
|
|
|
9
|
+
## 0.3.0 - 2026-07-28
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `mandate obligations` derives reviewable test obligations from the authority
|
|
14
|
+
a manifest actually makes reachable: irreversible effects, approval gates,
|
|
15
|
+
service-account principals, and value-bearing calls. A tool nobody can reach
|
|
16
|
+
produces no obligation, because listing it would pad a review with work that
|
|
17
|
+
protects nothing.
|
|
18
|
+
- `--reviewed` accepts an obligations file whose decisions have been mapped by
|
|
19
|
+
a human, and `--suite` renders those into an `agentverity.decision-suite/v1`
|
|
20
|
+
skeleton that AgentVerity loads directly.
|
|
21
|
+
- `docs/test-obligations.md` walks the whole path, including what deliberately
|
|
22
|
+
does not cross it.
|
|
23
|
+
|
|
24
|
+
- `README` and `DESIGN` name Policy in Amazon Bedrock AgentCore alongside the
|
|
25
|
+
other enforcement tools, and say precisely where it stops: the engine
|
|
26
|
+
evaluates all applicable Cedar policies for one invocation, while its
|
|
27
|
+
documented analysis is policy-level. Neither models a sequence of permitted
|
|
28
|
+
calls or a release-to-release comparison of reachable authority.
|
|
29
|
+
|
|
30
|
+
### Fixed
|
|
31
|
+
|
|
32
|
+
- `--reviewed` reconciles against the live manifest by stable identifier
|
|
33
|
+
instead of replacing it. A stale or unrelated review previously generated a
|
|
34
|
+
suite for authority the agent no longer had, which is the drift this package
|
|
35
|
+
exists to catch.
|
|
36
|
+
- A generated suite carries probes the reviewer wrote. It previously shipped
|
|
37
|
+
`REVIEW:` placeholder inputs that AgentVerity accepted and ran, producing
|
|
38
|
+
numbers about nothing. An obligation now needs both a decision and at least
|
|
39
|
+
one probe before it counts as reviewed.
|
|
40
|
+
- A malformed reviewed file produces a usage error rather than a traceback.
|
|
41
|
+
|
|
42
|
+
### Notes
|
|
43
|
+
|
|
44
|
+
- Decisions are never invented. An effect class such as `irreversible on case`
|
|
45
|
+
is an authority fact; a decision such as `refund_approved` is an application
|
|
46
|
+
label somebody chose. No parsing turns one into the other, so the command
|
|
47
|
+
exits non-zero until every row carries a reviewed decision.
|
|
48
|
+
- Compound breaches do not cross the bridge. A cumulative-value path is a
|
|
49
|
+
multi-call sequence, which is scenario testing rather than decision coverage.
|
|
50
|
+
|
|
9
51
|
## 0.2.0 - 2026-07-28
|
|
10
52
|
|
|
11
53
|
### Added
|
|
@@ -110,13 +110,31 @@ usage error.
|
|
|
110
110
|
|
|
111
111
|
## Deliberate overlap
|
|
112
112
|
|
|
113
|
-
`lint` covers ground that AgentWard, AgentShield, and
|
|
113
|
+
`lint` covers ground that AgentWard, AgentShield, AgentGuard, and Policy in
|
|
114
|
+
Amazon Bedrock AgentCore already cover.
|
|
114
115
|
That is on purpose. A tool that reported only compound findings would need one
|
|
115
116
|
of those running alongside it to be usable at all, and the first thing anyone
|
|
116
117
|
does with a new analysis tool is run it on its own.
|
|
117
118
|
|
|
118
119
|
The overlap is the floor. The contribution is `reach` and the diff built on it.
|
|
119
120
|
|
|
121
|
+
Policy in AgentCore is worth naming precisely, because it is the strongest
|
|
122
|
+
reason an AWS team would ask why this exists. It evaluates all applicable
|
|
123
|
+
Cedar policies for every gateway tool invocation with default-deny and
|
|
124
|
+
forbid-wins semantics. Its documented automated analysis flags policy-level
|
|
125
|
+
problems such as policies that always allow or always deny. That is real
|
|
126
|
+
analysis, but it does not model a sequence of individually permitted calls.
|
|
127
|
+
|
|
128
|
+
Cedar has no notion of a call sequence. "May this principal invoke
|
|
129
|
+
`issue_refund` now" is a different question from "do four individually
|
|
130
|
+
permitted calls compose into a 1,000 GBP breach", and no per-decision engine
|
|
131
|
+
answers the second by construction. Neither does any of them answer whether a
|
|
132
|
+
release widened reachable authority, because that needs two manifests and a
|
|
133
|
+
reachability computation over both.
|
|
134
|
+
|
|
135
|
+
So the split is enforcement against offline analysis, and per-decision against
|
|
136
|
+
compound. Running both is the intended shape.
|
|
137
|
+
|
|
120
138
|
## What was left out, and why
|
|
121
139
|
|
|
122
140
|
**Data-flow reachability.** Finding that a read tool feeds an exfiltration path
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agentmandate
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Compound-path and cross-release analysis of what an AI agent is permitted to do
|
|
5
5
|
Project-URL: Homepage, https://github.com/mrwersa/agentmandate
|
|
6
6
|
Project-URL: Repository, https://github.com/mrwersa/agentmandate
|
|
@@ -64,6 +64,8 @@ cannot:
|
|
|
64
64
|
authority, not configuration text.
|
|
65
65
|
- **Does runtime evidence match the declaration?** `mandate verify` fails
|
|
66
66
|
closed when a required control field is absent.
|
|
67
|
+
- **What must the tests exercise?** `mandate obligations` turns reachable
|
|
68
|
+
authority into reviewable test obligations.
|
|
67
69
|
|
|
68
70
|
Alpha. Apache-2.0.
|
|
69
71
|
|
|
@@ -200,6 +202,7 @@ A ceiling is the maximum **cumulative** value one tool may spend against one bin
|
|
|
200
202
|
| `mandate reach` | Bounded search for a legal call sequence that breaches a limit, reported as a counterexample |
|
|
201
203
|
| `mandate diff` | Effective-authority comparison of two manifests, including limits, preconditions, approvals, effects, and scope minting. `--record` emits a change record |
|
|
202
204
|
| `mandate verify` | Replays recorded tool calls against the manifest and fails closed when evidence required by a declared control is missing |
|
|
205
|
+
| `mandate obligations` | Derives reviewable test obligations from reachable authority, and renders reviewed ones as an [AgentVerity](https://github.com/mrwersa/agentverity) decision suite |
|
|
203
206
|
|
|
204
207
|
Every analysis command takes `--json` and exits non-zero on a finding, so they drop into CI unchanged. `scan` writes a manifest to standard output and is a one-off, not a gate.
|
|
205
208
|
|
|
@@ -228,9 +231,25 @@ does not pass as an empty value.
|
|
|
228
231
|
|
|
229
232
|
This is analysis, not enforcement. It runs in CI against a manifest, it does not sit in the request path.
|
|
230
233
|
|
|
231
|
-
|
|
234
|
+
| Tool | What it does | Relationship |
|
|
235
|
+
|---|---|---|
|
|
236
|
+
| [Policy in Amazon Bedrock AgentCore](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy.html) | Evaluates all applicable Cedar policies for each gateway tool invocation, with default-deny, forbid-wins, and analysis that flags always-allow and always-deny policies | Enforces each invocation. Its documented analysis is policy-level, not a model of a sequence of permitted calls |
|
|
237
|
+
| [AgentWard](https://github.com/agentward-ai/agentward) | Runtime proxy enforcing policy per call, diffs two policy files | Enforces. Diffs declared text rather than reachable authority |
|
|
238
|
+
| [AgentShield](https://github.com/affaan-m/agentshield) | Scans agent configuration and MCP servers, drift gate over findings | Scans. Drift is over finding counts, not permission direction |
|
|
239
|
+
| [AgentGuard](https://github.com/WhitzardAgent/AgentGuard) | Attribute-based access control for tool calls | Enforces |
|
|
240
|
+
| [OPA](https://www.openpolicyagent.org/docs), [Cedar](https://docs.cedarpolicy.com/) | Decide one authorisation at a time | Enforces |
|
|
232
241
|
|
|
233
|
-
Use those to enforce. AgentMandate
|
|
242
|
+
Use those to enforce. AgentMandate is the offline half: it analyses sequences of individually permitted calls and compares *effective* authority across releases.
|
|
243
|
+
|
|
244
|
+
**If you already run AgentCore Policy**, the gap is specific. The policy engine
|
|
245
|
+
answers "may this principal invoke this tool now" by evaluating all applicable
|
|
246
|
+
policies, and its documented analysis catches policy-level problems such as an
|
|
247
|
+
unconditional allow. It does not model whether four separately permitted calls
|
|
248
|
+
compose into a 1,000 GBP breach or whether a release widened what the agent can
|
|
249
|
+
reach. AgentMandate is vendor-neutral and runs in CI before deployment, so it
|
|
250
|
+
complements the gateway rather than duplicating it.
|
|
251
|
+
|
|
252
|
+
The closest prior art in a neighbouring domain is [IAM Access Analyzer](https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-concepts.html), which derives reachable access from policy by automated reasoning rather than waiting for a log event. This is that idea pointed at agent tool graphs.
|
|
234
253
|
|
|
235
254
|
The `lint` command deliberately overlaps the scanners above. A tool that reported only compound findings would need one of them running alongside it to be usable at all.
|
|
236
255
|
|
|
@@ -267,7 +286,7 @@ ruff check .
|
|
|
267
286
|
|
|
268
287
|
## Status
|
|
269
288
|
|
|
270
|
-
Alpha, version 0.
|
|
289
|
+
Alpha, version 0.3.0. The authority model is the part most likely to change,
|
|
271
290
|
because it has not yet been pointed at enough real tool graphs to know where it
|
|
272
291
|
is too coarse. Issues describing a graph it models badly are the most useful
|
|
273
292
|
thing you can file.
|
|
@@ -30,6 +30,8 @@ cannot:
|
|
|
30
30
|
authority, not configuration text.
|
|
31
31
|
- **Does runtime evidence match the declaration?** `mandate verify` fails
|
|
32
32
|
closed when a required control field is absent.
|
|
33
|
+
- **What must the tests exercise?** `mandate obligations` turns reachable
|
|
34
|
+
authority into reviewable test obligations.
|
|
33
35
|
|
|
34
36
|
Alpha. Apache-2.0.
|
|
35
37
|
|
|
@@ -166,6 +168,7 @@ A ceiling is the maximum **cumulative** value one tool may spend against one bin
|
|
|
166
168
|
| `mandate reach` | Bounded search for a legal call sequence that breaches a limit, reported as a counterexample |
|
|
167
169
|
| `mandate diff` | Effective-authority comparison of two manifests, including limits, preconditions, approvals, effects, and scope minting. `--record` emits a change record |
|
|
168
170
|
| `mandate verify` | Replays recorded tool calls against the manifest and fails closed when evidence required by a declared control is missing |
|
|
171
|
+
| `mandate obligations` | Derives reviewable test obligations from reachable authority, and renders reviewed ones as an [AgentVerity](https://github.com/mrwersa/agentverity) decision suite |
|
|
169
172
|
|
|
170
173
|
Every analysis command takes `--json` and exits non-zero on a finding, so they drop into CI unchanged. `scan` writes a manifest to standard output and is a one-off, not a gate.
|
|
171
174
|
|
|
@@ -194,9 +197,25 @@ does not pass as an empty value.
|
|
|
194
197
|
|
|
195
198
|
This is analysis, not enforcement. It runs in CI against a manifest, it does not sit in the request path.
|
|
196
199
|
|
|
197
|
-
|
|
200
|
+
| Tool | What it does | Relationship |
|
|
201
|
+
|---|---|---|
|
|
202
|
+
| [Policy in Amazon Bedrock AgentCore](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy.html) | Evaluates all applicable Cedar policies for each gateway tool invocation, with default-deny, forbid-wins, and analysis that flags always-allow and always-deny policies | Enforces each invocation. Its documented analysis is policy-level, not a model of a sequence of permitted calls |
|
|
203
|
+
| [AgentWard](https://github.com/agentward-ai/agentward) | Runtime proxy enforcing policy per call, diffs two policy files | Enforces. Diffs declared text rather than reachable authority |
|
|
204
|
+
| [AgentShield](https://github.com/affaan-m/agentshield) | Scans agent configuration and MCP servers, drift gate over findings | Scans. Drift is over finding counts, not permission direction |
|
|
205
|
+
| [AgentGuard](https://github.com/WhitzardAgent/AgentGuard) | Attribute-based access control for tool calls | Enforces |
|
|
206
|
+
| [OPA](https://www.openpolicyagent.org/docs), [Cedar](https://docs.cedarpolicy.com/) | Decide one authorisation at a time | Enforces |
|
|
198
207
|
|
|
199
|
-
Use those to enforce. AgentMandate
|
|
208
|
+
Use those to enforce. AgentMandate is the offline half: it analyses sequences of individually permitted calls and compares *effective* authority across releases.
|
|
209
|
+
|
|
210
|
+
**If you already run AgentCore Policy**, the gap is specific. The policy engine
|
|
211
|
+
answers "may this principal invoke this tool now" by evaluating all applicable
|
|
212
|
+
policies, and its documented analysis catches policy-level problems such as an
|
|
213
|
+
unconditional allow. It does not model whether four separately permitted calls
|
|
214
|
+
compose into a 1,000 GBP breach or whether a release widened what the agent can
|
|
215
|
+
reach. AgentMandate is vendor-neutral and runs in CI before deployment, so it
|
|
216
|
+
complements the gateway rather than duplicating it.
|
|
217
|
+
|
|
218
|
+
The closest prior art in a neighbouring domain is [IAM Access Analyzer](https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-concepts.html), which derives reachable access from policy by automated reasoning rather than waiting for a log event. This is that idea pointed at agent tool graphs.
|
|
200
219
|
|
|
201
220
|
The `lint` command deliberately overlaps the scanners above. A tool that reported only compound findings would need one of them running alongside it to be usable at all.
|
|
202
221
|
|
|
@@ -233,7 +252,7 @@ ruff check .
|
|
|
233
252
|
|
|
234
253
|
## Status
|
|
235
254
|
|
|
236
|
-
Alpha, version 0.
|
|
255
|
+
Alpha, version 0.3.0. The authority model is the part most likely to change,
|
|
237
256
|
because it has not yet been pointed at enough real tool graphs to know where it
|
|
238
257
|
is too coarse. Issues describing a graph it models badly are the most useful
|
|
239
258
|
thing you can file.
|
|
@@ -13,11 +13,20 @@ against the declaration, so it is usable without another tool alongside it.
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
-
__version__ = "0.
|
|
16
|
+
__version__ = "0.3.0"
|
|
17
17
|
|
|
18
18
|
from .diff import Change, Delta, compare
|
|
19
19
|
from .lint import Finding, check
|
|
20
20
|
from .manifest import Mandate, ManifestError, Money, Tool, load, loads
|
|
21
|
+
from .obligations import (
|
|
22
|
+
OBLIGATIONS_SCHEMA,
|
|
23
|
+
Obligation,
|
|
24
|
+
ObligationSet,
|
|
25
|
+
derive,
|
|
26
|
+
load_obligations,
|
|
27
|
+
save_obligations,
|
|
28
|
+
to_decision_suite,
|
|
29
|
+
)
|
|
21
30
|
from .reach import Authority, Breach, Step, analyse
|
|
22
31
|
from .scan import Proposal, propose, render, scan_file
|
|
23
32
|
from .verify import Conformance, Observation, Violation, replay, replay_file
|
|
@@ -31,7 +40,10 @@ __all__ = [
|
|
|
31
40
|
"Finding",
|
|
32
41
|
"Mandate",
|
|
33
42
|
"ManifestError",
|
|
43
|
+
"OBLIGATIONS_SCHEMA",
|
|
34
44
|
"Money",
|
|
45
|
+
"Obligation",
|
|
46
|
+
"ObligationSet",
|
|
35
47
|
"Observation",
|
|
36
48
|
"Proposal",
|
|
37
49
|
"Step",
|
|
@@ -40,9 +52,13 @@ __all__ = [
|
|
|
40
52
|
"__version__",
|
|
41
53
|
"analyse",
|
|
42
54
|
"check",
|
|
55
|
+
"derive",
|
|
43
56
|
"compare",
|
|
44
57
|
"load",
|
|
58
|
+
"load_obligations",
|
|
45
59
|
"loads",
|
|
60
|
+
"save_obligations",
|
|
61
|
+
"to_decision_suite",
|
|
46
62
|
"propose",
|
|
47
63
|
"render",
|
|
48
64
|
"replay",
|
|
@@ -11,6 +11,12 @@ from . import __version__
|
|
|
11
11
|
from .diff import compare
|
|
12
12
|
from .lint import ERROR, check
|
|
13
13
|
from .manifest import ManifestError, load
|
|
14
|
+
from .obligations import (
|
|
15
|
+
derive,
|
|
16
|
+
load_obligations,
|
|
17
|
+
reconcile,
|
|
18
|
+
to_decision_suite,
|
|
19
|
+
)
|
|
14
20
|
from .reach import analyse
|
|
15
21
|
from .scan import scan_file
|
|
16
22
|
from .verify import replay_file
|
|
@@ -74,6 +80,26 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
74
80
|
help="emit a markdown change record for a change advisory board",
|
|
75
81
|
)
|
|
76
82
|
|
|
83
|
+
obligations_parser = subparsers.add_parser(
|
|
84
|
+
"obligations",
|
|
85
|
+
help="derive reviewable test obligations from reachable authority",
|
|
86
|
+
)
|
|
87
|
+
_add_manifest(obligations_parser)
|
|
88
|
+
obligations_parser.add_argument("--depth", type=_positive_int, default=None)
|
|
89
|
+
obligations_parser.add_argument(
|
|
90
|
+
"--json", action="store_true", help="machine-readable output"
|
|
91
|
+
)
|
|
92
|
+
obligations_parser.add_argument(
|
|
93
|
+
"--suite",
|
|
94
|
+
action="store_true",
|
|
95
|
+
help="render reviewed obligations as an AgentVerity decision suite",
|
|
96
|
+
)
|
|
97
|
+
obligations_parser.add_argument(
|
|
98
|
+
"--reviewed",
|
|
99
|
+
default=None,
|
|
100
|
+
help="an obligations file whose decisions have been filled in",
|
|
101
|
+
)
|
|
102
|
+
|
|
77
103
|
verify_parser = subparsers.add_parser(
|
|
78
104
|
"verify", help="replay recorded calls against the manifest"
|
|
79
105
|
)
|
|
@@ -124,6 +150,27 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
124
150
|
print(f"error: {exc}", file=sys.stderr)
|
|
125
151
|
return EXIT_USAGE
|
|
126
152
|
|
|
153
|
+
if args.command == "obligations":
|
|
154
|
+
obligations = derive(mandate, depth=args.depth)
|
|
155
|
+
if args.reviewed:
|
|
156
|
+
# Reconciled against the live manifest rather than trusted on its
|
|
157
|
+
# own. A stale review would otherwise generate a suite for
|
|
158
|
+
# authority the agent no longer has.
|
|
159
|
+
try:
|
|
160
|
+
obligations = reconcile(obligations, load_obligations(args.reviewed))
|
|
161
|
+
except ValueError as exc:
|
|
162
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
163
|
+
return EXIT_USAGE
|
|
164
|
+
if args.suite:
|
|
165
|
+
try:
|
|
166
|
+
print(json.dumps(to_decision_suite(obligations), indent=2))
|
|
167
|
+
except ValueError as exc:
|
|
168
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
169
|
+
return EXIT_FINDING
|
|
170
|
+
return EXIT_OK
|
|
171
|
+
_emit(obligations.to_dict(), args.json, obligations.render())
|
|
172
|
+
return EXIT_FINDING if obligations.unreviewed else EXIT_OK
|
|
173
|
+
|
|
127
174
|
if args.command == "lint":
|
|
128
175
|
findings = check(mandate)
|
|
129
176
|
payload = {
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
"""Turn reachable authority into reviewable test obligations.
|
|
2
|
+
|
|
3
|
+
A mandate says what an agent is permitted to do. A test suite says what was
|
|
4
|
+
actually exercised. Nothing connects them today, so a manifest can declare an
|
|
5
|
+
irreversible refund the tests never approach and both artefacts look healthy.
|
|
6
|
+
|
|
7
|
+
This emits an obligation per consequential thing the agent can reach: a
|
|
8
|
+
statement that some reviewed test case ought to exercise this decision point.
|
|
9
|
+
|
|
10
|
+
What it deliberately does not do:
|
|
11
|
+
|
|
12
|
+
- It does not generate decision labels. An effect class such as
|
|
13
|
+
``irreversible on case`` is an authority fact. A decision such as
|
|
14
|
+
``card_security`` is an application label chosen by whoever designed the
|
|
15
|
+
router. No parsing turns one into the other, and inventing the mapping would
|
|
16
|
+
be a confident guess about somebody else's domain.
|
|
17
|
+
- It does not emit compound breaches. A cumulative-value path is a multi-call
|
|
18
|
+
sequence, which is scenario testing rather than decision coverage. Those
|
|
19
|
+
stay in ``reach`` and do not cross into a test suite.
|
|
20
|
+
|
|
21
|
+
So the output is a skeleton with a blank ``decision`` on every row, for a human
|
|
22
|
+
to fill in. Extract then annotate, the same discipline ``scan`` already follows.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Any
|
|
31
|
+
|
|
32
|
+
from .manifest import IRREVERSIBLE, SERVICE, Mandate
|
|
33
|
+
from .reach import analyse
|
|
34
|
+
|
|
35
|
+
OBLIGATIONS_SCHEMA = "agentmandate.obligations/v1"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class Obligation:
|
|
40
|
+
"""One reachable authority fact a reviewed test ought to exercise."""
|
|
41
|
+
|
|
42
|
+
kind: str
|
|
43
|
+
subject: str
|
|
44
|
+
reason: str
|
|
45
|
+
decision: str = ""
|
|
46
|
+
# Probe texts a reviewer wrote to reach this decision. A generated suite
|
|
47
|
+
# needs real inputs, and inventing them would produce a file that runs and
|
|
48
|
+
# measures nothing.
|
|
49
|
+
cases: tuple[str, ...] = ()
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def identifier(self) -> str:
|
|
53
|
+
"""A stable name, so a reviewed mapping survives regeneration."""
|
|
54
|
+
return f"{self.kind}:{self.subject}"
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def reviewed(self) -> bool:
|
|
58
|
+
"""Whether a human has supplied both the decision and a probe.
|
|
59
|
+
|
|
60
|
+
A decision without a case is half a review. The suite it generates
|
|
61
|
+
would carry placeholder text that AgentVerity happily runs, producing
|
|
62
|
+
numbers about nothing.
|
|
63
|
+
"""
|
|
64
|
+
return bool(self.decision) and bool(self.cases)
|
|
65
|
+
|
|
66
|
+
def to_dict(self) -> dict[str, Any]:
|
|
67
|
+
return {
|
|
68
|
+
"id": self.identifier,
|
|
69
|
+
"kind": self.kind,
|
|
70
|
+
"subject": self.subject,
|
|
71
|
+
"reason": self.reason,
|
|
72
|
+
"decision": self.decision,
|
|
73
|
+
"cases": list(self.cases),
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
@classmethod
|
|
77
|
+
def from_dict(cls, value: Any) -> Obligation:
|
|
78
|
+
if not isinstance(value, dict):
|
|
79
|
+
raise ValueError("obligation must be an object")
|
|
80
|
+
missing = {"kind", "subject"} - set(value)
|
|
81
|
+
if missing:
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"obligation is missing " + ", ".join(sorted(missing))
|
|
84
|
+
)
|
|
85
|
+
kind = value["kind"]
|
|
86
|
+
subject = value["subject"]
|
|
87
|
+
reason = value.get("reason", "")
|
|
88
|
+
decision = value.get("decision", "")
|
|
89
|
+
cases = value.get("cases", [])
|
|
90
|
+
for field, item in (
|
|
91
|
+
("kind", kind),
|
|
92
|
+
("subject", subject),
|
|
93
|
+
("reason", reason),
|
|
94
|
+
("decision", decision),
|
|
95
|
+
):
|
|
96
|
+
if not isinstance(item, str):
|
|
97
|
+
raise ValueError(f"obligation {field} must be a string")
|
|
98
|
+
if not kind.strip() or not subject.strip():
|
|
99
|
+
raise ValueError("obligation kind and subject must not be blank")
|
|
100
|
+
if decision and not decision.strip():
|
|
101
|
+
raise ValueError("obligation decision must not be whitespace")
|
|
102
|
+
if not isinstance(cases, list) or any(
|
|
103
|
+
not isinstance(case, str) or not case.strip() for case in cases
|
|
104
|
+
):
|
|
105
|
+
raise ValueError("obligation cases must be a list of non-blank strings")
|
|
106
|
+
if len(cases) != len(set(cases)):
|
|
107
|
+
raise ValueError("obligation cases must not contain duplicates")
|
|
108
|
+
identifier = f"{kind}:{subject}"
|
|
109
|
+
if "id" in value and value["id"] != identifier:
|
|
110
|
+
raise ValueError(
|
|
111
|
+
f"obligation id {value['id']!r} does not match {identifier!r}"
|
|
112
|
+
)
|
|
113
|
+
return cls(
|
|
114
|
+
kind=kind,
|
|
115
|
+
subject=subject,
|
|
116
|
+
reason=reason,
|
|
117
|
+
decision=decision,
|
|
118
|
+
cases=tuple(cases),
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class ObligationSet:
|
|
124
|
+
"""Every obligation derived from one mandate."""
|
|
125
|
+
|
|
126
|
+
agent: str
|
|
127
|
+
obligations: tuple[Obligation, ...]
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def unreviewed(self) -> tuple[str, ...]:
|
|
131
|
+
"""Obligations with no decision mapped to them yet."""
|
|
132
|
+
return tuple(o.identifier for o in self.obligations if not o.reviewed)
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def decisions(self) -> tuple[str, ...]:
|
|
136
|
+
"""The distinct decisions a reviewer has mapped."""
|
|
137
|
+
return tuple(sorted({o.decision for o in self.obligations if o.reviewed}))
|
|
138
|
+
|
|
139
|
+
def render(self) -> str:
|
|
140
|
+
lines = [f"test obligations for {self.agent}"]
|
|
141
|
+
if not self.obligations:
|
|
142
|
+
lines.append(" none: no consequential authority is reachable")
|
|
143
|
+
return "\n".join(lines)
|
|
144
|
+
for obligation in self.obligations:
|
|
145
|
+
lines.append(f" {obligation.identifier}")
|
|
146
|
+
lines.append(f" why: {obligation.reason}")
|
|
147
|
+
lines.append(
|
|
148
|
+
f" decision: {obligation.decision or 'REVIEW: map a decision'}"
|
|
149
|
+
)
|
|
150
|
+
lines.append(
|
|
151
|
+
" cases: "
|
|
152
|
+
+ (
|
|
153
|
+
", ".join(obligation.cases)
|
|
154
|
+
if obligation.cases
|
|
155
|
+
else "REVIEW: write at least one probe that reaches it"
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
if self.unreviewed:
|
|
159
|
+
lines.append("")
|
|
160
|
+
lines.append(
|
|
161
|
+
f"{len(self.unreviewed)} obligation(s) need a reviewed decision "
|
|
162
|
+
"before a suite can be generated"
|
|
163
|
+
)
|
|
164
|
+
return "\n".join(lines)
|
|
165
|
+
|
|
166
|
+
def to_dict(self) -> dict[str, Any]:
|
|
167
|
+
return {
|
|
168
|
+
"schema": OBLIGATIONS_SCHEMA,
|
|
169
|
+
"agent": self.agent,
|
|
170
|
+
"obligations": [o.to_dict() for o in self.obligations],
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
@classmethod
|
|
174
|
+
def from_dict(cls, value: Any) -> ObligationSet:
|
|
175
|
+
if not isinstance(value, dict):
|
|
176
|
+
raise ValueError("obligations root must be an object")
|
|
177
|
+
if value.get("schema") != OBLIGATIONS_SCHEMA:
|
|
178
|
+
raise ValueError(
|
|
179
|
+
f"unsupported obligations schema: {value.get('schema')!r}"
|
|
180
|
+
)
|
|
181
|
+
raw = value.get("obligations")
|
|
182
|
+
if not isinstance(raw, list):
|
|
183
|
+
raise ValueError("obligations must be a list")
|
|
184
|
+
return cls(
|
|
185
|
+
agent=str(value.get("agent", "")),
|
|
186
|
+
obligations=tuple(Obligation.from_dict(entry) for entry in raw),
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def derive(mandate: Mandate, depth: int | None = None) -> ObligationSet:
|
|
191
|
+
"""Derive obligations from what the mandate actually lets the agent reach.
|
|
192
|
+
|
|
193
|
+
Reachability rather than declaration: a tool nobody can get to needs no
|
|
194
|
+
test, and listing it would pad the review with work that protects nothing.
|
|
195
|
+
"""
|
|
196
|
+
authority = analyse(mandate, depth=depth)
|
|
197
|
+
obligations: list[Obligation] = []
|
|
198
|
+
|
|
199
|
+
for name in sorted(authority.reachable_tools):
|
|
200
|
+
tool = mandate.tool(name)
|
|
201
|
+
if tool is None: # pragma: no cover - reachable names come from tools
|
|
202
|
+
continue
|
|
203
|
+
if tool.effect == IRREVERSIBLE:
|
|
204
|
+
obligations.append(
|
|
205
|
+
Obligation(
|
|
206
|
+
kind="irreversible",
|
|
207
|
+
subject=name,
|
|
208
|
+
reason=(
|
|
209
|
+
"an irreversible effect is reachable, so some reviewed "
|
|
210
|
+
"case should exercise the decision that leads to it"
|
|
211
|
+
),
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
if tool.requires_approval:
|
|
215
|
+
obligations.append(
|
|
216
|
+
Obligation(
|
|
217
|
+
kind="approval-required",
|
|
218
|
+
subject=name,
|
|
219
|
+
reason=(
|
|
220
|
+
"an approval gate is reachable, and a gate no test "
|
|
221
|
+
"reaches has never been shown to hold"
|
|
222
|
+
),
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
if tool.principal == SERVICE:
|
|
226
|
+
obligations.append(
|
|
227
|
+
Obligation(
|
|
228
|
+
kind="service-principal",
|
|
229
|
+
subject=name,
|
|
230
|
+
reason=(
|
|
231
|
+
"this call spends a service account rather than the "
|
|
232
|
+
"caller's identity, which is the confused-deputy shape"
|
|
233
|
+
),
|
|
234
|
+
)
|
|
235
|
+
)
|
|
236
|
+
if tool.spends_value:
|
|
237
|
+
obligations.append(
|
|
238
|
+
Obligation(
|
|
239
|
+
kind="value-bearing",
|
|
240
|
+
subject=name,
|
|
241
|
+
reason=(
|
|
242
|
+
f"this call spends value up to {tool.ceiling}, so the "
|
|
243
|
+
"decision that authorises it deserves reviewed cases"
|
|
244
|
+
),
|
|
245
|
+
)
|
|
246
|
+
)
|
|
247
|
+
return ObligationSet(agent=mandate.agent, obligations=tuple(obligations))
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def reconcile(current: ObligationSet, reviewed: ObligationSet) -> ObligationSet:
|
|
251
|
+
"""Carry reviewed decisions onto obligations derived from the live manifest.
|
|
252
|
+
|
|
253
|
+
A reviewed file is a record of decisions somebody mapped, not a substitute
|
|
254
|
+
for the manifest. Trusting it on its own lets a stale or unrelated review
|
|
255
|
+
generate a suite for authority the agent no longer has, which is the exact
|
|
256
|
+
drift this package exists to catch.
|
|
257
|
+
|
|
258
|
+
Matching is by stable identifier, so:
|
|
259
|
+
|
|
260
|
+
- an obligation that still exists keeps its reviewed decision and cases
|
|
261
|
+
- an obligation the manifest no longer reaches is dropped
|
|
262
|
+
- a newly reachable obligation appears unreviewed, and blocks generation
|
|
263
|
+
until somebody looks at it
|
|
264
|
+
"""
|
|
265
|
+
mapped = {o.identifier: o for o in reviewed.obligations}
|
|
266
|
+
return ObligationSet(
|
|
267
|
+
agent=current.agent,
|
|
268
|
+
obligations=tuple(
|
|
269
|
+
(
|
|
270
|
+
Obligation(
|
|
271
|
+
kind=obligation.kind,
|
|
272
|
+
subject=obligation.subject,
|
|
273
|
+
reason=obligation.reason,
|
|
274
|
+
decision=mapped[obligation.identifier].decision,
|
|
275
|
+
cases=mapped[obligation.identifier].cases,
|
|
276
|
+
)
|
|
277
|
+
if obligation.identifier in mapped
|
|
278
|
+
else obligation
|
|
279
|
+
)
|
|
280
|
+
for obligation in current.obligations
|
|
281
|
+
),
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def to_decision_suite(
|
|
286
|
+
obligations: ObligationSet, *, critical: bool = True
|
|
287
|
+
) -> dict[str, Any]:
|
|
288
|
+
"""Render reviewed obligations as an AgentVerity decision-suite skeleton.
|
|
289
|
+
|
|
290
|
+
Every obligation must carry a reviewed decision first. Generating a suite
|
|
291
|
+
from unreviewed rows would invent application labels from authority facts,
|
|
292
|
+
which is exactly the guess this module refuses to make.
|
|
293
|
+
|
|
294
|
+
Raises:
|
|
295
|
+
ValueError: If any obligation still lacks a decision.
|
|
296
|
+
"""
|
|
297
|
+
if obligations.unreviewed:
|
|
298
|
+
raise ValueError(
|
|
299
|
+
"these obligations are not fully reviewed yet: "
|
|
300
|
+
+ ", ".join(obligations.unreviewed)
|
|
301
|
+
+ ". Each needs the decision your agent returns and at least one "
|
|
302
|
+
"probe that reaches it. A suite generated from placeholders would "
|
|
303
|
+
"run and measure nothing."
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
decisions = obligations.decisions
|
|
307
|
+
if not decisions:
|
|
308
|
+
raise ValueError("no obligations to generate a suite from")
|
|
309
|
+
seen: set[str] = set()
|
|
310
|
+
for obligation in obligations.obligations:
|
|
311
|
+
duplicates = seen & set(obligation.cases)
|
|
312
|
+
if duplicates:
|
|
313
|
+
raise ValueError(
|
|
314
|
+
"the same probe is reviewed twice: " + ", ".join(sorted(duplicates))
|
|
315
|
+
)
|
|
316
|
+
seen.update(obligation.cases)
|
|
317
|
+
|
|
318
|
+
contract: dict[str, Any] = {
|
|
319
|
+
"allowed": list(decisions),
|
|
320
|
+
"required": list(decisions),
|
|
321
|
+
}
|
|
322
|
+
if critical:
|
|
323
|
+
# An irreversible or value-bearing obligation is what "critical" is
|
|
324
|
+
# for. Approval gates and service principals matter, but they are not
|
|
325
|
+
# in themselves a consequence class.
|
|
326
|
+
contract["critical"] = sorted(
|
|
327
|
+
{
|
|
328
|
+
o.decision
|
|
329
|
+
for o in obligations.obligations
|
|
330
|
+
if o.kind in {"irreversible", "value-bearing"}
|
|
331
|
+
}
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
return {
|
|
335
|
+
"schema": "agentverity.decision-suite/v1",
|
|
336
|
+
"contract": contract,
|
|
337
|
+
# Inputs come from the reviewer. Authority analysis cannot invent a
|
|
338
|
+
# probe that reaches a decision, and shipping placeholder text would
|
|
339
|
+
# produce a suite AgentVerity runs happily while measuring nothing.
|
|
340
|
+
"cases": [
|
|
341
|
+
{"input": case, "expected": obligation.decision}
|
|
342
|
+
for obligation in obligations.obligations
|
|
343
|
+
for case in obligation.cases
|
|
344
|
+
],
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def load_obligations(path: str | Path) -> ObligationSet:
|
|
349
|
+
try:
|
|
350
|
+
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
351
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
352
|
+
raise ValueError(f"cannot load obligations: {exc}") from exc
|
|
353
|
+
return ObligationSet.from_dict(value)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def save_obligations(obligations: ObligationSet, path: str | Path) -> None:
|
|
357
|
+
Path(path).write_text(
|
|
358
|
+
json.dumps(obligations.to_dict(), indent=2, sort_keys=True) + "\n",
|
|
359
|
+
encoding="utf-8",
|
|
360
|
+
)
|