agentguard-eval 0.3.2__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/PKG-INFO +3 -3
  2. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/README.md +2 -2
  3. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/pyproject.toml +1 -1
  4. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/__init__.py +1 -1
  5. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/gate/__init__.py +2 -0
  6. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/gate/governor.py +78 -9
  7. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_governor.py +67 -5
  8. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/.github/workflows/ci.yml +0 -0
  9. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/.github/workflows/publish.yml +0 -0
  10. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/.gitignore +0 -0
  11. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/LICENSE +0 -0
  12. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/examples/anthropic-log.json +0 -0
  13. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/examples/ok-but-empty.json +0 -0
  14. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/examples/openai-log.json +0 -0
  15. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/cli.py +0 -0
  16. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/gate/ledger.py +0 -0
  17. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/gate/policy.py +0 -0
  18. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/__init__.py +0 -0
  19. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/adapters.py +0 -0
  20. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/check.py +0 -0
  21. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/failure.py +0 -0
  22. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/ids.py +0 -0
  23. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/types.py +0 -0
  24. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/provenance/values.py +0 -0
  25. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/py.typed +0 -0
  26. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/pytest_plugin.py +0 -0
  27. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/shell/__init__.py +0 -0
  28. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/src/agentguard/shell/model.py +0 -0
  29. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/fixtures/governance-ledger.json +0 -0
  30. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_adapters.py +0 -0
  31. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_cli_and_plugin.py +0 -0
  32. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_gate.py +0 -0
  33. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_parity.py +0 -0
  34. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_provenance.py +0 -0
  35. {agentguard_eval-0.3.2 → agentguard_eval-0.4.0}/tests/test_shell.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: agentguard-eval
3
- Version: 0.3.2
3
+ Version: 0.4.0
4
4
  Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
5
5
  Project-URL: Homepage, https://github.com/zbs-ops/agentguard
6
6
  Project-URL: Source, https://github.com/zbs-ops/agentguard
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
28
28
  >
29
29
  > **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
30
30
  >
31
- > **Proof:** 194 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
31
+ > **Proof:** 206 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
32
32
 
33
33
  ```bash
34
34
  pip install agentguard-eval
@@ -74,7 +74,7 @@ print(a.state.value, g.spent_cents) # EXECUTED 9000
74
74
  print(g.verify().message) # All 4 entries verify against their hashes.
75
75
  ```
76
76
 
77
- Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, it takes a token bound to that one action; an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
77
+ Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, it takes a token bound to that one action's contents (ids restart with every new Governor, so a token bound to `act_0001` alone once approved a different, bigger `act_0001`); an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
78
78
 
79
79
  ### 2. A provenance checker: catch invented facts
80
80
 
@@ -4,7 +4,7 @@
4
4
  >
5
5
  > **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
6
6
  >
7
- > **Proof:** 194 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
7
+ > **Proof:** 206 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
8
8
 
9
9
  ```bash
10
10
  pip install agentguard-eval
@@ -50,7 +50,7 @@ print(a.state.value, g.spent_cents) # EXECUTED 9000
50
50
  print(g.verify().message) # All 4 entries verify against their hashes.
51
51
  ```
52
52
 
53
- Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, it takes a token bound to that one action; an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
53
+ Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, it takes a token bound to that one action's contents (ids restart with every new Governor, so a token bound to `act_0001` alone once approved a different, bigger `act_0001`); an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
54
54
 
55
55
  ### 2. A provenance checker: catch invented facts
56
56
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "agentguard-eval"
7
- version = "0.3.2"
7
+ version = "0.4.0"
8
8
  description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -1,3 +1,3 @@
1
1
  """agentguard: guardrails and evals for AI agents."""
2
2
 
3
- __version__ = "0.3.2"
3
+ __version__ = "0.4.0"
@@ -10,6 +10,7 @@ from agentguard.gate.governor import (
10
10
  SelfApprovalError,
11
11
  State,
12
12
  mint_approval_token,
13
+ proposal_digest,
13
14
  verify_approval_token,
14
15
  )
15
16
  from agentguard.gate.ledger import ChainVerification, Entry, load_entries, verify_chain
@@ -45,6 +46,7 @@ __all__ = [
45
46
  "load_entries",
46
47
  "mint_approval_token",
47
48
  "money",
49
+ "proposal_digest",
48
50
  "verify_approval_token",
49
51
  "verify_chain",
50
52
  ]
@@ -7,7 +7,8 @@ when something actually runs, and writes every step to a tamper-evident ledger.
7
7
  The rules carried over from ``agent-governance``, each one a bug that project fixed:
8
8
 
9
9
  - The agent that proposed an action can't approve it.
10
- - An approval needs a token only the named reviewer's key can make, bound to that action.
10
+ - An approval needs a token only the named reviewer's key can make, bound to that action's
11
+ contents, not just its id (ids restart with every new Governor).
11
12
  - An approval only counts for 24 hours after the request (configurable).
12
13
  - The budget is checked again at the moment an approved action would run, because other
13
14
  actions may have spent while it waited.
@@ -18,6 +19,8 @@ from __future__ import annotations
18
19
 
19
20
  import hashlib
20
21
  import hmac
22
+ import json
23
+ import math
21
24
  import time
22
25
  from collections.abc import Callable, Mapping
23
26
  from dataclasses import dataclass, field, replace
@@ -36,6 +39,7 @@ __all__ = [
36
39
  "SelfApprovalError",
37
40
  "State",
38
41
  "mint_approval_token",
42
+ "proposal_digest",
39
43
  "verify_approval_token",
40
44
  ]
41
45
 
@@ -63,19 +67,72 @@ def _norm(name: str) -> str:
63
67
  return name.strip().lower()
64
68
 
65
69
 
66
- def mint_approval_token(key: str, proposal_id: str, reviewer: str) -> str:
67
- """HMAC that binds one reviewer to one proposal. Same bytes as the TypeScript version."""
68
- message = f"{proposal_id}:{_norm(reviewer)}".encode()
70
+ def _js_number(v: float) -> str:
71
+ """A float the way JavaScript's JSON.stringify writes it."""
72
+ if not math.isfinite(v):
73
+ return "null"
74
+ if v == int(v) and abs(v) < 1e21:
75
+ return str(int(v)) # 1.0 -> "1", -0.0 -> "0"
76
+ text = repr(v)
77
+ if "e" in text:
78
+ mantissa, exp = text.split("e")
79
+ sign = "-" if exp.startswith("-") else "+"
80
+ text = f"{mantissa}e{sign}{exp.lstrip('+-').lstrip('0') or '0'}"
81
+ return text
82
+
83
+
84
+ def _canonical(value: Any) -> str:
85
+ """JSON with object keys sorted at every level: the same bytes as the TypeScript ``canonical``."""
86
+ if value is None or isinstance(value, bool):
87
+ return json.dumps(value)
88
+ if isinstance(value, int):
89
+ return str(value)
90
+ if isinstance(value, float):
91
+ return _js_number(value)
92
+ if isinstance(value, str):
93
+ return json.dumps(value, ensure_ascii=False)
94
+ if isinstance(value, (list, tuple)):
95
+ return "[" + ",".join(_canonical(x) for x in value) + "]"
96
+ if isinstance(value, Mapping):
97
+ # JavaScript sorts keys by UTF-16 code units, so sort the same way.
98
+ keys = sorted((str(k) for k in value), key=lambda k: k.encode("utf-16-be"))
99
+ return "{" + ",".join(f"{json.dumps(k, ensure_ascii=False)}:{_canonical(value[k])}" for k in keys) + "}"
100
+ raise TypeError(f"can't fingerprint a {type(value).__name__}")
101
+
102
+
103
+ def proposal_digest(
104
+ proposal_id: str, agent: str, tool: str, args: Mapping[str, Any], estimated_cost_cents: int, created_at: str
105
+ ) -> str:
106
+ """A fingerprint of everything an approval is an approval OF. Same hex as agent-governance's
107
+ ``proposalDigest``. ``created_at`` is the ISO-8601 time the action was proposed."""
108
+ body = {
109
+ "id": proposal_id,
110
+ "agent": agent,
111
+ "tool": tool,
112
+ "args": args,
113
+ "estimatedCostCents": estimated_cost_cents,
114
+ "createdAt": created_at,
115
+ }
116
+ return hashlib.sha256(_canonical(body).encode()).hexdigest()
117
+
118
+
119
+ def mint_approval_token(key: str, proposal_id: str, reviewer: str, *, digest: str) -> str:
120
+ """HMAC that binds one reviewer to one proposal's id AND contents. Same bytes as the
121
+ TypeScript version. The digest matters: ids are a counter that restarts, so a token bound
122
+ to the id alone used to approve a different action that got the same id later."""
123
+ message = f"{proposal_id}:{digest}:{_norm(reviewer)}".encode()
69
124
  return hmac.new(key.encode(), message, hashlib.sha256).hexdigest()
70
125
 
71
126
 
72
- def verify_approval_token(keys: Mapping[str, str], proposal_id: str, reviewer: str, token: str | None) -> bool:
127
+ def verify_approval_token(
128
+ keys: Mapping[str, str], proposal_id: str, reviewer: str, token: str | None, *, digest: str
129
+ ) -> bool:
73
130
  if token is None:
74
131
  return False
75
132
  key = {_norm(k): v for k, v in keys.items()}.get(_norm(reviewer))
76
133
  if key is None:
77
134
  return False
78
- return hmac.compare_digest(mint_approval_token(key, proposal_id, reviewer), token)
135
+ return hmac.compare_digest(mint_approval_token(key, proposal_id, reviewer, digest=digest), token)
79
136
 
80
137
 
81
138
  @dataclass
@@ -92,6 +149,12 @@ class GatedAction:
92
149
  review_note: str | None = None
93
150
 
94
151
 
152
+ def _digest_of(action: GatedAction) -> str:
153
+ return proposal_digest(
154
+ action.id, action.agent, action.tool, action.args, action.estimated_cost_cents, _iso(action.created_at)
155
+ )
156
+
157
+
95
158
  def _iso(ts: float) -> str:
96
159
  # Millisecond ISO-8601 in UTC with a Z, matching JavaScript's Date.toISOString().
97
160
  return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.") + f"{int(ts * 1000) % 1000:03d}Z"
@@ -134,11 +197,15 @@ class Governor:
134
197
  return verify_chain(self._ledger)
135
198
 
136
199
  def mint_token(self, action_id: str, reviewer: str) -> str:
137
- """A token for a reviewer whose key this governor holds (what your UI would call)."""
200
+ """A token for a reviewer whose key this governor holds (what your UI would call).
201
+ It covers the action as it stands now, so it approves nothing else."""
138
202
  key = {_norm(k): v for k, v in self.reviewer_keys.items()}.get(_norm(reviewer))
139
203
  if key is None:
140
204
  raise ApprovalAuthError(f"{reviewer!r} is not a registered reviewer.")
141
- return mint_approval_token(key, action_id, reviewer)
205
+ action = self.get(action_id)
206
+ if action is None:
207
+ raise KeyError(f"no action {action_id!r}")
208
+ return mint_approval_token(key, action_id, reviewer, digest=_digest_of(action))
142
209
 
143
210
  # -- the workflow ----------------------------------------------------------------------
144
211
  def propose(
@@ -181,7 +248,9 @@ class Governor:
181
248
  # Separation of duties, checked before anything is written. Withdrawing your own is fine.
182
249
  if approve and _norm(reviewer) == _norm(action.agent):
183
250
  raise SelfApprovalError(f"{action.agent!r} proposed this action, so it can't also approve it.")
184
- if approve and not verify_approval_token(self.reviewer_keys, action_id, reviewer, token):
251
+ if approve and not verify_approval_token(
252
+ self.reviewer_keys, action_id, reviewer, token, digest=_digest_of(action)
253
+ ):
185
254
  raise ApprovalAuthError(f"{reviewer!r} did not present a valid approval token.")
186
255
 
187
256
  # Only a yes expires: rejecting a stale request is how the queue gets cleared. Checked
@@ -19,6 +19,7 @@ from agentguard.gate import (
19
19
  SelfApprovalError,
20
20
  State,
21
21
  mint_approval_token,
22
+ proposal_digest,
22
23
  )
23
24
  from agentguard.gate.ledger import Entry, load_entries, verify_chain
24
25
 
@@ -316,11 +317,72 @@ def test_the_same_calls_produce_the_same_ledger_byte_for_byte():
316
317
  assert [[a.id, a.state.value, a.decision.decided_by] for a in reversed(g.actions())] == data["states"]
317
318
 
318
319
 
319
- def test_approval_tokens_match_the_typescript_hmac():
320
- data = json.loads(FIXTURE.read_text(encoding="utf-8"))
321
- key = KEYS[HUMAN]
322
- assert mint_approval_token(key, "act_0007", HUMAN) == data["tokens"]["act_0007"]
323
- assert mint_approval_token(key, "act_0001", " You@Company.com ") == data["tokens"]["act_0001"]
320
+ # Reference values printed by agent-governance's proposalDigest / mintApprovalToken (PR #10).
321
+ TS_REFERENCE = [
322
+ (
323
+ ("act_0001", "billing-agent", "stripe.refund", {"customer": "acme", "amount_cents": 500}, 500,
324
+ "2026-10-28T05:00:00.000Z"),
325
+ "e977ac4bbc68be50d51fed5b8ee0e4355a940626f71ad09f7e1bd7e9e9f9bb1f",
326
+ "e84399c9cb692b2496ba9605b7487ba83883d43cc1f40b3c984e1e63715b8ddd",
327
+ ),
328
+ (
329
+ ("act_0042", "ops", "crm.update",
330
+ {"z": [1, 2.5, True, None], "a": {"\u00fc": 'caf\u00e9 "x"\n', "n": 1.0, "k": -0.0}}, 0,
331
+ "2026-10-28T05:00:00.123Z"),
332
+ "de5a8e0b80e945ab6a123f9107df81a874ce71a69913ea3dc08efd8990aaf21e",
333
+ "b17d53dfbfa446262249003512b4c592cbdf058fdd68ab0ff15ba817e3514d31",
334
+ ),
335
+ ]
336
+
337
+
338
+ @pytest.mark.parametrize(("proposal", "digest", "token"), TS_REFERENCE)
339
+ def test_digest_and_token_match_the_typescript_bytes(proposal, digest, token):
340
+ assert proposal_digest(*proposal) == digest
341
+ assert mint_approval_token(KEYS[HUMAN], proposal[0], HUMAN, digest=digest) == token
342
+ assert mint_approval_token(KEYS[HUMAN], proposal[0], " You@Company.com ", digest=digest) == token
343
+
344
+
345
+ def test_a_token_does_not_carry_over_to_a_new_action_with_the_same_id():
346
+ # Ids are a counter, so a fresh Governor (a restart) hands out act_0001 again.
347
+ first, _ = governor()
348
+ small = refund(first, 2_000, tool="stripe.refund")
349
+ old_token = first.mint_token(small.id, HUMAN)
350
+
351
+ second, _ = governor()
352
+ big = second.propose("billing-agent", "stripe.refund", {"amount_cents": 40_000, "customer": "someone-else"},
353
+ estimated_cost_cents=40_000)
354
+ assert big.id == small.id
355
+ with pytest.raises(ApprovalAuthError):
356
+ second.decide(big.id, True, HUMAN, token=old_token)
357
+ assert second.spent_cents == 0
358
+ assert second.get(big.id).state is State.PENDING_APPROVAL
359
+
360
+
361
+ @pytest.mark.parametrize(
362
+ "change",
363
+ [
364
+ {"args": {"amount_cents": 2_001}},
365
+ {"args": {"amount_cents": 2_000, "note": "and the rest"}},
366
+ {"tool": "stripe.payout"},
367
+ {"agent": "other-agent"},
368
+ {"estimated_cost_cents": 1},
369
+ {"created_at": 1_793_000_001.0},
370
+ ],
371
+ )
372
+ def test_a_token_is_refused_if_anything_it_covered_changed(change):
373
+ g, _ = governor()
374
+ a = refund(g, 2_000, tool="stripe.refund")
375
+ token = g.mint_token(a.id, HUMAN)
376
+ for k, v in change.items():
377
+ setattr(a, k, v)
378
+ with pytest.raises(ApprovalAuthError):
379
+ g.decide(a.id, True, HUMAN, token=token)
380
+
381
+
382
+ def test_argument_order_does_not_change_the_digest():
383
+ one = proposal_digest("act_0001", "a", "t", {"x": 1, "y": {"b": 2, "a": 1}}, 0, "2026-10-28T05:00:00.000Z")
384
+ two = proposal_digest("act_0001", "a", "t", {"y": {"a": 1, "b": 2}, "x": 1}, 0, "2026-10-28T05:00:00.000Z")
385
+ assert one == two
324
386
 
325
387
 
326
388
  def test_cli_verify_ledger(tmp_path, capsys):
File without changes