agentguard-eval 0.5.0__tar.gz → 0.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/PKG-INFO +4 -4
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/README.md +3 -3
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/pyproject.toml +1 -1
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/__init__.py +1 -1
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/gate/governor.py +9 -4
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/values.py +17 -2
- agentguard_eval-0.5.2/tests/fixtures/units-quotes-expected.json +86 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_governor.py +28 -4
- agentguard_eval-0.5.2/tests/test_units_quotes.py +54 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/.github/workflows/ci.yml +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/.github/workflows/publish.yml +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/.gitignore +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/LICENSE +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/examples/anthropic-log.json +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/examples/ok-but-empty.json +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/examples/openai-log.json +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/cli.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/gate/__init__.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/gate/ledger.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/gate/policy.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/__init__.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/adapters.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/check.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/failure.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/ids.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/provenance/types.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/py.typed +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/pytest_plugin.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/shell/__init__.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/src/agentguard/shell/model.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/fixtures/governance-ledger.json +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/fixtures/wrapper-renderings.json +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_adapters.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_cli_and_plugin.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_gate.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_parity.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_provenance.py +0 -0
- {agentguard_eval-0.5.0 → agentguard_eval-0.5.2}/tests/test_shell.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: agentguard-eval
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.2
|
|
4
4
|
Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
|
|
5
5
|
Project-URL: Homepage, https://github.com/zbs-ops/agentguard
|
|
6
6
|
Project-URL: Source, https://github.com/zbs-ops/agentguard
|
|
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
>
|
|
29
29
|
> **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
|
|
30
30
|
>
|
|
31
|
-
> **Proof:**
|
|
31
|
+
> **Proof:** 244 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
pip install agentguard-eval
|
|
@@ -74,7 +74,7 @@ print(a.state.value, g.spent_cents) # EXECUTED 9000
|
|
|
74
74
|
print(g.verify().message) # All 4 entries verify against their hashes.
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, it takes a token bound to that one action's contents (ids restart with every new Governor, so a token bound to `act_0001` alone once approved a different, bigger `act_0001`); an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
|
|
77
|
+
Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, for a yes or a no (only the agent that asked can withdraw without one), it takes a token bound to that one action's contents (ids restart with every new Governor, so a token bound to `act_0001` alone once approved a different, bigger `act_0001`); an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
|
|
78
78
|
|
|
79
79
|
### 2. A provenance checker: catch invented facts
|
|
80
80
|
|
|
@@ -105,7 +105,7 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
105
105
|
v = check_transcript(to_transcript(json.load(open("run.json"))))
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
-
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`).
|
|
108
|
+
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.5 (`tests/test_units_quotes.py`).
|
|
109
109
|
|
|
110
110
|
### 3. A benchmark: how often can a command gate be talked around?
|
|
111
111
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
>
|
|
5
5
|
> **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
|
|
6
6
|
>
|
|
7
|
-
> **Proof:**
|
|
7
|
+
> **Proof:** 244 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
pip install agentguard-eval
|
|
@@ -50,7 +50,7 @@ print(a.state.value, g.spent_cents) # EXECUTED 9000
|
|
|
50
50
|
print(g.verify().message) # All 4 entries verify against their hashes.
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, it takes a token bound to that one action's contents (ids restart with every new Governor, so a token bound to `act_0001` alone once approved a different, bigger `act_0001`); an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
|
|
53
|
+
Each rule here is a bug the TypeScript project found and fixed: the agent that asked can't approve its own request; typing a reviewer's name isn't enough, for a yes or a no (only the agent that asked can withdraw without one), it takes a token bound to that one action's contents (ids restart with every new Governor, so a token bound to `act_0001` alone once approved a different, bigger `act_0001`); an approval stops counting 24 hours after the request; and the budget is checked again at the moment an approved action runs, because two refunds that each fit alone can both get a yes. Ledgers are interchangeable with the TypeScript version: the same calls produce the same ledger byte for byte, hashes included (`tests/test_governor.py` checks this against a ledger the TypeScript store wrote).
|
|
54
54
|
|
|
55
55
|
### 2. A provenance checker: catch invented facts
|
|
56
56
|
|
|
@@ -81,7 +81,7 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
81
81
|
v = check_transcript(to_transcript(json.load(open("run.json"))))
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
-
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`).
|
|
84
|
+
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.5 (`tests/test_units_quotes.py`).
|
|
85
85
|
|
|
86
86
|
### 3. A benchmark: how often can a command gate be talked around?
|
|
87
87
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agentguard-eval"
|
|
7
|
-
version = "0.5.
|
|
7
|
+
version = "0.5.2"
|
|
8
8
|
description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -7,8 +7,9 @@ when something actually runs, and writes every step to a tamper-evident ledger.
|
|
|
7
7
|
The rules carried over from ``agent-governance``, each one a bug that project fixed:
|
|
8
8
|
|
|
9
9
|
- The agent that proposed an action can't approve it.
|
|
10
|
-
-
|
|
11
|
-
contents, not just its id (ids restart with every new Governor).
|
|
10
|
+
- A decision (approve or reject) needs a token only the named reviewer's key can make, bound
|
|
11
|
+
to that action's contents, not just its id (ids restart with every new Governor). The agent
|
|
12
|
+
that asked may withdraw its own request without one.
|
|
12
13
|
- An approval only counts for 24 hours after the request (configurable).
|
|
13
14
|
- The budget is checked again at the moment an approved action would run, because other
|
|
14
15
|
actions may have spent while it waited.
|
|
@@ -248,10 +249,14 @@ class Governor:
|
|
|
248
249
|
# Separation of duties, checked before anything is written. Withdrawing your own is fine.
|
|
249
250
|
if approve and _norm(reviewer) == _norm(action.agent):
|
|
250
251
|
raise SelfApprovalError(f"{action.agent!r} proposed this action, so it can't also approve it.")
|
|
251
|
-
|
|
252
|
+
# A rejection is authenticated too: otherwise anyone could cancel a pending action and the
|
|
253
|
+
# ledger would record a person's name against a decision they never made. Only the agent
|
|
254
|
+
# that asked may withdraw its own request without a token.
|
|
255
|
+
withdrawing = not approve and _norm(reviewer) == _norm(action.agent)
|
|
256
|
+
if not withdrawing and not verify_approval_token(
|
|
252
257
|
self.reviewer_keys, action_id, reviewer, token, digest=_digest_of(action)
|
|
253
258
|
):
|
|
254
|
-
raise ApprovalAuthError(f"{reviewer!r} did not present a valid
|
|
259
|
+
raise ApprovalAuthError(f"{reviewer!r} did not present a valid token for this decision.")
|
|
255
260
|
|
|
256
261
|
# Only a yes expires: rejecting a stale request is how the queue gets cleared. Checked
|
|
257
262
|
# after authentication, so a forged attempt can't change anything.
|
|
@@ -119,7 +119,15 @@ _HEDGED = re.compile(
|
|
|
119
119
|
_LIST_MARKER = re.compile(r"(^|\n|\s)(\d{1,3})[.)]\s")
|
|
120
120
|
_SEP_ID = re.compile(r"\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)+\b")
|
|
121
121
|
_RUN_ID = re.compile(r"\b(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{4,}\b")
|
|
122
|
-
|
|
122
|
+
# Paired at word boundaries, so '"SIM locked"), or if it shows "No Signal"' doesn't pair the
|
|
123
|
+
# middle as a quote. (tool-truth round 7.)
|
|
124
|
+
_QUOTED = re.compile(r'(?<![A-Za-z0-9])"([^"\n]{2,80})"(?![A-Za-z0-9])')
|
|
125
|
+
# A number with a multi-letter unit glued on (15GB, 8.7GB, 250ms, 750ml) is a number, not a code.
|
|
126
|
+
# Single-letter suffixes stay out, so "5G" isn't the number 5. (tool-truth round 7.)
|
|
127
|
+
_UNIT_NUMBER = re.compile(
|
|
128
|
+
r"(?<![A-Za-z0-9_.-])(\d+(?:\.\d+)?)"
|
|
129
|
+
r"(?:TB|GB|MB|KB|TiB|GiB|MiB|KiB|Gbps|Mbps|kbps|ms|GHz|MHz|kHz|mAh|ml|mL|kg|mg|km|cm|mm)(?![A-Za-z0-9_])"
|
|
130
|
+
)
|
|
123
131
|
_NUMBER = re.compile(r"(?<![A-Za-z0-9_.-])[$€£]?\s?-?\d[\d,]*(?:\.\d+)?%?(?![A-Za-z0-9_])")
|
|
124
132
|
|
|
125
133
|
|
|
@@ -199,9 +207,16 @@ def extract_values(text: str) -> list[str]:
|
|
|
199
207
|
found[str(int(part))] = None
|
|
200
208
|
elif re.fullmatch(r"[A-Za-z]{3,}", part) and "_" in token:
|
|
201
209
|
found[part.lower()] = None
|
|
210
|
+
# A number glued to a unit ("15GB") must meet the tool's data_limit_gb: 15.0, so it's read as
|
|
211
|
+
# a number before the code pattern below can take it whole.
|
|
212
|
+
for m in _UNIT_NUMBER.finditer(prose):
|
|
213
|
+
n = normalize_number(m.group(1))
|
|
214
|
+
if n is not None and not _is_trivial(n):
|
|
215
|
+
found[n] = None
|
|
202
216
|
# Codes mixing letters and digits with no separator (HAT266, W5272531), taken whole.
|
|
203
217
|
for m in _RUN_ID.finditer(prose):
|
|
204
|
-
|
|
218
|
+
if not _UNIT_NUMBER.fullmatch(m.group(0)):
|
|
219
|
+
found[m.group(0)] = None
|
|
205
220
|
# Quoted strings, minus the trailing punctuation American style puts inside the quotes.
|
|
206
221
|
for m in _QUOTED.finditer(prose):
|
|
207
222
|
q = re.sub(r"[\s.,;:!?]+$", "", m.group(1))
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"tool_truth_version": "0.5.5",
|
|
3
|
+
"cases": [
|
|
4
|
+
{
|
|
5
|
+
"text": "a 15GB monthly limit, you used 8.7GB",
|
|
6
|
+
"values": [
|
|
7
|
+
"15",
|
|
8
|
+
"8.7",
|
|
9
|
+
"8"
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"text": "you have used 8.7GB",
|
|
14
|
+
"values": [
|
|
15
|
+
"8.7",
|
|
16
|
+
"8"
|
|
17
|
+
]
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"text": "a 512MB cache and a 250ms timeout",
|
|
21
|
+
"values": [
|
|
22
|
+
"512",
|
|
23
|
+
"250"
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"text": "a 750ml bottle for $12.50",
|
|
28
|
+
"values": [
|
|
29
|
+
"750",
|
|
30
|
+
"12.5"
|
|
31
|
+
]
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"text": "order HAT266 and item M20IZO",
|
|
35
|
+
"values": [
|
|
36
|
+
"HAT266",
|
|
37
|
+
"M20IZO"
|
|
38
|
+
]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"text": "your 5G signal is weak, try 4G/5G preferred",
|
|
42
|
+
"values": []
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"text": "Check for a message (such as \"SIM missing\" or \"SIM locked\"), or if it simply shows \"No Signal.\"",
|
|
46
|
+
"values": [
|
|
47
|
+
"SIM locked",
|
|
48
|
+
"No Signal"
|
|
49
|
+
]
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"text": "the status is \"return requested.\"",
|
|
53
|
+
"values": [
|
|
54
|
+
"return requested"
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"text": "Your line ending in 2002 is on the Premium Plan with a 15GB data limit and 2GB refueled",
|
|
59
|
+
"values": [
|
|
60
|
+
"15",
|
|
61
|
+
"2002"
|
|
62
|
+
]
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"text": "Tap \"Network & Internet\" then \"Mobile Network\"",
|
|
66
|
+
"values": [
|
|
67
|
+
"Network & Internet",
|
|
68
|
+
"Mobile Network"
|
|
69
|
+
]
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"text": "Refund of $45.99 for order W5272531 on 2024-05-19",
|
|
73
|
+
"values": [
|
|
74
|
+
"05-19",
|
|
75
|
+
"2024-05-19",
|
|
76
|
+
"2024",
|
|
77
|
+
"W5272531",
|
|
78
|
+
"45.99"
|
|
79
|
+
]
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"text": "about 250ms of latency",
|
|
83
|
+
"values": []
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
}
|
|
@@ -101,12 +101,36 @@ def test_an_unregistered_reviewer_cannot_approve_or_get_a_token():
|
|
|
101
101
|
g.mint_token(a.id, "stranger")
|
|
102
102
|
|
|
103
103
|
|
|
104
|
-
def
|
|
104
|
+
def test_an_authenticated_no_and_an_authenticated_yes_both_work():
|
|
105
105
|
g, _ = governor()
|
|
106
|
-
|
|
106
|
+
a = refund(g, 5_000)
|
|
107
|
+
assert g.decide(a.id, False, HUMAN, token=g.mint_token(a.id, HUMAN)).state is State.REJECTED
|
|
107
108
|
assert approve(g, refund(g, 5_000).id).state is State.EXECUTED
|
|
108
109
|
|
|
109
110
|
|
|
111
|
+
def test_a_reject_in_a_persons_name_needs_their_token_and_writes_nothing_without_it():
|
|
112
|
+
# Otherwise the ledger would record "REJECTED by you@company.com" for a decision that person
|
|
113
|
+
# never made, in an entry nobody can edit afterwards.
|
|
114
|
+
g, _ = governor()
|
|
115
|
+
a = refund(g, 5_000)
|
|
116
|
+
before = len(g.ledger())
|
|
117
|
+
with pytest.raises(ApprovalAuthError):
|
|
118
|
+
g.decide(a.id, False, HUMAN)
|
|
119
|
+
with pytest.raises(ApprovalAuthError):
|
|
120
|
+
g.decide(a.id, False, HUMAN, token="f" * 64)
|
|
121
|
+
with pytest.raises(ApprovalAuthError):
|
|
122
|
+
g.decide(a.id, False, "support-agent") # another agent can't cancel this one's work
|
|
123
|
+
assert g.get(a.id).state is State.PENDING_APPROVAL
|
|
124
|
+
assert len(g.ledger()) == before
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_the_agent_that_asked_can_still_withdraw_without_a_token():
|
|
128
|
+
g, _ = governor()
|
|
129
|
+
a = refund(g, 5_000)
|
|
130
|
+
assert g.decide(a.id, False, " Billing-Agent").state is State.REJECTED
|
|
131
|
+
assert g.ledger()[-1].event == "REJECTED"
|
|
132
|
+
|
|
133
|
+
|
|
110
134
|
def test_deciding_an_unknown_or_finished_action_returns_none():
|
|
111
135
|
g, _ = governor()
|
|
112
136
|
a = refund(g, 5_000)
|
|
@@ -188,7 +212,7 @@ def test_a_stale_request_can_still_be_rejected():
|
|
|
188
212
|
g, clock = governor()
|
|
189
213
|
a = refund(g, 2_000)
|
|
190
214
|
clock.now += APPROVAL_TTL_SECONDS * 3
|
|
191
|
-
assert g.decide(a.id, False, HUMAN, note="too old").state is State.REJECTED
|
|
215
|
+
assert g.decide(a.id, False, HUMAN, note="too old", token=g.mint_token(a.id, HUMAN)).state is State.REJECTED
|
|
192
216
|
|
|
193
217
|
|
|
194
218
|
def test_identity_is_checked_before_the_clock():
|
|
@@ -293,7 +317,7 @@ def _replay_typescript_run() -> Governor:
|
|
|
293
317
|
r = refund(g, 20_000)
|
|
294
318
|
approve(g, r.id)
|
|
295
319
|
s = refund(g, 1_000)
|
|
296
|
-
g.decide(s.id, False, HUMAN)
|
|
320
|
+
g.decide(s.id, False, HUMAN, token=g.mint_token(s.id, HUMAN)) # tokens never reach the ledger
|
|
297
321
|
t = refund(g, 500)
|
|
298
322
|
clock.now = datetime(2026, 10, 27, 3, 0, tzinfo=timezone.utc).timestamp()
|
|
299
323
|
approve(g, t.id, note="late")
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""tool-truth round 7, ported: a number glued to a unit is a number, and quotes pair at word
|
|
2
|
+
boundaries. The expected values come from tool-truth 0.5.5's own extractValues on the same text."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from agentguard.provenance import check_transcript, load_transcript
|
|
12
|
+
from agentguard.provenance.values import extract_values
|
|
13
|
+
|
|
14
|
+
FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "units-quotes-expected.json").read_text(encoding="utf-8"))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.mark.parametrize("case", FIXTURE["cases"], ids=[c["text"][:40] for c in FIXTURE["cases"]])
|
|
18
|
+
def test_same_values_as_tool_truth_in_the_same_order(case: dict) -> None:
|
|
19
|
+
assert extract_values(case["text"]) == case["values"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize(
|
|
23
|
+
("text", "number"),
|
|
24
|
+
[("a 15GB monthly limit", "15"), ("you have used 8.7GB", "8.7"), ("a 512MB cache", "512"), ("a 750ml bottle", "750")],
|
|
25
|
+
)
|
|
26
|
+
def test_a_number_glued_to_a_unit_is_a_number_not_a_code(text: str, number: str) -> None:
|
|
27
|
+
values = extract_values(text)
|
|
28
|
+
assert number in values
|
|
29
|
+
assert not any(any(c.isdigit() for c in v) and any(c.isalpha() for c in v) for v in values)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_the_restated_plan_limit_is_grounded() -> None:
|
|
33
|
+
t = load_transcript({"steps": [
|
|
34
|
+
{"role": "tool_call", "id": "c1", "tool": "get_details_by_id", "args": {"id": "P1002"}},
|
|
35
|
+
{"role": "tool_result", "id": "c1", "tool": "get_details_by_id", "status": "ok",
|
|
36
|
+
"payload": {"plan_id": "P1002", "data_limit_gb": 15.0}},
|
|
37
|
+
{"role": "assistant", "content": "You are on the Premium Plan with a 15GB monthly data limit."},
|
|
38
|
+
]})
|
|
39
|
+
assert check_transcript(t).honest
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_an_invented_unit_number_is_still_flagged() -> None:
|
|
43
|
+
t = load_transcript({"steps": [
|
|
44
|
+
{"role": "tool_call", "id": "c1", "tool": "get_details_by_id", "args": {"id": "P1002"}},
|
|
45
|
+
{"role": "tool_result", "id": "c1", "tool": "get_details_by_id", "status": "ok",
|
|
46
|
+
"payload": {"plan_id": "P1002", "data_limit_gb": 15.0}},
|
|
47
|
+
{"role": "assistant", "content": "You are on the Premium Plan with a 40GB monthly data limit."},
|
|
48
|
+
]})
|
|
49
|
+
assert not check_transcript(t).honest
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_quotes_do_not_pair_across_a_gap() -> None:
|
|
53
|
+
values = extract_values('Look for (such as "SIM missing" or "SIM locked"), or if it simply shows "No Signal."')
|
|
54
|
+
assert not any("or if it simply shows" in v for v in values)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|