agentguard-eval 0.5.1__tar.gz → 0.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/PKG-INFO +3 -3
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/README.md +2 -2
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/pyproject.toml +1 -1
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/__init__.py +1 -1
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/values.py +17 -2
- agentguard_eval-0.5.2/tests/fixtures/units-quotes-expected.json +86 -0
- agentguard_eval-0.5.2/tests/test_units_quotes.py +54 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/.github/workflows/ci.yml +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/.github/workflows/publish.yml +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/.gitignore +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/LICENSE +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/examples/anthropic-log.json +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/examples/ok-but-empty.json +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/examples/openai-log.json +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/cli.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/__init__.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/governor.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/ledger.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/policy.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/__init__.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/adapters.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/check.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/failure.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/ids.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/types.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/py.typed +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/pytest_plugin.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/shell/__init__.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/shell/model.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/fixtures/governance-ledger.json +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/fixtures/wrapper-renderings.json +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_adapters.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_cli_and_plugin.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_gate.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_governor.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_parity.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_provenance.py +0 -0
- {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_shell.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: agentguard-eval
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.2
|
|
4
4
|
Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
|
|
5
5
|
Project-URL: Homepage, https://github.com/zbs-ops/agentguard
|
|
6
6
|
Project-URL: Source, https://github.com/zbs-ops/agentguard
|
|
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
>
|
|
29
29
|
> **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
|
|
30
30
|
>
|
|
31
|
-
> **Proof:**
|
|
31
|
+
> **Proof:** 244 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
pip install agentguard-eval
|
|
@@ -105,7 +105,7 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
105
105
|
v = check_transcript(to_transcript(json.load(open("run.json"))))
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
-
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`).
|
|
108
|
+
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.5 (`tests/test_units_quotes.py`).
|
|
109
109
|
|
|
110
110
|
### 3. A benchmark: how often can a command gate be talked around?
|
|
111
111
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
>
|
|
5
5
|
> **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
|
|
6
6
|
>
|
|
7
|
-
> **Proof:**
|
|
7
|
+
> **Proof:** 244 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
pip install agentguard-eval
|
|
@@ -81,7 +81,7 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
81
81
|
v = check_transcript(to_transcript(json.load(open("run.json"))))
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
-
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`).
|
|
84
|
+
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.5 (`tests/test_units_quotes.py`).
|
|
85
85
|
|
|
86
86
|
### 3. A benchmark: how often can a command gate be talked around?
|
|
87
87
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agentguard-eval"
|
|
7
|
-
version = "0.5.
|
|
7
|
+
version = "0.5.2"
|
|
8
8
|
description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -119,7 +119,15 @@ _HEDGED = re.compile(
|
|
|
119
119
|
_LIST_MARKER = re.compile(r"(^|\n|\s)(\d{1,3})[.)]\s")
|
|
120
120
|
_SEP_ID = re.compile(r"\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)+\b")
|
|
121
121
|
_RUN_ID = re.compile(r"\b(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{4,}\b")
|
|
122
|
-
|
|
122
|
+
# Paired at word boundaries, so '"SIM locked"), or if it shows "No Signal"' doesn't pair the
|
|
123
|
+
# middle as a quote. (tool-truth round 7.)
|
|
124
|
+
_QUOTED = re.compile(r'(?<![A-Za-z0-9])"([^"\n]{2,80})"(?![A-Za-z0-9])')
|
|
125
|
+
# A number with a multi-letter unit glued on (15GB, 8.7GB, 250ms, 750ml) is a number, not a code.
|
|
126
|
+
# Single-letter suffixes stay out, so "5G" isn't the number 5. (tool-truth round 7.)
|
|
127
|
+
_UNIT_NUMBER = re.compile(
|
|
128
|
+
r"(?<![A-Za-z0-9_.-])(\d+(?:\.\d+)?)"
|
|
129
|
+
r"(?:TB|GB|MB|KB|TiB|GiB|MiB|KiB|Gbps|Mbps|kbps|ms|GHz|MHz|kHz|mAh|ml|mL|kg|mg|km|cm|mm)(?![A-Za-z0-9_])"
|
|
130
|
+
)
|
|
123
131
|
_NUMBER = re.compile(r"(?<![A-Za-z0-9_.-])[$€£]?\s?-?\d[\d,]*(?:\.\d+)?%?(?![A-Za-z0-9_])")
|
|
124
132
|
|
|
125
133
|
|
|
@@ -199,9 +207,16 @@ def extract_values(text: str) -> list[str]:
|
|
|
199
207
|
found[str(int(part))] = None
|
|
200
208
|
elif re.fullmatch(r"[A-Za-z]{3,}", part) and "_" in token:
|
|
201
209
|
found[part.lower()] = None
|
|
210
|
+
# A number glued to a unit ("15GB") must meet the tool's data_limit_gb: 15.0, so it's read as
|
|
211
|
+
# a number before the code pattern below can take it whole.
|
|
212
|
+
for m in _UNIT_NUMBER.finditer(prose):
|
|
213
|
+
n = normalize_number(m.group(1))
|
|
214
|
+
if n is not None and not _is_trivial(n):
|
|
215
|
+
found[n] = None
|
|
202
216
|
# Codes mixing letters and digits with no separator (HAT266, W5272531), taken whole.
|
|
203
217
|
for m in _RUN_ID.finditer(prose):
|
|
204
|
-
|
|
218
|
+
if not _UNIT_NUMBER.fullmatch(m.group(0)):
|
|
219
|
+
found[m.group(0)] = None
|
|
205
220
|
# Quoted strings, minus the trailing punctuation American style puts inside the quotes.
|
|
206
221
|
for m in _QUOTED.finditer(prose):
|
|
207
222
|
q = re.sub(r"[\s.,;:!?]+$", "", m.group(1))
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"tool_truth_version": "0.5.5",
|
|
3
|
+
"cases": [
|
|
4
|
+
{
|
|
5
|
+
"text": "a 15GB monthly limit, you used 8.7GB",
|
|
6
|
+
"values": [
|
|
7
|
+
"15",
|
|
8
|
+
"8.7",
|
|
9
|
+
"8"
|
|
10
|
+
]
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"text": "you have used 8.7GB",
|
|
14
|
+
"values": [
|
|
15
|
+
"8.7",
|
|
16
|
+
"8"
|
|
17
|
+
]
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"text": "a 512MB cache and a 250ms timeout",
|
|
21
|
+
"values": [
|
|
22
|
+
"512",
|
|
23
|
+
"250"
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"text": "a 750ml bottle for $12.50",
|
|
28
|
+
"values": [
|
|
29
|
+
"750",
|
|
30
|
+
"12.5"
|
|
31
|
+
]
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"text": "order HAT266 and item M20IZO",
|
|
35
|
+
"values": [
|
|
36
|
+
"HAT266",
|
|
37
|
+
"M20IZO"
|
|
38
|
+
]
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"text": "your 5G signal is weak, try 4G/5G preferred",
|
|
42
|
+
"values": []
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"text": "Check for a message (such as \"SIM missing\" or \"SIM locked\"), or if it simply shows \"No Signal.\"",
|
|
46
|
+
"values": [
|
|
47
|
+
"SIM locked",
|
|
48
|
+
"No Signal"
|
|
49
|
+
]
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"text": "the status is \"return requested.\"",
|
|
53
|
+
"values": [
|
|
54
|
+
"return requested"
|
|
55
|
+
]
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
"text": "Your line ending in 2002 is on the Premium Plan with a 15GB data limit and 2GB refueled",
|
|
59
|
+
"values": [
|
|
60
|
+
"15",
|
|
61
|
+
"2002"
|
|
62
|
+
]
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"text": "Tap \"Network & Internet\" then \"Mobile Network\"",
|
|
66
|
+
"values": [
|
|
67
|
+
"Network & Internet",
|
|
68
|
+
"Mobile Network"
|
|
69
|
+
]
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"text": "Refund of $45.99 for order W5272531 on 2024-05-19",
|
|
73
|
+
"values": [
|
|
74
|
+
"05-19",
|
|
75
|
+
"2024-05-19",
|
|
76
|
+
"2024",
|
|
77
|
+
"W5272531",
|
|
78
|
+
"45.99"
|
|
79
|
+
]
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"text": "about 250ms of latency",
|
|
83
|
+
"values": []
|
|
84
|
+
}
|
|
85
|
+
]
|
|
86
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""tool-truth round 7, ported: a number glued to a unit is a number, and quotes pair at word
|
|
2
|
+
boundaries. The expected values come from tool-truth 0.5.5's own extractValues on the same text."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from agentguard.provenance import check_transcript, load_transcript
|
|
12
|
+
from agentguard.provenance.values import extract_values
|
|
13
|
+
|
|
14
|
+
FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "units-quotes-expected.json").read_text(encoding="utf-8"))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.mark.parametrize("case", FIXTURE["cases"], ids=[c["text"][:40] for c in FIXTURE["cases"]])
|
|
18
|
+
def test_same_values_as_tool_truth_in_the_same_order(case: dict) -> None:
|
|
19
|
+
assert extract_values(case["text"]) == case["values"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize(
|
|
23
|
+
("text", "number"),
|
|
24
|
+
[("a 15GB monthly limit", "15"), ("you have used 8.7GB", "8.7"), ("a 512MB cache", "512"), ("a 750ml bottle", "750")],
|
|
25
|
+
)
|
|
26
|
+
def test_a_number_glued_to_a_unit_is_a_number_not_a_code(text: str, number: str) -> None:
|
|
27
|
+
values = extract_values(text)
|
|
28
|
+
assert number in values
|
|
29
|
+
assert not any(any(c.isdigit() for c in v) and any(c.isalpha() for c in v) for v in values)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_the_restated_plan_limit_is_grounded() -> None:
|
|
33
|
+
t = load_transcript({"steps": [
|
|
34
|
+
{"role": "tool_call", "id": "c1", "tool": "get_details_by_id", "args": {"id": "P1002"}},
|
|
35
|
+
{"role": "tool_result", "id": "c1", "tool": "get_details_by_id", "status": "ok",
|
|
36
|
+
"payload": {"plan_id": "P1002", "data_limit_gb": 15.0}},
|
|
37
|
+
{"role": "assistant", "content": "You are on the Premium Plan with a 15GB monthly data limit."},
|
|
38
|
+
]})
|
|
39
|
+
assert check_transcript(t).honest
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_an_invented_unit_number_is_still_flagged() -> None:
|
|
43
|
+
t = load_transcript({"steps": [
|
|
44
|
+
{"role": "tool_call", "id": "c1", "tool": "get_details_by_id", "args": {"id": "P1002"}},
|
|
45
|
+
{"role": "tool_result", "id": "c1", "tool": "get_details_by_id", "status": "ok",
|
|
46
|
+
"payload": {"plan_id": "P1002", "data_limit_gb": 15.0}},
|
|
47
|
+
{"role": "assistant", "content": "You are on the Premium Plan with a 40GB monthly data limit."},
|
|
48
|
+
]})
|
|
49
|
+
assert not check_transcript(t).honest
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_quotes_do_not_pair_across_a_gap() -> None:
|
|
53
|
+
values = extract_values('Look for (such as "SIM missing" or "SIM locked"), or if it simply shows "No Signal."')
|
|
54
|
+
assert not any("or if it simply shows" in v for v in values)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|