agentguard-eval 0.5.1__tar.gz → 0.5.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/PKG-INFO +3 -3
  2. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/README.md +2 -2
  3. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/pyproject.toml +1 -1
  4. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/__init__.py +1 -1
  5. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/values.py +17 -2
  6. agentguard_eval-0.5.2/tests/fixtures/units-quotes-expected.json +86 -0
  7. agentguard_eval-0.5.2/tests/test_units_quotes.py +54 -0
  8. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/.github/workflows/ci.yml +0 -0
  9. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/.github/workflows/publish.yml +0 -0
  10. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/.gitignore +0 -0
  11. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/LICENSE +0 -0
  12. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/examples/anthropic-log.json +0 -0
  13. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/examples/ok-but-empty.json +0 -0
  14. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/examples/openai-log.json +0 -0
  15. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/cli.py +0 -0
  16. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/__init__.py +0 -0
  17. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/governor.py +0 -0
  18. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/ledger.py +0 -0
  19. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/gate/policy.py +0 -0
  20. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/__init__.py +0 -0
  21. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/adapters.py +0 -0
  22. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/check.py +0 -0
  23. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/failure.py +0 -0
  24. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/ids.py +0 -0
  25. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/provenance/types.py +0 -0
  26. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/py.typed +0 -0
  27. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/pytest_plugin.py +0 -0
  28. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/shell/__init__.py +0 -0
  29. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/src/agentguard/shell/model.py +0 -0
  30. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/fixtures/governance-ledger.json +0 -0
  31. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/fixtures/wrapper-renderings.json +0 -0
  32. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_adapters.py +0 -0
  33. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_cli_and_plugin.py +0 -0
  34. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_gate.py +0 -0
  35. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_governor.py +0 -0
  36. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_parity.py +0 -0
  37. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_provenance.py +0 -0
  38. {agentguard_eval-0.5.1 → agentguard_eval-0.5.2}/tests/test_shell.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: agentguard-eval
3
- Version: 0.5.1
3
+ Version: 0.5.2
4
4
  Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
5
5
  Project-URL: Homepage, https://github.com/zbs-ops/agentguard
6
6
  Project-URL: Source, https://github.com/zbs-ops/agentguard
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
28
28
  >
29
29
  > **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
30
30
  >
31
- > **Proof:** 225 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
31
+ > **Proof:** 244 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
32
32
 
33
33
  ```bash
34
34
  pip install agentguard-eval
@@ -105,7 +105,7 @@ from agentguard.provenance import check_transcript, to_transcript
105
105
  v = check_transcript(to_transcript(json.load(open("run.json"))))
106
106
  ```
107
107
 
108
- Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`).
108
+ Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.5 (`tests/test_units_quotes.py`).
109
109
 
110
110
  ### 3. A benchmark: how often can a command gate be talked around?
111
111
 
@@ -4,7 +4,7 @@
4
4
  >
5
5
  > **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
6
6
  >
7
- > **Proof:** 225 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
7
+ > **Proof:** 244 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
8
8
 
9
9
  ```bash
10
10
  pip install agentguard-eval
@@ -81,7 +81,7 @@ from agentguard.provenance import check_transcript, to_transcript
81
81
  v = check_transcript(to_transcript(json.load(open("run.json"))))
82
82
  ```
83
83
 
84
- Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`).
84
+ Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.5 (`tests/test_units_quotes.py`).
85
85
 
86
86
  ### 3. A benchmark: how often can a command gate be talked around?
87
87
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "agentguard-eval"
7
- version = "0.5.1"
7
+ version = "0.5.2"
8
8
  description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -1,3 +1,3 @@
1
1
  """agentguard: guardrails and evals for AI agents."""
2
2
 
3
- __version__ = "0.5.1"
3
+ __version__ = "0.5.2"
@@ -119,7 +119,15 @@ _HEDGED = re.compile(
119
119
  _LIST_MARKER = re.compile(r"(^|\n|\s)(\d{1,3})[.)]\s")
120
120
  _SEP_ID = re.compile(r"\b(?=[A-Za-z0-9_-]*\d)(?=[A-Za-z0-9_-]*[A-Za-z])[A-Za-z0-9]+(?:[-_][A-Za-z0-9]+)+\b")
121
121
  _RUN_ID = re.compile(r"\b(?=[A-Za-z0-9]*\d)(?=[A-Za-z0-9]*[A-Za-z])[A-Za-z0-9]{4,}\b")
122
- _QUOTED = re.compile(r'"([^"\n]{2,80})"')
122
+ # Paired at word boundaries, so '"SIM locked"), or if it shows "No Signal"' doesn't pair the
123
+ # middle as a quote. (tool-truth round 7.)
124
+ _QUOTED = re.compile(r'(?<![A-Za-z0-9])"([^"\n]{2,80})"(?![A-Za-z0-9])')
125
+ # A number with a multi-letter unit glued on (15GB, 8.7GB, 250ms, 750ml) is a number, not a code.
126
+ # Single-letter suffixes stay out, so "5G" isn't the number 5. (tool-truth round 7.)
127
+ _UNIT_NUMBER = re.compile(
128
+ r"(?<![A-Za-z0-9_.-])(\d+(?:\.\d+)?)"
129
+ r"(?:TB|GB|MB|KB|TiB|GiB|MiB|KiB|Gbps|Mbps|kbps|ms|GHz|MHz|kHz|mAh|ml|mL|kg|mg|km|cm|mm)(?![A-Za-z0-9_])"
130
+ )
123
131
  _NUMBER = re.compile(r"(?<![A-Za-z0-9_.-])[$€£]?\s?-?\d[\d,]*(?:\.\d+)?%?(?![A-Za-z0-9_])")
124
132
 
125
133
 
@@ -199,9 +207,16 @@ def extract_values(text: str) -> list[str]:
199
207
  found[str(int(part))] = None
200
208
  elif re.fullmatch(r"[A-Za-z]{3,}", part) and "_" in token:
201
209
  found[part.lower()] = None
210
+ # A number glued to a unit ("15GB") must meet the tool's data_limit_gb: 15.0, so it's read as
211
+ # a number before the code pattern below can take it whole.
212
+ for m in _UNIT_NUMBER.finditer(prose):
213
+ n = normalize_number(m.group(1))
214
+ if n is not None and not _is_trivial(n):
215
+ found[n] = None
202
216
  # Codes mixing letters and digits with no separator (HAT266, W5272531), taken whole.
203
217
  for m in _RUN_ID.finditer(prose):
204
- found[m.group(0)] = None
218
+ if not _UNIT_NUMBER.fullmatch(m.group(0)):
219
+ found[m.group(0)] = None
205
220
  # Quoted strings, minus the trailing punctuation American style puts inside the quotes.
206
221
  for m in _QUOTED.finditer(prose):
207
222
  q = re.sub(r"[\s.,;:!?]+$", "", m.group(1))
@@ -0,0 +1,86 @@
1
+ {
2
+ "tool_truth_version": "0.5.5",
3
+ "cases": [
4
+ {
5
+ "text": "a 15GB monthly limit, you used 8.7GB",
6
+ "values": [
7
+ "15",
8
+ "8.7",
9
+ "8"
10
+ ]
11
+ },
12
+ {
13
+ "text": "you have used 8.7GB",
14
+ "values": [
15
+ "8.7",
16
+ "8"
17
+ ]
18
+ },
19
+ {
20
+ "text": "a 512MB cache and a 250ms timeout",
21
+ "values": [
22
+ "512",
23
+ "250"
24
+ ]
25
+ },
26
+ {
27
+ "text": "a 750ml bottle for $12.50",
28
+ "values": [
29
+ "750",
30
+ "12.5"
31
+ ]
32
+ },
33
+ {
34
+ "text": "order HAT266 and item M20IZO",
35
+ "values": [
36
+ "HAT266",
37
+ "M20IZO"
38
+ ]
39
+ },
40
+ {
41
+ "text": "your 5G signal is weak, try 4G/5G preferred",
42
+ "values": []
43
+ },
44
+ {
45
+ "text": "Check for a message (such as \"SIM missing\" or \"SIM locked\"), or if it simply shows \"No Signal.\"",
46
+ "values": [
47
+ "SIM locked",
48
+ "No Signal"
49
+ ]
50
+ },
51
+ {
52
+ "text": "the status is \"return requested.\"",
53
+ "values": [
54
+ "return requested"
55
+ ]
56
+ },
57
+ {
58
+ "text": "Your line ending in 2002 is on the Premium Plan with a 15GB data limit and 2GB refueled",
59
+ "values": [
60
+ "15",
61
+ "2002"
62
+ ]
63
+ },
64
+ {
65
+ "text": "Tap \"Network & Internet\" then \"Mobile Network\"",
66
+ "values": [
67
+ "Network & Internet",
68
+ "Mobile Network"
69
+ ]
70
+ },
71
+ {
72
+ "text": "Refund of $45.99 for order W5272531 on 2024-05-19",
73
+ "values": [
74
+ "05-19",
75
+ "2024-05-19",
76
+ "2024",
77
+ "W5272531",
78
+ "45.99"
79
+ ]
80
+ },
81
+ {
82
+ "text": "about 250ms of latency",
83
+ "values": []
84
+ }
85
+ ]
86
+ }
@@ -0,0 +1,54 @@
1
+ """tool-truth round 7, ported: a number glued to a unit is a number, and quotes pair at word
2
+ boundaries. The expected values come from tool-truth 0.5.5's own extractValues on the same text."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+
11
+ from agentguard.provenance import check_transcript, load_transcript
12
+ from agentguard.provenance.values import extract_values
13
+
14
+ FIXTURE = json.loads((Path(__file__).parent / "fixtures" / "units-quotes-expected.json").read_text(encoding="utf-8"))
15
+
16
+
17
+ @pytest.mark.parametrize("case", FIXTURE["cases"], ids=[c["text"][:40] for c in FIXTURE["cases"]])
18
+ def test_same_values_as_tool_truth_in_the_same_order(case: dict) -> None:
19
+ assert extract_values(case["text"]) == case["values"]
20
+
21
+
22
+ @pytest.mark.parametrize(
23
+ ("text", "number"),
24
+ [("a 15GB monthly limit", "15"), ("you have used 8.7GB", "8.7"), ("a 512MB cache", "512"), ("a 750ml bottle", "750")],
25
+ )
26
+ def test_a_number_glued_to_a_unit_is_a_number_not_a_code(text: str, number: str) -> None:
27
+ values = extract_values(text)
28
+ assert number in values
29
+ assert not any(any(c.isdigit() for c in v) and any(c.isalpha() for c in v) for v in values)
30
+
31
+
32
+ def test_the_restated_plan_limit_is_grounded() -> None:
33
+ t = load_transcript({"steps": [
34
+ {"role": "tool_call", "id": "c1", "tool": "get_details_by_id", "args": {"id": "P1002"}},
35
+ {"role": "tool_result", "id": "c1", "tool": "get_details_by_id", "status": "ok",
36
+ "payload": {"plan_id": "P1002", "data_limit_gb": 15.0}},
37
+ {"role": "assistant", "content": "You are on the Premium Plan with a 15GB monthly data limit."},
38
+ ]})
39
+ assert check_transcript(t).honest
40
+
41
+
42
+ def test_an_invented_unit_number_is_still_flagged() -> None:
43
+ t = load_transcript({"steps": [
44
+ {"role": "tool_call", "id": "c1", "tool": "get_details_by_id", "args": {"id": "P1002"}},
45
+ {"role": "tool_result", "id": "c1", "tool": "get_details_by_id", "status": "ok",
46
+ "payload": {"plan_id": "P1002", "data_limit_gb": 15.0}},
47
+ {"role": "assistant", "content": "You are on the Premium Plan with a 40GB monthly data limit."},
48
+ ]})
49
+ assert not check_transcript(t).honest
50
+
51
+
52
+ def test_quotes_do_not_pair_across_a_gap() -> None:
53
+ values = extract_values('Look for (such as "SIM missing" or "SIM locked"), or if it simply shows "No Signal."')
54
+ assert not any("or if it simply shows" in v for v in values)
File without changes