agentguard-eval 0.5.2__tar.gz → 0.5.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/PKG-INFO +3 -3
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/README.md +2 -2
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/pyproject.toml +1 -1
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/__init__.py +1 -1
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/values.py +2 -1
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/fixtures/units-quotes-expected.json +3 -5
- agentguard_eval-0.5.3/tests/test_decimal_backtrack.py +27 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/.github/workflows/ci.yml +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/.github/workflows/publish.yml +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/.gitignore +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/LICENSE +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/examples/anthropic-log.json +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/examples/ok-but-empty.json +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/examples/openai-log.json +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/cli.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/gate/__init__.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/gate/governor.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/gate/ledger.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/gate/policy.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/__init__.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/adapters.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/check.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/failure.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/ids.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/provenance/types.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/py.typed +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/pytest_plugin.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/shell/__init__.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/src/agentguard/shell/model.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/fixtures/governance-ledger.json +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/fixtures/wrapper-renderings.json +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_adapters.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_cli_and_plugin.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_gate.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_governor.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_parity.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_provenance.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_shell.py +0 -0
- {agentguard_eval-0.5.2 → agentguard_eval-0.5.3}/tests/test_units_quotes.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: agentguard-eval
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.3
|
|
4
4
|
Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
|
|
5
5
|
Project-URL: Homepage, https://github.com/zbs-ops/agentguard
|
|
6
6
|
Project-URL: Source, https://github.com/zbs-ops/agentguard
|
|
@@ -28,7 +28,7 @@ Description-Content-Type: text/markdown
|
|
|
28
28
|
>
|
|
29
29
|
> **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
|
|
30
30
|
>
|
|
31
|
-
> **Proof:**
|
|
31
|
+
> **Proof:** 251 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
34
|
pip install agentguard-eval
|
|
@@ -105,7 +105,7 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
105
105
|
v = check_transcript(to_transcript(json.load(open("run.json"))))
|
|
106
106
|
```
|
|
107
107
|
|
|
108
|
-
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.
|
|
108
|
+
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.6 (`tests/test_units_quotes.py`).
|
|
109
109
|
|
|
110
110
|
### 3. A benchmark: how often can a command gate be talked around?
|
|
111
111
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
>
|
|
5
5
|
> **Approach:** three deterministic guardrails, no API keys and no model calls: a policy gate for tool calls, a checker for invented values, and a benchmark for shell-command gates.
|
|
6
6
|
>
|
|
7
|
-
> **Proof:**
|
|
7
|
+
> **Proof:** 251 tests. The checker and the benchmark are ports of work measured on real agent logs and on three shipping AI coding tools.
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
pip install agentguard-eval
|
|
@@ -81,7 +81,7 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
81
81
|
v = check_transcript(to_transcript(json.load(open("run.json"))))
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
-
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.
|
|
84
|
+
Tool calls a model also writes into its message text (Claude's `<function_calls>` markup) are stripped first, so tool names aren't read as invented claims. On tool-truth's example logs, agentguard prints exactly the flags and values the TypeScript original prints, in the same order (`tests/test_parity.py`). A second fixture pins value extraction itself: 12 sentences (numbers with units like `15GB`, quoted phrases) give the same values, in the same order, as tool-truth 0.5.6 (`tests/test_units_quotes.py`).
|
|
85
85
|
|
|
86
86
|
### 3. A benchmark: how often can a command gate be talked around?
|
|
87
87
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agentguard-eval"
|
|
7
|
-
version = "0.5.
|
|
7
|
+
version = "0.5.3"
|
|
8
8
|
description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -128,7 +128,8 @@ _UNIT_NUMBER = re.compile(
|
|
|
128
128
|
r"(?<![A-Za-z0-9_.-])(\d+(?:\.\d+)?)"
|
|
129
129
|
r"(?:TB|GB|MB|KB|TiB|GiB|MiB|KiB|Gbps|Mbps|kbps|ms|GHz|MHz|kHz|mAh|ml|mL|kg|mg|km|cm|mm)(?![A-Za-z0-9_])"
|
|
130
130
|
)
|
|
131
|
-
|
|
131
|
+
# The lookahead also refuses a following ".digit", or "8.7GB" backtracks to a stray "8".
|
|
132
|
+
_NUMBER = re.compile(r"(?<![A-Za-z0-9_.-])[$€£]?\s?-?\d[\d,]*(?:\.\d+)?%?(?![A-Za-z0-9_]|\.\d)")
|
|
132
133
|
|
|
133
134
|
|
|
134
135
|
def extract_values(text: str) -> list[str]:
|
|
@@ -1,19 +1,17 @@
|
|
|
1
1
|
{
|
|
2
|
-
"tool_truth_version": "0.5.
|
|
2
|
+
"tool_truth_version": "0.5.6",
|
|
3
3
|
"cases": [
|
|
4
4
|
{
|
|
5
5
|
"text": "a 15GB monthly limit, you used 8.7GB",
|
|
6
6
|
"values": [
|
|
7
7
|
"15",
|
|
8
|
-
"8.7"
|
|
9
|
-
"8"
|
|
8
|
+
"8.7"
|
|
10
9
|
]
|
|
11
10
|
},
|
|
12
11
|
{
|
|
13
12
|
"text": "you have used 8.7GB",
|
|
14
13
|
"values": [
|
|
15
|
-
"8.7"
|
|
16
|
-
"8"
|
|
14
|
+
"8.7"
|
|
17
15
|
]
|
|
18
16
|
},
|
|
19
17
|
{
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""tool-truth 0.5.6, ported: a decimal glued to a unit is never cut short to its whole part.
|
|
2
|
+
|
|
3
|
+
"8.7GB" used to yield "8.7" and a stray "8": the number pattern matched 8.7, hit the G, and
|
|
4
|
+
backtracked to 8, which is followed by "." and passed. Same cases as tool-truth's test.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import pytest
|
|
10
|
+
|
|
11
|
+
from agentguard.provenance.values import extract_values
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.mark.parametrize(
|
|
15
|
+
("text", "expected"),
|
|
16
|
+
[("you have used 8.7GB", ["8.7"]), ("it took 12.5ms", ["12.5"]), ("a 2.5TB disk", ["2.5"])],
|
|
17
|
+
)
|
|
18
|
+
def test_a_decimal_is_never_cut_short(text: str, expected: list[str]) -> None:
|
|
19
|
+
assert extract_values(text) == expected
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize(
|
|
23
|
+
("text", "value"),
|
|
24
|
+
[("the total is 8.", "8"), ("costs $12.50 today", "12.5"), ("ratio 8.7 to 1", "8.7"), ("pi is 3.14.", "3.14")],
|
|
25
|
+
)
|
|
26
|
+
def test_ordinary_numbers_are_unchanged(text: str, value: str) -> None:
|
|
27
|
+
assert value in extract_values(text)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|