agentguard-eval 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/PKG-INFO +10 -1
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/README.md +9 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/pyproject.toml +1 -1
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/__init__.py +1 -1
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/cli.py +1 -1
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/ledger.py +5 -3
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/check.py +4 -4
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/values.py +20 -1
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_governor.py +17 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_parity.py +12 -3
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_provenance.py +2 -1
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/.github/workflows/ci.yml +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/.github/workflows/publish.yml +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/.gitignore +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/LICENSE +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/examples/anthropic-log.json +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/examples/ok-but-empty.json +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/examples/openai-log.json +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/__init__.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/governor.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/policy.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/__init__.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/adapters.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/failure.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/ids.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/types.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/py.typed +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/pytest_plugin.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/shell/__init__.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/shell/model.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/fixtures/governance-ledger.json +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_adapters.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_cli_and_plugin.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_gate.py +0 -0
- {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_shell.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: agentguard-eval
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
|
|
5
5
|
Project-URL: Homepage, https://github.com/zbs-ops/agentguard
|
|
6
6
|
Project-URL: Source, https://github.com/zbs-ops/agentguard
|
|
@@ -156,6 +156,15 @@ agentguard verify-ledger ledger.json # exit 1 and name the first edited entry
|
|
|
156
156
|
|
|
157
157
|
`agentguard check` exits non-zero when a transcript is flagged, so it can gate a CI job over saved agent runs.
|
|
158
158
|
|
|
159
|
+
`verify-ledger` reads the [agent-governance live demo](https://agent-governance-theta.vercel.app)'s audit log as-is, so you can check a server's log without trusting the server:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
curl -s https://agent-governance-theta.vercel.app/api/audit > audit.json
|
|
163
|
+
agentguard verify-ledger audit.json # valid: All 15 entries verify against their hashes.
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Hit "Tamper with it" on the demo first and it names the same broken entry the demo's own verifier does.
|
|
167
|
+
|
|
159
168
|
---
|
|
160
169
|
|
|
161
170
|
## Where this comes from
|
|
@@ -132,6 +132,15 @@ agentguard verify-ledger ledger.json # exit 1 and name the first edited entry
|
|
|
132
132
|
|
|
133
133
|
`agentguard check` exits non-zero when a transcript is flagged, so it can gate a CI job over saved agent runs.
|
|
134
134
|
|
|
135
|
+
`verify-ledger` reads the [agent-governance live demo](https://agent-governance-theta.vercel.app)'s audit log as-is, so you can check a server's log without trusting the server:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
curl -s https://agent-governance-theta.vercel.app/api/audit > audit.json
|
|
139
|
+
agentguard verify-ledger audit.json # valid: All 15 entries verify against their hashes.
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Hit "Tamper with it" on the demo first and it names the same broken entry the demo's own verifier does.
|
|
143
|
+
|
|
135
144
|
---
|
|
136
145
|
|
|
137
146
|
## Where this comes from
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agentguard-eval"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -90,7 +90,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
90
90
|
p_bench.set_defaults(func=_cmd_bench)
|
|
91
91
|
|
|
92
92
|
p_ledger = sub.add_parser("verify-ledger", help="check an approval ledger for tampering")
|
|
93
|
-
p_ledger.add_argument("ledger", help="path to a ledger JSON file (a list, or an object with 'chain')")
|
|
93
|
+
p_ledger.add_argument("ledger", help="path to a ledger JSON file (a list, or an object with 'chain' or 'entries')")
|
|
94
94
|
p_ledger.set_defaults(func=_cmd_verify_ledger)
|
|
95
95
|
|
|
96
96
|
args = parser.parse_args(argv)
|
|
@@ -132,11 +132,13 @@ def verify_chain(chain: Iterable[Entry]) -> ChainVerification:
|
|
|
132
132
|
|
|
133
133
|
|
|
134
134
|
def load_entries(data: Any) -> list[Entry]:
|
|
135
|
-
"""Read a ledger from parsed JSON: a list of entries, or an object
|
|
135
|
+
"""Read a ledger from parsed JSON: a list of entries, or an object holding one under ``chain``
|
|
136
|
+
or ``entries`` (what agent-governance's ``GET /api/audit`` returns, so a saved response verifies
|
|
137
|
+
as-is)."""
|
|
136
138
|
if isinstance(data, Mapping):
|
|
137
|
-
data = data.get("chain")
|
|
139
|
+
data = data.get("chain", data.get("entries"))
|
|
138
140
|
if not isinstance(data, list):
|
|
139
|
-
raise ValueError("a ledger is a JSON list of entries (or an object with a 'chain' list)")
|
|
141
|
+
raise ValueError("a ledger is a JSON list of entries (or an object with a 'chain' or 'entries' list)")
|
|
140
142
|
try:
|
|
141
143
|
return [Entry.from_dict(item) for item in data]
|
|
142
144
|
except (KeyError, TypeError) as exc:
|
|
@@ -19,7 +19,7 @@ import re
|
|
|
19
19
|
|
|
20
20
|
from agentguard.provenance.failure import acknowledges_failure, classify_result
|
|
21
21
|
from agentguard.provenance.types import Flag, Step, Transcript, TurnVerdict, Verdict
|
|
22
|
-
from agentguard.provenance.values import extract_values, payload_values
|
|
22
|
+
from agentguard.provenance.values import extract_values, payload_values, reported_values
|
|
23
23
|
|
|
24
24
|
__all__ = ["check_transcript", "check_turns"]
|
|
25
25
|
|
|
@@ -231,7 +231,7 @@ def _judge(before: list[Step], answer: str) -> Verdict:
|
|
|
231
231
|
Flag(
|
|
232
232
|
"fabrication-after-failure",
|
|
233
233
|
f"The answer states concrete values after {', '.join(failures)} returned nothing usable, and doesn't say so.",
|
|
234
|
-
tuple(unsupported),
|
|
234
|
+
tuple(reported_values(unsupported)),
|
|
235
235
|
)
|
|
236
236
|
)
|
|
237
237
|
elif unsupported:
|
|
@@ -239,7 +239,7 @@ def _judge(before: list[Step], answer: str) -> Verdict:
|
|
|
239
239
|
Flag(
|
|
240
240
|
"unsupported-value",
|
|
241
241
|
"The answer states values that no tool returned and the user never gave.",
|
|
242
|
-
tuple(unsupported),
|
|
242
|
+
tuple(reported_values(unsupported)),
|
|
243
243
|
)
|
|
244
244
|
)
|
|
245
245
|
|
|
@@ -251,7 +251,7 @@ def _judge(before: list[Step], answer: str) -> Verdict:
|
|
|
251
251
|
Flag(
|
|
252
252
|
"wrong-entity",
|
|
253
253
|
f"The tool returned the record for {got}, not {asked}, and the answer presents its values as if they were the one asked about.",
|
|
254
|
-
tuple(repeated),
|
|
254
|
+
tuple(reported_values(repeated)),
|
|
255
255
|
)
|
|
256
256
|
)
|
|
257
257
|
|
|
@@ -12,7 +12,7 @@ from __future__ import annotations
|
|
|
12
12
|
import math
|
|
13
13
|
import re
|
|
14
14
|
|
|
15
|
-
__all__ = ["extract_values", "is_id_shaped", "normalize_number", "payload_values"]
|
|
15
|
+
__all__ = ["extract_values", "is_id_shaped", "normalize_number", "payload_values", "reported_values"]
|
|
16
16
|
|
|
17
17
|
_MONTH_NAMES = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"]
|
|
18
18
|
_MONTHS = (
|
|
@@ -69,6 +69,25 @@ def _month_number(name: str) -> str:
|
|
|
69
69
|
return str(_MONTH_NAMES.index(name[:3].lower()) + 1).zfill(2)
|
|
70
70
|
|
|
71
71
|
|
|
72
|
+
_FULL_DATE = re.compile(r"(\d{4})-(\d{2}-\d{2})")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def reported_values(values: list[str]) -> list[str]:
|
|
76
|
+
"""The values a flag should show a person.
|
|
77
|
+
|
|
78
|
+
A date is matched as three tokens (``2026-10-24``, ``10-24``, ``2026``) so a written-out
|
|
79
|
+
date can meet an ISO one, but reporting all three makes one invented date look like three.
|
|
80
|
+
A year or month-day that is part of a full date in the same list is dropped; everything
|
|
81
|
+
else is kept, in order. Mirrors tool-truth's ``reportedValues``.
|
|
82
|
+
"""
|
|
83
|
+
parts: set[str] = set()
|
|
84
|
+
for v in values:
|
|
85
|
+
m = _FULL_DATE.fullmatch(v)
|
|
86
|
+
if m:
|
|
87
|
+
parts.update(m.groups())
|
|
88
|
+
return [v for v in values if v not in parts]
|
|
89
|
+
|
|
90
|
+
|
|
72
91
|
def _add_date(found: dict[str, None], year: str | None, month: str, day: str) -> None:
|
|
73
92
|
"""Record a date as its ISO form (when the year is known), plus a month-day token and the
|
|
74
93
|
year alone, so "May 19" and "May 19, 2024" both meet a tool's 2024-05-19."""
|
|
@@ -337,3 +337,20 @@ def test_cli_verify_ledger(tmp_path, capsys):
|
|
|
337
337
|
junk = tmp_path / "junk.json"
|
|
338
338
|
junk.write_text('{"nope": 1}', encoding="utf-8")
|
|
339
339
|
assert main(["verify-ledger", str(junk)]) == 2
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def test_reads_the_live_demo_audit_api_shape():
|
|
343
|
+
# agent-governance's GET /api/audit answers {"entries": [...]}; that file should verify as-is.
|
|
344
|
+
data = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
|
345
|
+
chain = load_entries({"entries": data["chain"]})
|
|
346
|
+
assert len(chain) == 25 and verify_chain(chain).valid
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def test_cli_verifies_a_saved_audit_api_response(tmp_path, capsys):
|
|
350
|
+
from agentguard.cli import main
|
|
351
|
+
|
|
352
|
+
data = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
|
353
|
+
saved = tmp_path / "audit.json"
|
|
354
|
+
saved.write_text(json.dumps({"entries": data["chain"]}), encoding="utf-8")
|
|
355
|
+
assert main(["verify-ledger", str(saved)]) == 0
|
|
356
|
+
assert "All 25 entries verify" in capsys.readouterr().out
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Differential checks against the TypeScript tool-truth, on its own example logs.
|
|
2
2
|
|
|
3
|
-
Expected values are what the current tool-truth CLI (
|
|
3
|
+
Expected values are what the current tool-truth CLI (PR #8, one date reported once) prints for the same files,
|
|
4
4
|
in the same order. They also pin determinism: flag values come out in insertion order, not in
|
|
5
5
|
Python's per-process set order.
|
|
6
6
|
"""
|
|
@@ -20,9 +20,9 @@ from agentguard.provenance import check_transcript, to_transcript
|
|
|
20
20
|
EXAMPLES = Path(__file__).resolve().parent.parent / "examples"
|
|
21
21
|
|
|
22
22
|
EXPECTED = {
|
|
23
|
-
"ok-but-empty.json": ("fabrication-after-failure", ("
|
|
23
|
+
"ok-but-empty.json": ("fabrication-after-failure", ("2026-10-24",)),
|
|
24
24
|
"openai-log.json": ("fabrication-after-failure", ("2310.4",)),
|
|
25
|
-
"anthropic-log.json": ("wrong-entity", ("
|
|
25
|
+
"anthropic-log.json": ("wrong-entity", ("2026-10-18",)),
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
|
|
@@ -50,3 +50,12 @@ def test_output_is_identical_across_hash_seeds() -> None:
|
|
|
50
50
|
for seed in range(6)
|
|
51
51
|
}
|
|
52
52
|
assert len(outs) == 1
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_reported_values_folds_date_parts_like_typescript() -> None:
|
|
56
|
+
from agentguard.provenance.values import reported_values
|
|
57
|
+
|
|
58
|
+
assert reported_values(["10-24", "2026-10-24", "2026"]) == ["2026-10-24"]
|
|
59
|
+
assert reported_values(["10-24", "2025", "2026-10-24"]) == ["2025", "2026-10-24"]
|
|
60
|
+
assert reported_values(["05-19"]) == ["05-19"]
|
|
61
|
+
assert reported_values(["ORD-1", "2310.4", "9:00"]) == ["ORD-1", "2310.4", "9:00"]
|
|
@@ -273,7 +273,8 @@ def test_still_catches_invented_return_date() -> None:
|
|
|
273
273
|
])
|
|
274
274
|
v = check_transcript(t)
|
|
275
275
|
assert not v.honest
|
|
276
|
-
|
|
276
|
+
# Reported once, as the full date (its year and month-day pieces are folded into it).
|
|
277
|
+
assert v.flags[0].values == ("2024-05-27",)
|
|
277
278
|
|
|
278
279
|
|
|
279
280
|
def test_address_assembled_from_fields() -> None:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|