agentguard-eval 0.3.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/PKG-INFO +10 -1
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/README.md +9 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/pyproject.toml +1 -1
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/__init__.py +1 -1
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/cli.py +1 -1
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/gate/ledger.py +5 -3
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_governor.py +17 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/.github/workflows/ci.yml +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/.github/workflows/publish.yml +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/.gitignore +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/LICENSE +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/examples/anthropic-log.json +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/examples/ok-but-empty.json +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/examples/openai-log.json +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/gate/__init__.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/gate/governor.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/gate/policy.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/__init__.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/adapters.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/check.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/failure.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/ids.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/types.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/provenance/values.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/py.typed +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/pytest_plugin.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/shell/__init__.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/src/agentguard/shell/model.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/fixtures/governance-ledger.json +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_adapters.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_cli_and_plugin.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_gate.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_parity.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_provenance.py +0 -0
- {agentguard_eval-0.3.1 → agentguard_eval-0.3.2}/tests/test_shell.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: agentguard-eval
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
|
|
5
5
|
Project-URL: Homepage, https://github.com/zbs-ops/agentguard
|
|
6
6
|
Project-URL: Source, https://github.com/zbs-ops/agentguard
|
|
@@ -156,6 +156,15 @@ agentguard verify-ledger ledger.json # exit 1 and name the first edited entry
|
|
|
156
156
|
|
|
157
157
|
`agentguard check` exits non-zero when a transcript is flagged, so it can gate a CI job over saved agent runs.
|
|
158
158
|
|
|
159
|
+
`verify-ledger` reads the [agent-governance live demo](https://agent-governance-theta.vercel.app)'s audit log as-is, so you can check a server's log without trusting the server:
|
|
160
|
+
|
|
161
|
+
```bash
|
|
162
|
+
curl -s https://agent-governance-theta.vercel.app/api/audit > audit.json
|
|
163
|
+
agentguard verify-ledger audit.json # valid: All 15 entries verify against their hashes.
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
Hit "Tamper with it" on the demo first and it names the same broken entry the demo's own verifier does.
|
|
167
|
+
|
|
159
168
|
---
|
|
160
169
|
|
|
161
170
|
## Where this comes from
|
|
@@ -132,6 +132,15 @@ agentguard verify-ledger ledger.json # exit 1 and name the first edited entry
|
|
|
132
132
|
|
|
133
133
|
`agentguard check` exits non-zero when a transcript is flagged, so it can gate a CI job over saved agent runs.
|
|
134
134
|
|
|
135
|
+
`verify-ledger` reads the [agent-governance live demo](https://agent-governance-theta.vercel.app)'s audit log as-is, so you can check a server's log without trusting the server:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
curl -s https://agent-governance-theta.vercel.app/api/audit > audit.json
|
|
139
|
+
agentguard verify-ledger audit.json # valid: All 15 entries verify against their hashes.
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Hit "Tamper with it" on the demo first and it names the same broken entry the demo's own verifier does.
|
|
143
|
+
|
|
135
144
|
---
|
|
136
145
|
|
|
137
146
|
## Where this comes from
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agentguard-eval"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -90,7 +90,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
90
90
|
p_bench.set_defaults(func=_cmd_bench)
|
|
91
91
|
|
|
92
92
|
p_ledger = sub.add_parser("verify-ledger", help="check an approval ledger for tampering")
|
|
93
|
-
p_ledger.add_argument("ledger", help="path to a ledger JSON file (a list, or an object with 'chain')")
|
|
93
|
+
p_ledger.add_argument("ledger", help="path to a ledger JSON file (a list, or an object with 'chain' or 'entries')")
|
|
94
94
|
p_ledger.set_defaults(func=_cmd_verify_ledger)
|
|
95
95
|
|
|
96
96
|
args = parser.parse_args(argv)
|
|
@@ -132,11 +132,13 @@ def verify_chain(chain: Iterable[Entry]) -> ChainVerification:
|
|
|
132
132
|
|
|
133
133
|
|
|
134
134
|
def load_entries(data: Any) -> list[Entry]:
|
|
135
|
-
"""Read a ledger from parsed JSON: a list of entries, or an object
|
|
135
|
+
"""Read a ledger from parsed JSON: a list of entries, or an object holding one under ``chain``
|
|
136
|
+
or ``entries`` (what agent-governance's ``GET /api/audit`` returns, so a saved response verifies
|
|
137
|
+
as-is)."""
|
|
136
138
|
if isinstance(data, Mapping):
|
|
137
|
-
data = data.get("chain")
|
|
139
|
+
data = data.get("chain", data.get("entries"))
|
|
138
140
|
if not isinstance(data, list):
|
|
139
|
-
raise ValueError("a ledger is a JSON list of entries (or an object with a 'chain' list)")
|
|
141
|
+
raise ValueError("a ledger is a JSON list of entries (or an object with a 'chain' or 'entries' list)")
|
|
140
142
|
try:
|
|
141
143
|
return [Entry.from_dict(item) for item in data]
|
|
142
144
|
except (KeyError, TypeError) as exc:
|
|
@@ -337,3 +337,20 @@ def test_cli_verify_ledger(tmp_path, capsys):
|
|
|
337
337
|
junk = tmp_path / "junk.json"
|
|
338
338
|
junk.write_text('{"nope": 1}', encoding="utf-8")
|
|
339
339
|
assert main(["verify-ledger", str(junk)]) == 2
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def test_reads_the_live_demo_audit_api_shape():
|
|
343
|
+
# agent-governance's GET /api/audit answers {"entries": [...]}; that file should verify as-is.
|
|
344
|
+
data = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
|
345
|
+
chain = load_entries({"entries": data["chain"]})
|
|
346
|
+
assert len(chain) == 25 and verify_chain(chain).valid
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def test_cli_verifies_a_saved_audit_api_response(tmp_path, capsys):
|
|
350
|
+
from agentguard.cli import main
|
|
351
|
+
|
|
352
|
+
data = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
|
353
|
+
saved = tmp_path / "audit.json"
|
|
354
|
+
saved.write_text(json.dumps({"entries": data["chain"]}), encoding="utf-8")
|
|
355
|
+
assert main(["verify-ledger", str(saved)]) == 0
|
|
356
|
+
assert "All 25 entries verify" in capsys.readouterr().out
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|