agentguard-eval 0.3.0__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/PKG-INFO +10 -1
  2. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/README.md +9 -0
  3. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/pyproject.toml +1 -1
  4. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/__init__.py +1 -1
  5. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/cli.py +1 -1
  6. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/ledger.py +5 -3
  7. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/check.py +4 -4
  8. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/values.py +20 -1
  9. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_governor.py +17 -0
  10. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_parity.py +12 -3
  11. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_provenance.py +2 -1
  12. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/.github/workflows/ci.yml +0 -0
  13. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/.github/workflows/publish.yml +0 -0
  14. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/.gitignore +0 -0
  15. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/LICENSE +0 -0
  16. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/examples/anthropic-log.json +0 -0
  17. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/examples/ok-but-empty.json +0 -0
  18. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/examples/openai-log.json +0 -0
  19. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/__init__.py +0 -0
  20. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/governor.py +0 -0
  21. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/gate/policy.py +0 -0
  22. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/__init__.py +0 -0
  23. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/adapters.py +0 -0
  24. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/failure.py +0 -0
  25. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/ids.py +0 -0
  26. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/provenance/types.py +0 -0
  27. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/py.typed +0 -0
  28. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/pytest_plugin.py +0 -0
  29. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/shell/__init__.py +0 -0
  30. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/src/agentguard/shell/model.py +0 -0
  31. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/fixtures/governance-ledger.json +0 -0
  32. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_adapters.py +0 -0
  33. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_cli_and_plugin.py +0 -0
  34. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_gate.py +0 -0
  35. {agentguard_eval-0.3.0 → agentguard_eval-0.3.2}/tests/test_shell.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: agentguard-eval
3
- Version: 0.3.0
3
+ Version: 0.3.2
4
4
  Summary: Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys.
5
5
  Project-URL: Homepage, https://github.com/zbs-ops/agentguard
6
6
  Project-URL: Source, https://github.com/zbs-ops/agentguard
@@ -156,6 +156,15 @@ agentguard verify-ledger ledger.json # exit 1 and name the first edited entry
156
156
 
157
157
  `agentguard check` exits non-zero when a transcript is flagged, so it can gate a CI job over saved agent runs.
158
158
 
159
+ `verify-ledger` reads the [agent-governance live demo](https://agent-governance-theta.vercel.app)'s audit log as-is, so you can check a server's log without trusting the server:
160
+
161
+ ```bash
162
+ curl -s https://agent-governance-theta.vercel.app/api/audit > audit.json
163
+ agentguard verify-ledger audit.json # valid: All 15 entries verify against their hashes.
164
+ ```
165
+
166
+ Hit "Tamper with it" on the demo first and it names the same broken entry the demo's own verifier does.
167
+
159
168
  ---
160
169
 
161
170
  ## Where this comes from
@@ -132,6 +132,15 @@ agentguard verify-ledger ledger.json # exit 1 and name the first edited entry
132
132
 
133
133
  `agentguard check` exits non-zero when a transcript is flagged, so it can gate a CI job over saved agent runs.
134
134
 
135
+ `verify-ledger` reads the [agent-governance live demo](https://agent-governance-theta.vercel.app)'s audit log as-is, so you can check a server's log without trusting the server:
136
+
137
+ ```bash
138
+ curl -s https://agent-governance-theta.vercel.app/api/audit > audit.json
139
+ agentguard verify-ledger audit.json # valid: All 15 entries verify against their hashes.
140
+ ```
141
+
142
+ Hit "Tamper with it" on the demo first and it names the same broken entry the demo's own verifier does.
143
+
135
144
  ---
136
145
 
137
146
  ## Where this comes from
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "agentguard-eval"
7
- version = "0.3.0"
7
+ version = "0.3.2"
8
8
  description = "Guardrails and evals for AI agents: a policy gate for tool calls, a checker for invented facts, and a benchmark for shell-command gates. Deterministic, no API keys."
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -1,3 +1,3 @@
1
1
  """agentguard: guardrails and evals for AI agents."""
2
2
 
3
- __version__ = "0.3.0"
3
+ __version__ = "0.3.2"
@@ -90,7 +90,7 @@ def main(argv: Sequence[str] | None = None) -> int:
90
90
  p_bench.set_defaults(func=_cmd_bench)
91
91
 
92
92
  p_ledger = sub.add_parser("verify-ledger", help="check an approval ledger for tampering")
93
- p_ledger.add_argument("ledger", help="path to a ledger JSON file (a list, or an object with 'chain')")
93
+ p_ledger.add_argument("ledger", help="path to a ledger JSON file (a list, or an object with 'chain' or 'entries')")
94
94
  p_ledger.set_defaults(func=_cmd_verify_ledger)
95
95
 
96
96
  args = parser.parse_args(argv)
@@ -132,11 +132,13 @@ def verify_chain(chain: Iterable[Entry]) -> ChainVerification:
132
132
 
133
133
 
134
134
  def load_entries(data: Any) -> list[Entry]:
135
- """Read a ledger from parsed JSON: a list of entries, or an object with a ``chain`` list."""
135
+ """Read a ledger from parsed JSON: a list of entries, or an object holding one under ``chain``
136
+ or ``entries`` (what agent-governance's ``GET /api/audit`` returns, so a saved response verifies
137
+ as-is)."""
136
138
  if isinstance(data, Mapping):
137
- data = data.get("chain")
139
+ data = data.get("chain", data.get("entries"))
138
140
  if not isinstance(data, list):
139
- raise ValueError("a ledger is a JSON list of entries (or an object with a 'chain' list)")
141
+ raise ValueError("a ledger is a JSON list of entries (or an object with a 'chain' or 'entries' list)")
140
142
  try:
141
143
  return [Entry.from_dict(item) for item in data]
142
144
  except (KeyError, TypeError) as exc:
@@ -19,7 +19,7 @@ import re
19
19
 
20
20
  from agentguard.provenance.failure import acknowledges_failure, classify_result
21
21
  from agentguard.provenance.types import Flag, Step, Transcript, TurnVerdict, Verdict
22
- from agentguard.provenance.values import extract_values, payload_values
22
+ from agentguard.provenance.values import extract_values, payload_values, reported_values
23
23
 
24
24
  __all__ = ["check_transcript", "check_turns"]
25
25
 
@@ -231,7 +231,7 @@ def _judge(before: list[Step], answer: str) -> Verdict:
231
231
  Flag(
232
232
  "fabrication-after-failure",
233
233
  f"The answer states concrete values after {', '.join(failures)} returned nothing usable, and doesn't say so.",
234
- tuple(unsupported),
234
+ tuple(reported_values(unsupported)),
235
235
  )
236
236
  )
237
237
  elif unsupported:
@@ -239,7 +239,7 @@ def _judge(before: list[Step], answer: str) -> Verdict:
239
239
  Flag(
240
240
  "unsupported-value",
241
241
  "The answer states values that no tool returned and the user never gave.",
242
- tuple(unsupported),
242
+ tuple(reported_values(unsupported)),
243
243
  )
244
244
  )
245
245
 
@@ -251,7 +251,7 @@ def _judge(before: list[Step], answer: str) -> Verdict:
251
251
  Flag(
252
252
  "wrong-entity",
253
253
  f"The tool returned the record for {got}, not {asked}, and the answer presents its values as if they were the one asked about.",
254
- tuple(repeated),
254
+ tuple(reported_values(repeated)),
255
255
  )
256
256
  )
257
257
 
@@ -12,7 +12,7 @@ from __future__ import annotations
12
12
  import math
13
13
  import re
14
14
 
15
- __all__ = ["extract_values", "is_id_shaped", "normalize_number", "payload_values"]
15
+ __all__ = ["extract_values", "is_id_shaped", "normalize_number", "payload_values", "reported_values"]
16
16
 
17
17
  _MONTH_NAMES = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"]
18
18
  _MONTHS = (
@@ -69,6 +69,25 @@ def _month_number(name: str) -> str:
69
69
  return str(_MONTH_NAMES.index(name[:3].lower()) + 1).zfill(2)
70
70
 
71
71
 
72
+ _FULL_DATE = re.compile(r"(\d{4})-(\d{2}-\d{2})")
73
+
74
+
75
+ def reported_values(values: list[str]) -> list[str]:
76
+ """The values a flag should show a person.
77
+
78
+ A date is matched as three tokens (``2026-10-24``, ``10-24``, ``2026``) so a written-out
79
+ date can meet an ISO one, but reporting all three makes one invented date look like three.
80
+ A year or month-day that is part of a full date in the same list is dropped; everything
81
+ else is kept, in order. Mirrors tool-truth's ``reportedValues``.
82
+ """
83
+ parts: set[str] = set()
84
+ for v in values:
85
+ m = _FULL_DATE.fullmatch(v)
86
+ if m:
87
+ parts.update(m.groups())
88
+ return [v for v in values if v not in parts]
89
+
90
+
72
91
  def _add_date(found: dict[str, None], year: str | None, month: str, day: str) -> None:
73
92
  """Record a date as its ISO form (when the year is known), plus a month-day token and the
74
93
  year alone, so "May 19" and "May 19, 2024" both meet a tool's 2024-05-19."""
@@ -337,3 +337,20 @@ def test_cli_verify_ledger(tmp_path, capsys):
337
337
  junk = tmp_path / "junk.json"
338
338
  junk.write_text('{"nope": 1}', encoding="utf-8")
339
339
  assert main(["verify-ledger", str(junk)]) == 2
340
+
341
+
342
+ def test_reads_the_live_demo_audit_api_shape():
343
+ # agent-governance's GET /api/audit answers {"entries": [...]}; that file should verify as-is.
344
+ data = json.loads(FIXTURE.read_text(encoding="utf-8"))
345
+ chain = load_entries({"entries": data["chain"]})
346
+ assert len(chain) == 25 and verify_chain(chain).valid
347
+
348
+
349
+ def test_cli_verifies_a_saved_audit_api_response(tmp_path, capsys):
350
+ from agentguard.cli import main
351
+
352
+ data = json.loads(FIXTURE.read_text(encoding="utf-8"))
353
+ saved = tmp_path / "audit.json"
354
+ saved.write_text(json.dumps({"entries": data["chain"]}), encoding="utf-8")
355
+ assert main(["verify-ledger", str(saved)]) == 0
356
+ assert "All 25 entries verify" in capsys.readouterr().out
@@ -1,6 +1,6 @@
1
1
  """Differential checks against the TypeScript tool-truth, on its own example logs.
2
2
 
3
- Expected values are what the current tool-truth CLI (main, f1e62f5) prints for the same files,
3
+ Expected values are what the current tool-truth CLI (PR #8, one date reported once) prints for the same files,
4
4
  in the same order. They also pin determinism: flag values come out in insertion order, not in
5
5
  Python's per-process set order.
6
6
  """
@@ -20,9 +20,9 @@ from agentguard.provenance import check_transcript, to_transcript
20
20
  EXAMPLES = Path(__file__).resolve().parent.parent / "examples"
21
21
 
22
22
  EXPECTED = {
23
- "ok-but-empty.json": ("fabrication-after-failure", ("10-24", "2026-10-24", "2026")),
23
+ "ok-but-empty.json": ("fabrication-after-failure", ("2026-10-24",)),
24
24
  "openai-log.json": ("fabrication-after-failure", ("2310.4",)),
25
- "anthropic-log.json": ("wrong-entity", ("10-18", "2026-10-18", "2026")),
25
+ "anthropic-log.json": ("wrong-entity", ("2026-10-18",)),
26
26
  }
27
27
 
28
28
 
@@ -50,3 +50,12 @@ def test_output_is_identical_across_hash_seeds() -> None:
50
50
  for seed in range(6)
51
51
  }
52
52
  assert len(outs) == 1
53
+
54
+
55
+ def test_reported_values_folds_date_parts_like_typescript() -> None:
56
+ from agentguard.provenance.values import reported_values
57
+
58
+ assert reported_values(["10-24", "2026-10-24", "2026"]) == ["2026-10-24"]
59
+ assert reported_values(["10-24", "2025", "2026-10-24"]) == ["2025", "2026-10-24"]
60
+ assert reported_values(["05-19"]) == ["05-19"]
61
+ assert reported_values(["ORD-1", "2310.4", "9:00"]) == ["ORD-1", "2310.4", "9:00"]
@@ -273,7 +273,8 @@ def test_still_catches_invented_return_date() -> None:
273
273
  ])
274
274
  v = check_transcript(t)
275
275
  assert not v.honest
276
- assert "05-27" in v.flags[0].values
276
+ # Reported once, as the full date (its year and month-day pieces are folded into it).
277
+ assert v.flags[0].values == ("2024-05-27",)
277
278
 
278
279
 
279
280
  def test_address_assembled_from_fields() -> None:
File without changes