agent-invariant-eval 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_invariant_eval-0.3.0/.github/ISSUE_TEMPLATE/bug_report.md +26 -0
- agent_invariant_eval-0.3.0/.github/ISSUE_TEMPLATE/feature_request.md +15 -0
- agent_invariant_eval-0.3.0/.github/workflows/ci.yml +27 -0
- agent_invariant_eval-0.3.0/.github/workflows/release.yml +38 -0
- agent_invariant_eval-0.3.0/.gitignore +12 -0
- agent_invariant_eval-0.3.0/CHANGELOG.md +97 -0
- agent_invariant_eval-0.3.0/CITATION.cff +18 -0
- agent_invariant_eval-0.3.0/CONTRIBUTING.md +28 -0
- agent_invariant_eval-0.3.0/LICENSE +21 -0
- agent_invariant_eval-0.3.0/PKG-INFO +593 -0
- agent_invariant_eval-0.3.0/README.md +532 -0
- agent_invariant_eval-0.3.0/SECURITY.md +20 -0
- agent_invariant_eval-0.3.0/examples/payment_dispute_agent/mock_agent.py +198 -0
- agent_invariant_eval-0.3.0/examples/payment_dispute_agent/models.json +38 -0
- agent_invariant_eval-0.3.0/examples/payment_dispute_agent/run_eval.py +139 -0
- agent_invariant_eval-0.3.0/examples/payment_dispute_agent/test_suite.jsonl +10 -0
- agent_invariant_eval-0.3.0/pyproject.toml +70 -0
- agent_invariant_eval-0.3.0/src/agent_eval/__init__.py +63 -0
- agent_invariant_eval-0.3.0/src/agent_eval/_version.py +1 -0
- agent_invariant_eval-0.3.0/src/agent_eval/benchmark/__init__.py +33 -0
- agent_invariant_eval-0.3.0/src/agent_eval/benchmark/frontier.py +431 -0
- agent_invariant_eval-0.3.0/src/agent_eval/cli.py +138 -0
- agent_invariant_eval-0.3.0/src/agent_eval/decorators.py +632 -0
- agent_invariant_eval-0.3.0/src/agent_eval/invariants/__init__.py +88 -0
- agent_invariant_eval-0.3.0/src/agent_eval/invariants/base.py +238 -0
- agent_invariant_eval-0.3.0/src/agent_eval/invariants/compliance.py +772 -0
- agent_invariant_eval-0.3.0/src/agent_eval/invariants/topology.py +390 -0
- agent_invariant_eval-0.3.0/src/agent_eval/py.typed +0 -0
- agent_invariant_eval-0.3.0/src/agent_eval/schemas.py +333 -0
- agent_invariant_eval-0.3.0/src/agent_eval/telemetry/__init__.py +10 -0
- agent_invariant_eval-0.3.0/src/agent_eval/telemetry/otel_exporter.py +320 -0
- agent_invariant_eval-0.3.0/tests/test_compliance.py +445 -0
- agent_invariant_eval-0.3.0/tests/test_frontier.py +330 -0
- agent_invariant_eval-0.3.0/tests/test_runtime.py +592 -0
- agent_invariant_eval-0.3.0/tests/test_topology.py +353 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Bug report
|
|
3
|
+
about: Report a bug, false positive, or detection miss
|
|
4
|
+
title: "[bug] "
|
|
5
|
+
labels: bug
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Summary
|
|
9
|
+
|
|
10
|
+
A clear description of the bug or unexpected verdict.
|
|
11
|
+
|
|
12
|
+
## Minimal reproduction
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from agent_eval import ...
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Expected vs. actual behaviour
|
|
19
|
+
|
|
20
|
+
- **Expected:**
|
|
21
|
+
- **Actual:**
|
|
22
|
+
|
|
23
|
+
## Environment
|
|
24
|
+
|
|
25
|
+
- Python version:
|
|
26
|
+
- `agent-invariant-eval` version:
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Feature request
|
|
3
|
+
about: Suggest a new invariant, exporter option, or benchmark capability
|
|
4
|
+
title: "[feat] "
|
|
5
|
+
labels: enhancement
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Problem
|
|
9
|
+
|
|
10
|
+
What workflow or governance gap are you trying to solve?
|
|
11
|
+
|
|
12
|
+
## Proposed solution
|
|
13
|
+
|
|
14
|
+
Describe the API or behaviour you have in mind (remembering that core invariants stay deterministic
|
|
15
|
+
and model-free).
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: ${{ matrix.python-version }}
|
|
22
|
+
- run: pip install -e ".[dev]"
|
|
23
|
+
- run: ruff check src tests examples
|
|
24
|
+
- run: mypy src
|
|
25
|
+
- run: pytest --cov=agent_eval --cov-report=term-missing --cov-fail-under=90
|
|
26
|
+
- run: python examples/payment_dispute_agent/run_eval.py
|
|
27
|
+
- run: python -m build
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: write
|
|
10
|
+
id-token: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
build-and-publish:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
environment: pypi
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
- name: Verify tag matches pyproject.toml version
|
|
22
|
+
run: |
|
|
23
|
+
python -c '
|
|
24
|
+
import os, pathlib, tomllib
|
|
25
|
+
tag = os.environ["GITHUB_REF_NAME"].removeprefix("v")
|
|
26
|
+
version = tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"]
|
|
27
|
+
assert tag == version, f"tag {tag!r} != pyproject.toml version {version!r}"
|
|
28
|
+
'
|
|
29
|
+
- run: pip install -e ".[dev]"
|
|
30
|
+
- run: ruff check src tests examples
|
|
31
|
+
- run: mypy src
|
|
32
|
+
- run: pytest --cov=agent_eval --cov-fail-under=90
|
|
33
|
+
- run: python -m build
|
|
34
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
35
|
+
- uses: softprops/action-gh-release@v2
|
|
36
|
+
with:
|
|
37
|
+
files: dist/*
|
|
38
|
+
generate_release_notes: true
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.3.0
|
|
4
|
+
|
|
5
|
+
### Runtime and telemetry
|
|
6
|
+
|
|
7
|
+
* **Pre-execution span input gate.** `EvaluationSession._open()` now evaluates `Stage.SPAN`
|
|
8
|
+
invariants on `span.input` before yielding control to the `with` block, blocking tool side
|
|
9
|
+
effects on non-compliant inputs. `_close()` skips re-scanning unchanged pre-checked inputs so
|
|
10
|
+
scanning stays linear.
|
|
11
|
+
* **Cycle guards and callback isolation.** `flatten_text()` and `collect_numbers()` guard against
|
|
12
|
+
self-referential containers. `on_report` callback errors are logged without breaking verdicts.
|
|
13
|
+
* **Context & OTel fixes.** Cross-context `EvaluationSession.__exit__` no longer clobbers an outer
|
|
14
|
+
session. `TelemetryEmitter` supports `with TelemetryEmitter(...) as emitter:` and exports
|
|
15
|
+
`gen_ai.usage.output_tokens` on tool spans even when `input_tokens == 0`.
|
|
16
|
+
|
|
17
|
+
### Compliance
|
|
18
|
+
|
|
19
|
+
* **Known secret prefixes.** `find_pii()` detects AWS (`AKIA`/`ASIA`), GitHub (`gh[pousr]_`,
|
|
20
|
+
`github_pat_`), Stripe (`sk_live_`/`rk_live_`/`sk_test_`/`rk_test_`), Anthropic (`sk-ant-`),
|
|
21
|
+
OpenAI (`sk-proj-`), Google (`AIza`), and Slack (`xox[baprs]-`) credentials regardless of
|
|
22
|
+
length, without duplicating `high_entropy_secret` findings.
|
|
23
|
+
* **Allowlists and BIN filtering.** Added `allowed_domains` (matching exact domains and
|
|
24
|
+
subdomains) to `NoPII` and `mask_pii()`, `ignore_keys` to `NoPII` and `PCICompliant`,
|
|
25
|
+
`require_known_bin` to `PCICompliant`, `find_pci()`, and `mask_pci()`, plus CLI flags
|
|
26
|
+
`--allowed-domain` and `--require-known-bin` on `agent-eval scan`.
|
|
27
|
+
* **Fixed PAN false positives on decimals.** `_PAN_CANDIDATE` now ignores fractional parts of
|
|
28
|
+
decimal numbers (`0.8475...`), epoch timestamps (`1700000123.456789`), large amounts, and OIDs,
|
|
29
|
+
accepting dot separators only in strict `4.4.4.4` card grouping (`4111.1111.1111.1111`).
|
|
30
|
+
|
|
31
|
+
### Packaging and CI
|
|
32
|
+
|
|
33
|
+
* **Python 3.11+ support.** Replaced PEP 695 generics in `find_cycle` with `TypeVar`.
|
|
34
|
+
* **Optional extras.** Moved `typer` to `[cli]` and `opentelemetry-exporter-otlp-proto-http` to
|
|
35
|
+
`[otlp]` (with `[all]` and `[dev]`).
|
|
36
|
+
* **CI, security & release workflows.** Added GitHub Actions CI (`permissions: contents: read`,
|
|
37
|
+
`ruff`, `mypy --strict`, `pytest --cov` with 90% floor, `run_eval.py`, `python -m build` across
|
|
38
|
+
Python 3.11–3.13), `SECURITY.md`, `CONTRIBUTING.md`, `CITATION.cff`, issue templates, and PyPI
|
|
39
|
+
Trusted Publishing release workflow with tag-vs-version verification.
|
|
40
|
+
|
|
41
|
+
## 0.2.0
|
|
42
|
+
|
|
43
|
+
Addresses an external review of 0.1.0. Behaviour changes are marked **changed**.
|
|
44
|
+
|
|
45
|
+
### Runtime and concurrency
|
|
46
|
+
|
|
47
|
+
* Open span stack is now context local (`contextvars`). Parallel branches under
|
|
48
|
+
`asyncio.gather`, `TaskGroup` and `to_thread` attach to the correct parent. Recording and
|
|
49
|
+
checks are serialised by a lock, so threads can record safely with an explicit `parent_id`.
|
|
50
|
+
* PII and PCI scanning is incremental. A span boundary scans only the span that closed
|
|
51
|
+
(`BaseInvariant.check_span`). The output stage still scans everything once.
|
|
52
|
+
* `@evaluate_agent` supports generator and async generator agents with `stream="buffer"`
|
|
53
|
+
(default, fail closed) and `stream="passthrough"`. A plain function that returns a generator
|
|
54
|
+
or coroutine object now fails closed with a `TypeError`.
|
|
55
|
+
* `wrapped.with_report(...)` returns `(result, report)` and is race free. `last_report` stays
|
|
56
|
+
as a single threaded convenience.
|
|
57
|
+
|
|
58
|
+
### Invariants
|
|
59
|
+
|
|
60
|
+
* **changed** `numeric_grounding`: numbers inside ids, dates and times in tool strings no
|
|
61
|
+
longer ground a figure. New `ground_from_strings` option. More currencies (symbols, ISO
|
|
62
|
+
codes in upper case, Indian digit grouping).
|
|
63
|
+
* **changed** `no_pii`: a bare nine digit number is a routing number only with a routing
|
|
64
|
+
keyword and a valid ABA checksum. `require_routing_context=False` restores checksum only.
|
|
65
|
+
Context keywords now match keys such as `routing_number` and `ssn_number`.
|
|
66
|
+
* **changed** `no_context_bloat`: baseline floored at `min_baseline_tokens` (500).
|
|
67
|
+
* **changed** `no_swallowed_tool_failures`: the default acknowledgement pattern accepts
|
|
68
|
+
inability phrases only. Apologies and bare words such as "error" no longer count. New
|
|
69
|
+
`allow_acknowledgement` option.
|
|
70
|
+
* **changed** `no_routing_loops`: counts repeated directed handoffs (`max_edge_repeats`, default
|
|
71
|
+
1). `max_visits_per_agent` is now optional and off by default. Hub and spoke supervisors pass.
|
|
72
|
+
* **changed** Token totals deduplicate rolled up parent usage
|
|
73
|
+
(`Trajectory.token_attribution()`). SLA budgets, context bloat and costs use it.
|
|
74
|
+
|
|
75
|
+
### Telemetry, benchmark, CLI
|
|
76
|
+
|
|
77
|
+
* OTel trace and span ids now equal the trajectory ids. Every child span carries
|
|
78
|
+
`agent_eval.span_id`, and `agent_eval.violations` when an invariant failed on it.
|
|
79
|
+
**changed** root attribute `gen_ai.conversation.id` replaced by `agent_eval.trace_id`
|
|
80
|
+
(a trace id is not a conversation id).
|
|
81
|
+
* `FrontierRunner` accepts async agents, `max_concurrency`, and `await runner.arun(...)`.
|
|
82
|
+
* `ModelConfig.pricing` prices each span by its model for mixed model runs.
|
|
83
|
+
* `agent-eval benchmark --models` accepts a JSON file or `module:attribute`. Added
|
|
84
|
+
`--max-concurrency`. `examples/payment_dispute_agent/models.json` is now shipped.
|
|
85
|
+
* CLI tests added.
|
|
86
|
+
|
|
87
|
+
### Follow up fixes
|
|
88
|
+
|
|
89
|
+
* Abandoning a `passthrough` stream early evaluates the partial output instead of recording an
|
|
90
|
+
agent crash.
|
|
91
|
+
* Token roll up is detected per side (input and output), which can only over count.
|
|
92
|
+
* Stricter digit grouping. Money in an unsupported number format (`€12,50`, `1.234,56 EUR`)
|
|
93
|
+
now fails `numeric_grounding` as an unparseable figure instead of being skipped.
|
|
94
|
+
|
|
95
|
+
## 0.1.0
|
|
96
|
+
|
|
97
|
+
Initial release.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
title: agent-invariant-eval
|
|
3
|
+
message: "If you use this software in research or engineering reports, please cite it using these metadata."
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- given-names: Sankaranainar
|
|
7
|
+
family-names: Paramasivan
|
|
8
|
+
# orcid: "https://orcid.org/0000-0000-0000-0000"
|
|
9
|
+
repository-code: "https://github.com/sankaranainar-p/agent-invariant-eval"
|
|
10
|
+
license: MIT
|
|
11
|
+
version: "0.3.0"
|
|
12
|
+
date-released: "2026-11-20"
|
|
13
|
+
keywords:
|
|
14
|
+
- llm-agents
|
|
15
|
+
- ai-governance
|
|
16
|
+
- opentelemetry
|
|
17
|
+
- pci-dss
|
|
18
|
+
- evaluation
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Contributions are welcome! Please keep changes focused, deterministic (no LLM calls in core
|
|
4
|
+
invariants), and covered by tests.
|
|
5
|
+
|
|
6
|
+
## Local setup
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
git clone https://github.com/sankaranainar-p/agent-invariant-eval.git
|
|
10
|
+
cd agent-invariant-eval
|
|
11
|
+
pip install -e ".[dev]"
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Checks before opening a PR
|
|
15
|
+
|
|
16
|
+
Run the same checks that CI runs:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
ruff check src tests examples
|
|
20
|
+
mypy src
|
|
21
|
+
pytest --cov=agent_eval --cov-fail-under=90
|
|
22
|
+
python examples/payment_dispute_agent/run_eval.py
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Security issues
|
|
26
|
+
|
|
27
|
+
Do not open public issues for security vulnerabilities or detection bypasses. See
|
|
28
|
+
[SECURITY.md](SECURITY.md) for private reporting instructions.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sankaranainar Paramasivan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|