agent-invariant-eval 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. agent_invariant_eval-0.3.0/.github/ISSUE_TEMPLATE/bug_report.md +26 -0
  2. agent_invariant_eval-0.3.0/.github/ISSUE_TEMPLATE/feature_request.md +15 -0
  3. agent_invariant_eval-0.3.0/.github/workflows/ci.yml +27 -0
  4. agent_invariant_eval-0.3.0/.github/workflows/release.yml +38 -0
  5. agent_invariant_eval-0.3.0/.gitignore +12 -0
  6. agent_invariant_eval-0.3.0/CHANGELOG.md +97 -0
  7. agent_invariant_eval-0.3.0/CITATION.cff +18 -0
  8. agent_invariant_eval-0.3.0/CONTRIBUTING.md +28 -0
  9. agent_invariant_eval-0.3.0/LICENSE +21 -0
  10. agent_invariant_eval-0.3.0/PKG-INFO +593 -0
  11. agent_invariant_eval-0.3.0/README.md +532 -0
  12. agent_invariant_eval-0.3.0/SECURITY.md +20 -0
  13. agent_invariant_eval-0.3.0/examples/payment_dispute_agent/mock_agent.py +198 -0
  14. agent_invariant_eval-0.3.0/examples/payment_dispute_agent/models.json +38 -0
  15. agent_invariant_eval-0.3.0/examples/payment_dispute_agent/run_eval.py +139 -0
  16. agent_invariant_eval-0.3.0/examples/payment_dispute_agent/test_suite.jsonl +10 -0
  17. agent_invariant_eval-0.3.0/pyproject.toml +70 -0
  18. agent_invariant_eval-0.3.0/src/agent_eval/__init__.py +63 -0
  19. agent_invariant_eval-0.3.0/src/agent_eval/_version.py +1 -0
  20. agent_invariant_eval-0.3.0/src/agent_eval/benchmark/__init__.py +33 -0
  21. agent_invariant_eval-0.3.0/src/agent_eval/benchmark/frontier.py +431 -0
  22. agent_invariant_eval-0.3.0/src/agent_eval/cli.py +138 -0
  23. agent_invariant_eval-0.3.0/src/agent_eval/decorators.py +632 -0
  24. agent_invariant_eval-0.3.0/src/agent_eval/invariants/__init__.py +88 -0
  25. agent_invariant_eval-0.3.0/src/agent_eval/invariants/base.py +238 -0
  26. agent_invariant_eval-0.3.0/src/agent_eval/invariants/compliance.py +772 -0
  27. agent_invariant_eval-0.3.0/src/agent_eval/invariants/topology.py +390 -0
  28. agent_invariant_eval-0.3.0/src/agent_eval/py.typed +0 -0
  29. agent_invariant_eval-0.3.0/src/agent_eval/schemas.py +333 -0
  30. agent_invariant_eval-0.3.0/src/agent_eval/telemetry/__init__.py +10 -0
  31. agent_invariant_eval-0.3.0/src/agent_eval/telemetry/otel_exporter.py +320 -0
  32. agent_invariant_eval-0.3.0/tests/test_compliance.py +445 -0
  33. agent_invariant_eval-0.3.0/tests/test_frontier.py +330 -0
  34. agent_invariant_eval-0.3.0/tests/test_runtime.py +592 -0
  35. agent_invariant_eval-0.3.0/tests/test_topology.py +353 -0
@@ -0,0 +1,26 @@
1
+ ---
2
+ name: Bug report
3
+ about: Report a bug, false positive, or detection miss
4
+ title: "[bug] "
5
+ labels: bug
6
+ ---
7
+
8
+ ## Summary
9
+
10
+ A clear description of the bug or unexpected verdict.
11
+
12
+ ## Minimal reproduction
13
+
14
+ ```python
15
+ from agent_eval import ...
16
+ ```
17
+
18
+ ## Expected vs. actual behaviour
19
+
20
+ - **Expected:**
21
+ - **Actual:**
22
+
23
+ ## Environment
24
+
25
+ - Python version:
26
+ - `agent-invariant-eval` version:
@@ -0,0 +1,15 @@
1
+ ---
2
+ name: Feature request
3
+ about: Suggest a new invariant, exporter option, or benchmark capability
4
+ title: "[feat] "
5
+ labels: enhancement
6
+ ---
7
+
8
+ ## Problem
9
+
10
+ What workflow or governance gap are you trying to solve?
11
+
12
+ ## Proposed solution
13
+
14
+ Describe the API or behaviour you have in mind (remembering that core invariants stay deterministic
15
+ and model-free).
@@ -0,0 +1,27 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ jobs:
12
+ test:
13
+ runs-on: ubuntu-latest
14
+ strategy:
15
+ matrix:
16
+ python-version: ["3.11", "3.12", "3.13"]
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - uses: actions/setup-python@v5
20
+ with:
21
+ python-version: ${{ matrix.python-version }}
22
+ - run: pip install -e ".[dev]"
23
+ - run: ruff check src tests examples
24
+ - run: mypy src
25
+ - run: pytest --cov=agent_eval --cov-report=term-missing --cov-fail-under=90
26
+ - run: python examples/payment_dispute_agent/run_eval.py
27
+ - run: python -m build
@@ -0,0 +1,38 @@
1
+ name: release
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*"
7
+
8
+ permissions:
9
+ contents: write
10
+ id-token: write
11
+
12
+ jobs:
13
+ build-and-publish:
14
+ runs-on: ubuntu-latest
15
+ environment: pypi
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: actions/setup-python@v5
19
+ with:
20
+ python-version: "3.12"
21
+ - name: Verify tag matches pyproject.toml version
22
+ run: |
23
+ python -c '
24
+ import os, pathlib, tomllib
25
+ tag = os.environ["GITHUB_REF_NAME"].removeprefix("v")
26
+ version = tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"]
27
+ assert tag == version, f"tag {tag!r} != pyproject.toml version {version!r}"
28
+ '
29
+ - run: pip install -e ".[dev]"
30
+ - run: ruff check src tests examples
31
+ - run: mypy src
32
+ - run: pytest --cov=agent_eval --cov-fail-under=90
33
+ - run: python -m build
34
+ - uses: pypa/gh-action-pypi-publish@release/v1
35
+ - uses: softprops/action-gh-release@v2
36
+ with:
37
+ files: dist/*
38
+ generate_release_notes: true
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .pytest_cache/
5
+ .venv/
6
+ dist/
7
+ build/
8
+ out/
9
+ .DS_Store
10
+ .coverage
11
+ .ruff_cache/
12
+ .mypy_cache/
@@ -0,0 +1,97 @@
1
+ # Changelog
2
+
3
+ ## 0.3.0
4
+
5
+ ### Runtime and telemetry
6
+
7
+ * **Pre-execution span input gate.** `EvaluationSession._open()` now evaluates `Stage.SPAN`
8
+ invariants on `span.input` before yielding control to the `with` block, blocking tool side
9
+ effects on non-compliant inputs. `_close()` skips re-scanning unchanged pre-checked inputs so
10
+ scanning stays linear.
11
+ * **Cycle guards and callback isolation.** `flatten_text()` and `collect_numbers()` guard against
12
+ self-referential containers. `on_report` callback errors are logged without breaking verdicts.
13
+ * **Context & OTel fixes.** Cross-context `EvaluationSession.__exit__` no longer clobbers an outer
14
+ session. `TelemetryEmitter` supports `with TelemetryEmitter(...) as emitter:` and exports
15
+ `gen_ai.usage.output_tokens` on tool spans even when `input_tokens == 0`.
16
+
17
+ ### Compliance
18
+
19
+ * **Known secret prefixes.** `find_pii()` detects AWS (`AKIA`/`ASIA`), GitHub (`gh[pousr]_`,
20
+ `github_pat_`), Stripe (`sk_live_`/`rk_live_`/`sk_test_`/`rk_test_`), Anthropic (`sk-ant-`),
21
+ OpenAI (`sk-proj-`), Google (`AIza`), and Slack (`xox[baprs]-`) credentials regardless of
22
+ length, without duplicating `high_entropy_secret` findings.
23
+ * **Allowlists and BIN filtering.** Added `allowed_domains` (matching exact domains and
24
+ subdomains) to `NoPII` and `mask_pii()`, `ignore_keys` to `NoPII` and `PCICompliant`,
25
+ `require_known_bin` to `PCICompliant`, `find_pci()`, and `mask_pci()`, plus CLI flags
26
+ `--allowed-domain` and `--require-known-bin` on `agent-eval scan`.
27
+ * **Fixed PAN false positives on decimals.** `_PAN_CANDIDATE` now ignores fractional parts of
28
+ decimal numbers (`0.8475...`), epoch timestamps (`1700000123.456789`), large amounts, and OIDs,
29
+ accepting dot separators only in strict `4.4.4.4` card grouping (`4111.1111.1111.1111`).
30
+
31
+ ### Packaging and CI
32
+
33
+ * **Python 3.11+ support.** Replaced PEP 695 generics in `find_cycle` with `TypeVar`.
34
+ * **Optional extras.** Moved `typer` to `[cli]` and `opentelemetry-exporter-otlp-proto-http` to
35
+ `[otlp]` (with `[all]` and `[dev]`).
36
+ * **CI, security & release workflows.** Added GitHub Actions CI (`permissions: contents: read`,
37
+ `ruff`, `mypy --strict`, `pytest --cov` with 90% floor, `run_eval.py`, `python -m build` across
38
+ Python 3.11–3.13), `SECURITY.md`, `CONTRIBUTING.md`, `CITATION.cff`, issue templates, and PyPI
39
+ Trusted Publishing release workflow with tag-vs-version verification.
40
+
41
+ ## 0.2.0
42
+
43
+ Addresses an external review of 0.1.0. Behaviour changes are marked **changed**.
44
+
45
+ ### Runtime and concurrency
46
+
47
+ * Open span stack is now context local (`contextvars`). Parallel branches under
48
+ `asyncio.gather`, `TaskGroup` and `to_thread` attach to the correct parent. Recording and
49
+ checks are serialised by a lock, so threads can record safely with an explicit `parent_id`.
50
+ * PII and PCI scanning is incremental. A span boundary scans only the span that closed
51
+ (`BaseInvariant.check_span`). The output stage still scans everything once.
52
+ * `@evaluate_agent` supports generator and async generator agents with `stream="buffer"`
53
+ (default, fail closed) and `stream="passthrough"`. A plain function that returns a generator
54
+ or coroutine object now fails closed with a `TypeError`.
55
+ * `wrapped.with_report(...)` returns `(result, report)` and is race free. `last_report` stays
56
+ as a single threaded convenience.
57
+
58
+ ### Invariants
59
+
60
+ * **changed** `numeric_grounding`: numbers inside ids, dates and times in tool strings no
61
+ longer ground a figure. New `ground_from_strings` option. More currencies (symbols, ISO
62
+ codes in upper case, Indian digit grouping).
63
+ * **changed** `no_pii`: a bare nine digit number is a routing number only with a routing
64
+ keyword and a valid ABA checksum. `require_routing_context=False` restores checksum only.
65
+ Context keywords now match keys such as `routing_number` and `ssn_number`.
66
+ * **changed** `no_context_bloat`: baseline floored at `min_baseline_tokens` (500).
67
+ * **changed** `no_swallowed_tool_failures`: the default acknowledgement pattern accepts
68
+ inability phrases only. Apologies and bare words such as "error" no longer count. New
69
+ `allow_acknowledgement` option.
70
+ * **changed** `no_routing_loops`: counts repeated directed handoffs (`max_edge_repeats`, default
71
+ 1). `max_visits_per_agent` is now optional and off by default. Hub and spoke supervisors pass.
72
+ * **changed** Token totals deduplicate rolled up parent usage
73
+ (`Trajectory.token_attribution()`). SLA budgets, context bloat and costs use it.
74
+
75
+ ### Telemetry, benchmark, CLI
76
+
77
+ * OTel trace and span ids now equal the trajectory ids. Every child span carries
78
+ `agent_eval.span_id`, and `agent_eval.violations` when an invariant failed on it.
79
+ **changed** root attribute `gen_ai.conversation.id` replaced by `agent_eval.trace_id`
80
+ (a trace id is not a conversation id).
81
+ * `FrontierRunner` accepts async agents, `max_concurrency`, and `await runner.arun(...)`.
82
+ * `ModelConfig.pricing` prices each span by its model for mixed model runs.
83
+ * `agent-eval benchmark --models` accepts a JSON file or `module:attribute`. Added
84
+ `--max-concurrency`. `examples/payment_dispute_agent/models.json` is now shipped.
85
+ * CLI tests added.
86
+
87
+ ### Follow up fixes
88
+
89
+ * Abandoning a `passthrough` stream early evaluates the partial output instead of recording an
90
+ agent crash.
91
+ * Token roll up is detected per side (input and output), which can only over count.
92
+ * Stricter digit grouping. Money in an unsupported number format (`€12,50`, `1.234,56 EUR`)
93
+ now fails `numeric_grounding` as an unparseable figure instead of being skipped.
94
+
95
+ ## 0.1.0
96
+
97
+ Initial release.
@@ -0,0 +1,18 @@
1
+ cff-version: 1.2.0
2
+ title: agent-invariant-eval
3
+ message: "If you use this software in research or engineering reports, please cite it using these metadata."
4
+ type: software
5
+ authors:
6
+ - given-names: Sankaranainar
7
+ family-names: Paramasivan
8
+ # orcid: "https://orcid.org/0000-0000-0000-0000"
9
+ repository-code: "https://github.com/sankaranainar-p/agent-invariant-eval"
10
+ license: MIT
11
+ version: "0.3.0"
12
+ date-released: "2026-11-20"
13
+ keywords:
14
+ - llm-agents
15
+ - ai-governance
16
+ - opentelemetry
17
+ - pci-dss
18
+ - evaluation
@@ -0,0 +1,28 @@
1
+ # Contributing
2
+
3
+ Contributions are welcome! Please keep changes focused, deterministic (no LLM calls in core
4
+ invariants), and covered by tests.
5
+
6
+ ## Local setup
7
+
8
+ ```bash
9
+ git clone https://github.com/sankaranainar-p/agent-invariant-eval.git
10
+ cd agent-invariant-eval
11
+ pip install -e ".[dev]"
12
+ ```
13
+
14
+ ## Checks before opening a PR
15
+
16
+ Run the same checks that CI runs:
17
+
18
+ ```bash
19
+ ruff check src tests examples
20
+ mypy src
21
+ pytest --cov=agent_eval --cov-fail-under=90
22
+ python examples/payment_dispute_agent/run_eval.py
23
+ ```
24
+
25
+ ## Security issues
26
+
27
+ Do not open public issues for security vulnerabilities or detection bypasses. See
28
+ [SECURITY.md](SECURITY.md) for private reporting instructions.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Sankaranainar Paramasivan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.