agent-reliability-harness 0.2.1__tar.gz

agent_reliability_harness-0.2.1/CHANGELOG.md

@@ -0,0 +1,122 @@
+# Changelog
+
+All notable changes to this project are documented here. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project
+adheres to [Semantic Versioning](https://semver.org/).
+
+## [Unreleased]
+
+## [0.2.1] - 2026-07-14
+
+### Fixed
+
+- Modernized packaging license metadata to the PEP 639/SPDX form
+  (`license = "MIT"`, `license-files = ["LICENSE"]`) and removed the
+  deprecated license classifier; builds no longer emit setuptools license
+  deprecation warnings.
+- Converted README documentation links to absolute GitHub URLs so they work
+  from the PyPI long-description renderer.
+- Corrected stale compatibility language that incorrectly claimed every
+  v0.1 score remained identical despite the documented nested-safety-scan
+  fix (70.0 -> 70.83 for one sample; verdict unchanged).
+
+
+## [0.2.0] - 2026-07-14
+
+### Added
+
+- **Trajectory rules** (`sequence`): required tools, forbidden tools,
+  partial-order `call_order`, per-tool `min_calls`/`max_calls`
+  (`ARH-SEQ-001..005`).
+- **Flow rules** (`error_handling`, `side_effect`): ignored tool errors,
+  retry storms, duplicate side-effect protection (`ARH-FLW-001..003`).
+- **Completion rules** (`completion`): required final response, max step
+  count (`ARH-CMP-001..002`).
+- **Argument value constraints**: `enum`, full-match `pattern`, numeric
+  `min`/`max` on tool arguments (`ARH-SCH-007..009`), alongside the legacy
+  bare type-name form.
+- **Token budgets** (`budgets.max_total_tokens`, `ARH-BUD-004`) with an
+  explicit warning when the trace carries no token data (`ARH-BUD-005`)
+  instead of a silent pass.
+- **Citation validity** (`grounding.require_valid_citation_urls`,
+  `ARH-GRD-003`) and step-level `status`/`error`, `input_tokens`/
+  `output_tokens`, and `metadata` trace fields.
+- **Stable rule IDs** (`ARH-*`) on every finding, plus `expected`/`observed`/
+  `remediation` evidence fields and a generated rule reference
+  (`docs/rules.md`).
+- **Regression engine**: `arh compare` diffs a candidate run against a
+  baseline report (new/resolved findings, pass/fail transitions,
+  score/latency/cost deltas) with gate modes `regressions` (default),
+  `failures`, `never`, and `--max-score-drop`.
+- **Adapters**: `--format openai-chat` (Chat Completions `tool_calls`
+  transcripts) and `--format anthropic-messages` (`tool_use`/`tool_result`
+  conversations), with deterministic auto-detection and lossless handling of
+  unmappable input via `metadata.adapter.notes`.
+- **Reports**: JUnit XML (`--junit-out`) and SARIF 2.1.0 (`--sarif-out`)
+  renderers; rule IDs shown in console and Markdown output; JSON reports
+  carry `schema_version`.
+- **Schema versioning**: traces/policies accept `schema_version` (major `1`);
+  unknown major versions are rejected with a precise error.
+- **JSON Schemas** for trace, policy, and report formats (`schemas/`).
+- **Benchmark suite**: 34 seeded failure cases with a generator, a runner
+  enforcing thresholds (precision/recall 1.0, byte-identical determinism,
+  <50 ms/trace, metamorphic invariances, adapter equivalence, regression
+  scenarios), plus `BENCHMARK-METHODOLOGY.md` and generated
+  `BENCHMARK-RESULTS.md`.
+- **GitHub Action** (`action.yml`): validate + optional baseline compare,
+  step-summary reports, SARIF/JUnit outputs, fork-safe, no secrets.
+- **CI**: 3-OS × Python 3.11-3.13 matrix, ruff, strict mypy, benchmark
+  thresholds, packaging build with clean-environment install check.
+- Documentation set (`docs/`), specs (`SPEC.md`, `TRACE-SPEC.md`,
+  `POLICY-SPEC.md`, `ARCHITECTURE.md`, and more), and community files.
+
+### Fixed
+
+- **(review)** Regression loose-matching could silently cancel two findings
+  with *different* rule IDs on the same step whenever any legacy finding was
+  present, hiding real regressions from the CI gate. Loose matching now
+  applies only when at least one side of a pair lacks a `rule_id`.
+- **(review)** Safety scanning now recurses into nested lists/dicts and the
+  step `error` string; unsafe content one level deep no longer evades
+  detection.
+- **(review)** Warning-only traces can no longer fail: unverifiable budget
+  warnings (ARH-BUD-005, new ARH-BUD-006 latency, new ARH-BUD-007 cost) are
+  score-neutral.
+- **(review)** Malformed telemetry (negative/non-finite/mistyped latency,
+  cost, tokens; non-string text/tool_name/error) is rejected at parse time.
+- **(review)** `ArgSpec` patterns are compiled at policy load (precise error
+  instead of a traceback), `min > max` and non-finite bounds are rejected,
+  NaN values can no longer bypass range constraints, and unknown declared
+  types no longer disable enum/pattern/range checks.
+- **(review)** JUnit output strips XML-1.0-illegal control characters so
+  strict CI consumers can always parse it; Markdown reports escape
+  trace-derived fields in table cells.
+- **(review)** The GitHub Action passes all inputs via `env:` bindings
+  instead of interpolating `${{ inputs.* }}` into the script, closing a
+  shell-injection vector.
+- **(review)** CLI: `--fail-under` is bounds-checked (usage error, exit 2);
+  pathologically nested JSON fails with a clean message instead of a
+  `RecursionError` traceback; the module docstring documents real exit
+  codes.
+
+- v0.1.x reported type errors for **required** arguments twice (the argument
+  was checked in two loops). Each argument is now checked exactly once; the
+  duplicate finding is gone. Scores and verdicts are unchanged.
+
+### Compatibility
+
+- v0.1.x trace and policy files remain accepted; verdicts and finding
+  messages are preserved. The documented nested-safety-scan fix changes one
+  sample score from 70.0 to 70.83 (verdict unchanged). New report fields are
+  additive.
+  `arh compare` accepts v0.1.x baselines via a rule-id-agnostic fallback
+  match. Exit codes and console markers are unchanged. See COMPATIBILITY.md.
+
+## [0.1.0] - 2026-07-13
+
+### Added
+
+- Initial release: policy-driven trace validation (tool-call contracts,
+  latency/cost budgets, unsafe-pattern detection, citation coverage),
+  weighted 0-100 scoring, console/JSON/Markdown reports, `arh validate` CLI,
+  35 unit/CLI tests, GitHub Actions CI (Python 3.11/3.12).

agent_reliability_harness-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Felmon Fekadu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agent_reliability_harness-0.2.1/MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE README.md CHANGELOG.md
+recursive-include schemas *.json
+recursive-include samples *.json
+recursive-include tests *.py
+recursive-include tests/fixtures *.json

agent_reliability_harness-0.2.1/PKG-INFO

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: agent-reliability-harness
+Version: 0.2.1
+Summary: Local-first policy and trajectory-regression harness for tool-using AI agents: deterministic trace validation, policy-as-code trajectory rules, baseline-vs-candidate regression gates, and CI-ready JSON/Markdown/JUnit/SARIF reports.
+Author: Felmon Fekadu
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/felmonon/agent-reliability-harness
+Project-URL: Documentation, https://github.com/felmonon/agent-reliability-harness/tree/main/docs
+Project-URL: Changelog, https://github.com/felmonon/agent-reliability-harness/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/felmonon/agent-reliability-harness/issues
+Keywords: agents,llm,evaluation,reliability,regression-testing,grounding,tool-use,trajectory,policy-as-code,ci
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: mypy>=1.8; extra == "dev"
+Requires-Dist: ruff>=0.5; extra == "dev"
+Requires-Dist: jsonschema>=4.21; extra == "dev"
+Requires-Dist: build>=1.0; extra == "dev"
+Dynamic: license-file
+
+# Agent Reliability Harness
+
+A provider-neutral, local-first reliability and regression-testing harness for
+tool-using AI agents. It validates recorded agent execution traces against
+declarative policies (tool-call contracts, trajectory rules, budgets, unsafe
+patterns, grounding), produces deterministic findings with stable rule IDs,
+and compares candidate runs against a saved baseline so CI fails only on real
+regressions.
+
+The core has **zero runtime dependencies**, makes **no network calls**, sends
+**no telemetry**, and produces **byte-identical reports** for identical input.
+
+## Why this exists
+
+An agent can reach the correct final answer while still behaving unreliably.
+It may:
+
+- select the wrong tool, or skip a required one;
+- use correct tools in the wrong order;
+- pass invalid or out-of-range arguments;
+- repeat an irreversible action (double refund, duplicate email);
+- ignore a tool failure and answer anyway;
+- get stuck in a retry storm;
+- stop before finishing, or fail to terminate;
+- return facts without citations;
+- leak secrets or propagate prompt-injection text;
+- blow past latency, cost, or token budgets;
+- regress silently after a model, prompt, or tool change.
+
+Final-answer evals miss all of this. This harness evaluates the **trajectory**,
+and turns each of these concerns into a versioned policy rule with a stable ID.
+
+## 60-second example
+
+```bash
+python -m pip install -e .   # not yet on PyPI; install from a checkout
+
+arh validate --policy samples/policy_trajectory.json \
+  samples/traces/refund_workflow_pass.json \
+  samples/traces/refund_workflow_double_refund.json
+```
+
+The second trace ends with a perfectly plausible final answer — and still
+fails, because the trajectory was unacceptable:
+
+```text
+[FAIL] refund-workflow-1002  (agent=refund-copilot, workflow=refund)
+  score=84.5/100  policy=refund-workflow-v1  latency=1300ms  cost=$0.0080
+  - [ERROR  ] [flow     ] ARH-FLW-003: duplicate side effect: tool 'issue_refund' was called again with identical arguments after already succeeding (step=s3)
+  - [ERROR  ] [sequence ] ARH-SEQ-001: required tool 'check_refund_eligibility' was never called
+  - [ERROR  ] [sequence ] ARH-SEQ-004: tool 'issue_refund' was called 2 times, above max_calls of 1
+  - [ERROR  ] [sequence ] ARH-SEQ-003: call order violation: first call of 'lookup_order' must precede first call of 'issue_refund' (step=s1)
+```
+
+Exit code `0` when everything passes, `1` on failure, `2` on usage errors —
+usable as a CI gate as-is.
+
+## Regression gates
+
+Save a baseline once, then gate future runs on *changes*, not absolutes:
+
+```bash
+arh validate --policy samples/policy_trajectory.json \
+  samples/traces/refund_workflow_pass.json \
+  --json-out baseline.json --quiet
+
+arh compare --baseline baseline.json \
+  --policy samples/policy_trajectory.json \
+  samples/traces/refund_workflow_pass.json \
+  samples/traces/refund_workflow_double_refund.json
+```
+
+```text
+GATE: FAIL
+  - 4 new error finding(s): refund-workflow-1002:ARH-FLW-003, refund-workflow-1002:ARH-SEQ-001, refund-workflow-1002:ARH-SEQ-003, refund-workflow-1002:ARH-SEQ-004
+  - added trace 'refund-workflow-1002' fails
+```
+
+The gate fails on new error findings, pass→fail transitions, and added failing
+traces — resolved findings and unchanged failures don't re-alarm. Gate modes:
+`--fail-on regressions` (default), `failures`, `never`; add `--max-score-drop`
+for score-based gating. See [docs/regression-testing.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/regression-testing.md).
+
+## What it checks
+
+Seven deterministic categories; every finding carries a stable rule ID
+(`ARH-<CATEGORY>-<NNN>`), an expected/observed pair, and remediation guidance:
+
+| Category | Rules | Examples |
+|---|---|---|
+| schema | ARH-SCH-001..010 | unlisted tool, missing/mistyped args, enum/pattern/range violations |
+| budget | ARH-BUD-001..007 | step/total latency, cost, and token budgets, plus unverifiable-budget warnings |
+| safety | ARH-SAF-001 | prompt-injection phrases, secret-like strings (policy regexes) |
+| grounding | ARH-GRD-001..003 | missing citations, low coverage, malformed citation URLs |
+| sequence | ARH-SEQ-001..005 | required/forbidden tools, partial-order violations, call counts |
+| flow | ARH-FLW-001..003 | ignored tool errors, retry storms, duplicate side effects |
+| completion | ARH-CMP-001..002 | missing final response, failure to terminate |
+
+Full reference: [docs/rules.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/rules.md). Ordering rules are a
+deliberate **partial order** (constraints), not an exact golden-trajectory
+match — agents can legitimately reach a goal via different paths.
+
+## Trace formats
+
+`arh validate` accepts (auto-detected, or forced with `--format`):
+
+- **arh** — the canonical JSON trace format ([TRACE-SPEC.md](https://github.com/felmonon/agent-reliability-harness/blob/main/TRACE-SPEC.md));
+- **openai-chat** — OpenAI Chat Completions message lists with `tool_calls`;
+- **anthropic-messages** — Anthropic Messages conversations with
+  `tool_use`/`tool_result` blocks.
+
+Adapters never guess: fields a transcript format cannot carry (latency, cost,
+tokens) are left unset, which marks the dependent checks *not applicable*
+instead of silently passing them. Unparseable input is recorded in the trace's
+metadata, not dropped. See [docs/adapters.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/adapters.md).
+
+## Reports
+
+Console, JSON (doubles as the regression baseline), Markdown (PR-comment
+ready), JUnit XML, and SARIF 2.1.0 (GitHub code-scanning annotations). All
+renderers are deterministic: identical input produces byte-identical output —
+no timestamps, no randomness, no clock reads.
+
+## CI
+
+```yaml
+- uses: felmonon/agent-reliability-harness@main
+  with:
+    policy: policies/agent-policy.json
+    traces: traces/*.json
+    baseline: baselines/main.json   # optional: gate on regressions
+```
+
+The action writes JSON/Markdown/JUnit/SARIF reports, appends the Markdown
+summary to the workflow step summary, and fails according to the gate. It uses
+no secrets and is fork-safe. Full reference and SARIF/JUnit upload examples:
+[docs/ci.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/ci.md).
+
+## Design principles
+
+- **Zero runtime dependencies.** `pip install` adds nothing to your
+  application's dependency graph.
+- **Deterministic core.** No model calls, no network, no telemetry, no clock
+  reads. Semantic (model-graded) evaluation is a planned, clearly separated
+  optional extra — never hidden inside deterministic scores ([ROADMAP.md](https://github.com/felmonon/agent-reliability-harness/blob/main/ROADMAP.md)).
+- **Additive schema evolution.** v0.1.x traces, policies, and baselines work
+  unchanged; unknown major schema versions are rejected loudly
+  ([COMPATIBILITY.md](https://github.com/felmonon/agent-reliability-harness/blob/main/COMPATIBILITY.md)).
+- **Evidence over claims.** The benchmark suite contains 34 seeded cases
+  (28 expected-fail, 6 expected-pass controls) and measures detection:
+  currently 34/34 correct, precision 1.0, recall 1.0, 0 false
+  positives/negatives, byte-identical repeat runs, ~0.03 ms per trace. Those
+  numbers cover *seeded, deterministically detectable* failures only — scope
+  and limits are documented in [BENCHMARK-METHODOLOGY.md](https://github.com/felmonon/agent-reliability-harness/blob/main/BENCHMARK-METHODOLOGY.md),
+  results in [BENCHMARK-RESULTS.md](https://github.com/felmonon/agent-reliability-harness/blob/main/BENCHMARK-RESULTS.md).
+
+## Documentation
+
+| Doc | What it covers |
+|---|---|
+| [docs/quickstart.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/quickstart.md) | Five minutes from install to a regression gate |
+| [docs/concepts.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/concepts.md) | Traces, policies, findings, scores, baselines |
+| [docs/policy-cookbook.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/policy-cookbook.md) | Copy-paste recipes for every rule type |
+| [docs/adapters.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/adapters.md) | OpenAI/Anthropic transcript ingestion, field mapping |
+| [docs/regression-testing.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/regression-testing.md) | Baselines, fingerprints, gates |
+| [docs/ci.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/ci.md) | GitHub Action reference and workflows |
+| [docs/rules.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/rules.md) | Every rule ID with remediation |
+| [docs/troubleshooting.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/troubleshooting.md) | Error messages and fixes |
+| [TRACE-SPEC.md](https://github.com/felmonon/agent-reliability-harness/blob/main/TRACE-SPEC.md) / [POLICY-SPEC.md](https://github.com/felmonon/agent-reliability-harness/blob/main/POLICY-SPEC.md) | Normative formats |
+| [ARCHITECTURE.md](https://github.com/felmonon/agent-reliability-harness/blob/main/ARCHITECTURE.md) / [DECISIONS.md](https://github.com/felmonon/agent-reliability-harness/blob/main/DECISIONS.md) | Design and ADRs |
+
+## Compatibility
+
+v0.1.x trace and policy files remain accepted and preserve verdicts and finding
+messages. Two review-driven detection fixes are documented exceptions: nested safety
+scanning changes one sample score from 70.0 to 70.83, and strict telemetry validation
+rejects malformed values that v0.1 accepted. New report fields are additive.
+Details: [COMPATIBILITY.md](https://github.com/felmonon/agent-reliability-harness/blob/main/COMPATIBILITY.md).
+
+## Development
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+python -m pip install -e ".[dev]"
+python -m unittest discover -s tests   # full suite (194 tests as of 0.2.0)
+ruff check src tests benchmarks
+mypy src                               # strict
+python benchmarks/run.py               # thresholds enforced
+```
+
+CI runs the suite on Linux/macOS/Windows across Python 3.11-3.13, plus
+packaging checks. Contributions welcome — see [CONTRIBUTING.md](https://github.com/felmonon/agent-reliability-harness/blob/main/CONTRIBUTING.md).
+
+## Author
+
+[Felmon Fekadu](https://felmon.tech/proof)
+[GitHub](https://github.com/felmonon)
+
+## License
+
+MIT

agent_reliability_harness-0.2.1/README.md

@@ -0,0 +1,201 @@
+# Agent Reliability Harness
+
+A provider-neutral, local-first reliability and regression-testing harness for
+tool-using AI agents. It validates recorded agent execution traces against
+declarative policies (tool-call contracts, trajectory rules, budgets, unsafe
+patterns, grounding), produces deterministic findings with stable rule IDs,
+and compares candidate runs against a saved baseline so CI fails only on real
+regressions.
+
+The core has **zero runtime dependencies**, makes **no network calls**, sends
+**no telemetry**, and produces **byte-identical reports** for identical input.
+
+## Why this exists
+
+An agent can reach the correct final answer while still behaving unreliably.
+It may:
+
+- select the wrong tool, or skip a required one;
+- use correct tools in the wrong order;
+- pass invalid or out-of-range arguments;
+- repeat an irreversible action (double refund, duplicate email);
+- ignore a tool failure and answer anyway;
+- get stuck in a retry storm;
+- stop before finishing, or fail to terminate;
+- return facts without citations;
+- leak secrets or propagate prompt-injection text;
+- blow past latency, cost, or token budgets;
+- regress silently after a model, prompt, or tool change.
+
+Final-answer evals miss all of this. This harness evaluates the **trajectory**,
+and turns each of these concerns into a versioned policy rule with a stable ID.
+
+## 60-second example
+
+```bash
+python -m pip install -e .   # not yet on PyPI; install from a checkout
+
+arh validate --policy samples/policy_trajectory.json \
+  samples/traces/refund_workflow_pass.json \
+  samples/traces/refund_workflow_double_refund.json
+```
+
+The second trace ends with a perfectly plausible final answer — and still
+fails, because the trajectory was unacceptable:
+
+```text
+[FAIL] refund-workflow-1002  (agent=refund-copilot, workflow=refund)
+  score=84.5/100  policy=refund-workflow-v1  latency=1300ms  cost=$0.0080
+  - [ERROR  ] [flow     ] ARH-FLW-003: duplicate side effect: tool 'issue_refund' was called again with identical arguments after already succeeding (step=s3)
+  - [ERROR  ] [sequence ] ARH-SEQ-001: required tool 'check_refund_eligibility' was never called
+  - [ERROR  ] [sequence ] ARH-SEQ-004: tool 'issue_refund' was called 2 times, above max_calls of 1
+  - [ERROR  ] [sequence ] ARH-SEQ-003: call order violation: first call of 'lookup_order' must precede first call of 'issue_refund' (step=s1)
+```
+
+Exit code `0` when everything passes, `1` on failure, `2` on usage errors —
+usable as a CI gate as-is.
+
+## Regression gates
+
+Save a baseline once, then gate future runs on *changes*, not absolutes:
+
+```bash
+arh validate --policy samples/policy_trajectory.json \
+  samples/traces/refund_workflow_pass.json \
+  --json-out baseline.json --quiet
+
+arh compare --baseline baseline.json \
+  --policy samples/policy_trajectory.json \
+  samples/traces/refund_workflow_pass.json \
+  samples/traces/refund_workflow_double_refund.json
+```
+
+```text
+GATE: FAIL
+  - 4 new error finding(s): refund-workflow-1002:ARH-FLW-003, refund-workflow-1002:ARH-SEQ-001, refund-workflow-1002:ARH-SEQ-003, refund-workflow-1002:ARH-SEQ-004
+  - added trace 'refund-workflow-1002' fails
+```
+
+The gate fails on new error findings, pass→fail transitions, and added failing
+traces — resolved findings and unchanged failures don't re-alarm. Gate modes:
+`--fail-on regressions` (default), `failures`, `never`; add `--max-score-drop`
+for score-based gating. See [docs/regression-testing.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/regression-testing.md).
+
+## What it checks
+
+Seven deterministic categories; every finding carries a stable rule ID
+(`ARH-<CATEGORY>-<NNN>`), an expected/observed pair, and remediation guidance:
+
+| Category | Rules | Examples |
+|---|---|---|
+| schema | ARH-SCH-001..010 | unlisted tool, missing/mistyped args, enum/pattern/range violations |
+| budget | ARH-BUD-001..007 | step/total latency, cost, and token budgets, plus unverifiable-budget warnings |
+| safety | ARH-SAF-001 | prompt-injection phrases, secret-like strings (policy regexes) |
+| grounding | ARH-GRD-001..003 | missing citations, low coverage, malformed citation URLs |
+| sequence | ARH-SEQ-001..005 | required/forbidden tools, partial-order violations, call counts |
+| flow | ARH-FLW-001..003 | ignored tool errors, retry storms, duplicate side effects |
+| completion | ARH-CMP-001..002 | missing final response, failure to terminate |
+
+Full reference: [docs/rules.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/rules.md). Ordering rules are a
+deliberate **partial order** (constraints), not an exact golden-trajectory
+match — agents can legitimately reach a goal via different paths.
+
+## Trace formats
+
+`arh validate` accepts (auto-detected, or forced with `--format`):
+
+- **arh** — the canonical JSON trace format ([TRACE-SPEC.md](https://github.com/felmonon/agent-reliability-harness/blob/main/TRACE-SPEC.md));
+- **openai-chat** — OpenAI Chat Completions message lists with `tool_calls`;
+- **anthropic-messages** — Anthropic Messages conversations with
+  `tool_use`/`tool_result` blocks.
+
+Adapters never guess: fields a transcript format cannot carry (latency, cost,
+tokens) are left unset, which marks the dependent checks *not applicable*
+instead of silently passing them. Unparseable input is recorded in the trace's
+metadata, not dropped. See [docs/adapters.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/adapters.md).
+
+## Reports
+
+Console, JSON (doubles as the regression baseline), Markdown (PR-comment
+ready), JUnit XML, and SARIF 2.1.0 (GitHub code-scanning annotations). All
+renderers are deterministic: identical input produces byte-identical output —
+no timestamps, no randomness, no clock reads.
+
+## CI
+
+```yaml
+- uses: felmonon/agent-reliability-harness@main
+  with:
+    policy: policies/agent-policy.json
+    traces: traces/*.json
+    baseline: baselines/main.json   # optional: gate on regressions
+```
+
+The action writes JSON/Markdown/JUnit/SARIF reports, appends the Markdown
+summary to the workflow step summary, and fails according to the gate. It uses
+no secrets and is fork-safe. Full reference and SARIF/JUnit upload examples:
+[docs/ci.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/ci.md).
+
+## Design principles
+
+- **Zero runtime dependencies.** `pip install` adds nothing to your
+  application's dependency graph.
+- **Deterministic core.** No model calls, no network, no telemetry, no clock
+  reads. Semantic (model-graded) evaluation is a planned, clearly separated
+  optional extra — never hidden inside deterministic scores ([ROADMAP.md](https://github.com/felmonon/agent-reliability-harness/blob/main/ROADMAP.md)).
+- **Additive schema evolution.** v0.1.x traces, policies, and baselines work
+  unchanged; unknown major schema versions are rejected loudly
+  ([COMPATIBILITY.md](https://github.com/felmonon/agent-reliability-harness/blob/main/COMPATIBILITY.md)).
+- **Evidence over claims.** The benchmark suite contains 34 seeded cases
+  (28 expected-fail, 6 expected-pass controls) and measures detection:
+  currently 34/34 correct, precision 1.0, recall 1.0, 0 false
+  positives/negatives, byte-identical repeat runs, ~0.03 ms per trace. Those
+  numbers cover *seeded, deterministically detectable* failures only — scope
+  and limits are documented in [BENCHMARK-METHODOLOGY.md](https://github.com/felmonon/agent-reliability-harness/blob/main/BENCHMARK-METHODOLOGY.md),
+  results in [BENCHMARK-RESULTS.md](https://github.com/felmonon/agent-reliability-harness/blob/main/BENCHMARK-RESULTS.md).
+
+## Documentation
+
+| Doc | What it covers |
+|---|---|
+| [docs/quickstart.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/quickstart.md) | Five minutes from install to a regression gate |
+| [docs/concepts.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/concepts.md) | Traces, policies, findings, scores, baselines |
+| [docs/policy-cookbook.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/policy-cookbook.md) | Copy-paste recipes for every rule type |
+| [docs/adapters.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/adapters.md) | OpenAI/Anthropic transcript ingestion, field mapping |
+| [docs/regression-testing.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/regression-testing.md) | Baselines, fingerprints, gates |
+| [docs/ci.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/ci.md) | GitHub Action reference and workflows |
+| [docs/rules.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/rules.md) | Every rule ID with remediation |
+| [docs/troubleshooting.md](https://github.com/felmonon/agent-reliability-harness/blob/main/docs/troubleshooting.md) | Error messages and fixes |
+| [TRACE-SPEC.md](https://github.com/felmonon/agent-reliability-harness/blob/main/TRACE-SPEC.md) / [POLICY-SPEC.md](https://github.com/felmonon/agent-reliability-harness/blob/main/POLICY-SPEC.md) | Normative formats |
+| [ARCHITECTURE.md](https://github.com/felmonon/agent-reliability-harness/blob/main/ARCHITECTURE.md) / [DECISIONS.md](https://github.com/felmonon/agent-reliability-harness/blob/main/DECISIONS.md) | Design and ADRs |
+
+## Compatibility
+
+v0.1.x trace and policy files remain accepted and preserve verdicts and finding
+messages. Two review-driven detection fixes are documented exceptions: nested safety
+scanning changes one sample score from 70.0 to 70.83, and strict telemetry validation
+rejects malformed values that v0.1 accepted. New report fields are additive.
+Details: [COMPATIBILITY.md](https://github.com/felmonon/agent-reliability-harness/blob/main/COMPATIBILITY.md).
+
+## Development
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+python -m pip install -e ".[dev]"
+python -m unittest discover -s tests   # full suite (194 tests as of 0.2.0)
+ruff check src tests benchmarks
+mypy src                               # strict
+python benchmarks/run.py               # thresholds enforced
+```
+
+CI runs the suite on Linux/macOS/Windows across Python 3.11-3.13, plus
+packaging checks. Contributions welcome — see [CONTRIBUTING.md](https://github.com/felmonon/agent-reliability-harness/blob/main/CONTRIBUTING.md).
+
+## Author
+
+[Felmon Fekadu](https://felmon.tech/proof)
+[GitHub](https://github.com/felmonon)
+
+## License
+
+MIT

agent_reliability_harness-0.2.1/pyproject.toml

@@ -0,0 +1,70 @@
+[build-system]
+requires = ["setuptools>=77.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "agent-reliability-harness"
+version = "0.2.1"
+description = "Local-first policy and trajectory-regression harness for tool-using AI agents: deterministic trace validation, policy-as-code trajectory rules, baseline-vs-candidate regression gates, and CI-ready JSON/Markdown/JUnit/SARIF reports."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Felmon Fekadu" }]
+keywords = [
+    "agents",
+    "llm",
+    "evaluation",
+    "reliability",
+    "regression-testing",
+    "grounding",
+    "tool-use",
+    "trajectory",
+    "policy-as-code",
+    "ci",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/felmonon/agent-reliability-harness"
+Documentation = "https://github.com/felmonon/agent-reliability-harness/tree/main/docs"
+Changelog = "https://github.com/felmonon/agent-reliability-harness/blob/main/CHANGELOG.md"
+Issues = "https://github.com/felmonon/agent-reliability-harness/issues"
+
+[project.optional-dependencies]
+dev = ["mypy>=1.8", "ruff>=0.5", "jsonschema>=4.21", "build>=1.0"]
+
+[project.scripts]
+arh = "agent_reliability_harness.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+agent_reliability_harness = ["py.typed"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B"]
+ignore = ["E501"]  # long lines allowed in messages/docstrings; format stays readable
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_unused_ignores = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+strict = false