traceval 0.2.0__tar.gz → 0.2.2__tar.gz

{traceval-0.2.0 → traceval-0.2.2}/.github/workflows/ci.yml

@@ -34,6 +34,33 @@ jobs:
     - name: Run Tests with Coverage
       run: uv run pytest --cov=src/traceval --cov-fail-under=85
 
+  demo-wheel:
+    # The pip-user path is what ships: build the wheel, install it in a
+    # clean venv, and run the packaged demo from an empty directory.
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+
+    - name: Build wheel
+      run: |
+        pip install build
+        python -m build --wheel
+
+    - name: Install wheel in a clean venv and run the demo
+      run: |
+        python -m venv /tmp/wheel-venv
+        /tmp/wheel-venv/bin/pip install dist/*.whl
+        mkdir /tmp/demo-smoke
+        cd /tmp/demo-smoke
+        /tmp/wheel-venv/bin/traceval demo
+        /tmp/wheel-venv/bin/traceval analyze traceval-demo/traces.db --json | python -m json.tool
+
   demo-action:
     # Dogfoods action.yml against the demo agent: the only CI coverage the
     # user-facing GitHub Action gets.
@@ -50,15 +77,13 @@ jobs:
     - name: Generate demo eval suite
       run: |
         pip install .
-        python examples/make_traces.py
-        traceval ingest examples/synthetic_traces.jsonl -o demo_traces.db
-        traceval generate demo_traces.db -o demo_evals/ --include-failures
+        traceval demo -o demo-artifacts
 
     - name: Run evals via the action (healthy agent must pass)
       uses: ./
       with:
-        evals-dir: demo_evals
-        target: examples.demo_agent.core:invoke_agent
+        evals-dir: demo-artifacts/evals
+        target: traceval.demo.agent:invoke_agent
         traceval-version: local
 
     - name: Run evals via the action (buggy agent must fail)
@@ -68,20 +93,20 @@ jobs:
       env:
         BUGGY: "true"
       with:
-        evals-dir: demo_evals
-        target: examples.demo_agent.core:invoke_agent
+        evals-dir: demo-artifacts/evals
+        target: traceval.demo.agent:invoke_agent
         traceval-version: local
 
     - name: Assert buggy run failed
       run: |
         if [ "${{ steps.buggy.outcome }}" != "failure" ]; then
-          echo "❌ Buggy agent unexpectedly passed the eval suite"
+          echo "Buggy agent unexpectedly passed the eval suite"
           exit 1
         fi
-        echo "✅ Action correctly failed the job for the buggy agent"
+        echo "Action correctly failed the job for the buggy agent"
 
   tag-and-release:
-    needs: [test, demo-action]
+    needs: [test, demo-wheel, demo-action]
     if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
     runs-on: ubuntu-latest
     permissions:

{traceval-0.2.0 → traceval-0.2.2}/CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.2] - 2026-07-02
+
+### Fixed
+- A run in which zero cases execute (e.g. unresolvable target) now writes a self-describing run report with an `errors` section instead of writing nothing; `--json` never reports `null`. Added a single clear top-level error line on target-resolution failure. Added `make test` and CONTRIBUTING as the canonical dev commands. Reported via external review of a failed-target invocation.
+- Run report schema additions (existing fields unchanged): `summary.errored` counts cases that never executed due to setup/collection errors; top-level `errors` lists `{stage, detail}` entries for `target_resolution`, `collection`, and `setup` failures, with identical details deduplicated into one entry carrying a `count`; `results` is `[]` rather than absent when nothing executed. The terminal summary now shows `Errored: n`.
+
+## [0.2.1] - 2026-07-02
+
+### Added
+- `--json` on `ingest`, `analyze`, `generate`, and `run`: suppresses human-readable output and prints a single JSON object to stdout for scripting (`run` still exits nonzero on failures). The README previously claimed this flag existed; now it does.
+- `traceval demo`: runs the full trace-to-eval loop end-to-end with a built-in demo agent from a plain pip install (no repo clone). Creates `./traceval-demo/` (override with `-o`), refuses to write into a non-empty directory unless `--force`, and `--force` only ever replaces the demo's own artifacts. The demo agent and trace generator moved into the package (`traceval.demo`); `examples/` keeps thin wrappers.
+- `traceval serve [dir]`: serves the analysis report directory on localhost with Python's stdlib http.server and prints the report URL. Not a web UI.
+- Generated `test_generated.py` opens with the three commands a new teammate needs; the run summary ends with a `traceval calibrate` hint for the report just written.
+- CI: wheel-based demo smoke job (build wheel, install into a clean venv, run `traceval demo` from an empty directory) so the pip-user path is what CI tests. Releases now gate on it.
+
+### Changed
+- Failure-signature tokens for `not_contains` checks are now distinctiveness-filtered: a token qualifies only if it appears in fewer than 10% of success outputs (same-cluster successes preferred, all successes in the db as fallback). If no token survives, the check is omitted rather than emitting a junk forbidden list.
+- `not_contains` matching is word-boundary based and case-insensitive instead of raw substring: forbidding "error" no longer false-fails a healthy "no errors found".
+- README rewritten: real command outputs (regenerable via `scripts/readme-outputs.sh`), a live screenshot of the analysis report, evidence-based feature claims, plain pipeline diagram, and the GitHub Action example pinned to a release tag.
+
+### Fixed
+- Generated run summary printed a literal `\n` before "traceval Run Summary" (jinja over-escaping in the conftest template).
+- `analysis/report.html` version badge was hardcoded to v0.1.0; it now shows the installed traceval version.
+
 ## [0.2.0] - 2026-07-02
 
 ### Added

traceval-0.2.2/CONTRIBUTING.md

@@ -0,0 +1,22 @@
+# Contributing to traceval
+
+## Setup
+
+```bash
+git clone https://github.com/theramkm/traceval.git
+cd traceval
+uv sync          # installs the package and dev dependencies (same as CI)
+```
+
+## Development loop
+
+```bash
+make test        # pytest -q
+make lint        # ruff check, ruff format --check, mypy
+make demo        # end-to-end smoke: healthy agent passes, buggy agent fails
+make all         # lint + test
+```
+
+CI runs the same commands on Python 3.11, 3.12, and 3.13, plus a
+wheel-based demo smoke job, and enforces 85% coverage. Keep all of it
+green; add a test for every behavior change.

traceval-0.2.2/Makefile

@@ -0,0 +1,14 @@
+.PHONY: test lint demo all
+
+test:
+	uv run pytest -q
+
+lint:
+	uv run ruff check src tests examples
+	uv run ruff format --check src tests examples
+	uv run mypy src/traceval
+
+demo:
+	uv run traceval demo -o /tmp/traceval-demo --force
+
+all: lint test

traceval-0.2.2/PKG-INFO

@@ -0,0 +1,212 @@
+Metadata-Version: 2.4
+Name: traceval
+Version: 0.2.2
+Summary: Trace-to-Eval Compiler
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: jsonschema>=4.17.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pytest>=8.0.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0
+Description-Content-Type: text/markdown
+
+# traceval: Trace-to-Eval Compiler
+
+<p align="center">
+  <img src="https://img.shields.io/badge/Python-3.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Version" />
+  <img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License" />
+</p>
+
+***"Your traces already know how your agent fails. traceval turns them into the test suite you never wrote."***
+
+Teams running LLM agents in production have observability traces, but only a fraction maintain evals. The raw material for good tests, thousands of real traces full of edge cases and errors, sits unused because turning it into a regression suite is manual and tedious.
+
+traceval ingests agent traces from standard sources, normalizes them into a canonical Pydantic model, labels outcomes, clusters task shapes, and compiles the result into a human-editable eval suite: YAML cases, a pytest harness, and judge rubric scaffolds.
+
+![Failure-cluster coverage report generated by traceval analyze](docs/img/report.png)
+
+## Quickstart
+
+```bash
+pip install traceval
+traceval demo
+open traceval-demo/analysis/report.html   # xdg-open on Linux
+```
+
+`traceval demo` runs the entire loop against a built-in demo agent: it generates 200 synthetic traces, ingests them, clusters the failures, compiles an eval suite, and then proves the headline claim by running that suite twice:
+
+```text
+=== Demo complete: healthy agent PASSED, buggy agent FAILED ===
+Failure-cluster report: traceval-demo/analysis/report.html
+Run report: traceval-demo/evals/runs/run_20260702T072029851802Z.json
+Run report: traceval-demo/evals/runs/run_20260702T072030171406Z.json
+
+Re-run any stage manually:
+  traceval ingest traceval-demo/synthetic_traces.jsonl -o traceval-demo/traces.db
+  traceval analyze traceval-demo/traces.db -o traceval-demo/analysis
+  traceval generate traceval-demo/traces.db -o traceval-demo/evals --include-failures
+  traceval run traceval-demo/evals --target traceval.demo.agent:invoke_agent --judge fake
+  traceval calibrate traceval-demo/evals/runs/run_20260702T072030171406Z.json
+```
+
+## How it works
+
+```mermaid
+graph LR
+    A[OTel / Langfuse / LangSmith traces] --> B[Canonical trace DB]
+    B --> C[Label and cluster]
+    C --> D[YAML cases + pytest + rubrics]
+    D --> E[Run, diff, calibrate]
+```
+
+## Features
+
+* Ingests OpenTelemetry GenAI, Langfuse, and LangSmith exports, plus generic JSONL. Malformed lines are logged as warnings instead of crashing the run (tested against corrupt fixtures in `tests/fixtures/`).
+* Labels every trace with a rule-based outcome taxonomy (`success`, `tool_error`, `validation_error`, `loop`, `timeout`, `bad_output`) that you can extend with your own Python rules via `--rules`.
+* Clusters task shapes with Jaccard shingle similarity, fully offline: no embeddings, no API calls. Numeric tokens are normalized, so "order 57978" and "order 12345" land in the same cluster.
+* Deterministic generation: regenerating a suite from the same database is byte-identical, so evals diff cleanly in git.
+* Regression cases are inverted: a failure trace asserts the failure does *not* recur (forbidden error signatures, tool-loop bounds, non-empty output), never that the agent reproduces it.
+* Redacts emails, credit cards, phone numbers, and API tokens before case inputs are written (add your own scrubber with `--redact-hook`).
+* `traceval run` exits nonzero on any failing case and diffs against a previous report with `--compare`, so CI can gate deploys on it.
+* `traceval calibrate` measures judge-vs-human agreement per cluster and flags rubrics the automated judge scores unreliably.
+
+## Walkthrough on your own traces
+
+The command outputs below are real, captured from a run over the demo trace set (regenerate them with `scripts/readme-outputs.sh`).
+
+### 1. Ingest
+
+```bash
+traceval ingest traces.jsonl -o traces.db   # --format auto|otel|langfuse|langsmith|generic
+```
+
+```text
+Ingested 200 traces (209 spans).
+```
+
+Malformed spans do not abort the ingest; warnings are written to `<traces.db>.log`.
+
+### 2. Analyze
+
+```bash
+traceval analyze traces.db -o analysis
+```
+
+```text
+Outcomes: success 60% · tool_error 15% · loop 10% · timeout 8% · validation_error 8%
+Clusters: 8 task clusters found.
+Top failure cluster: "refund stripe -> stripe_lookup -> (tool_error)" (30 traces)
+Report written to analysis/report.html
+```
+
+`analysis/report.html` is the single-file page shown in the screenshot above. Pass `--evals evals/` to overlay eval coverage per cluster, and `--rules my_rules.py` to add your own labeling rules. To view it over HTTP instead of `file://`, `traceval serve analysis` starts a stdlib localhost server and prints the report URL.
+
+### 3. Generate
+
+```bash
+traceval generate traces.db -o evals --include-failures
+```
+
+```text
+Wrote 8 eval cases across 8 clusters → evals/cases/*.yaml
+Wrote judge rubrics → evals/rubrics/*.md
+Wrote pytest harness → evals/test_generated.py, evals/conftest.py
+```
+
+Every case is a reviewable YAML file. Golden cases assert the recorded successful behavior. Regression cases, generated from failure traces, assert the failure does **not** recur: forbidden error tokens (word-boundary matched, filtered against tokens that success traces also use), tool-loop bounds, and non-empty output. A regression case passes for any agent that avoids that failure mode; golden cases carry general bug detection.
+
+### 4. Run
+
+```bash
+traceval run evals --target myapp.agent:invoke_agent --judge fake
+```
+
+```text
+traceval Run Summary
+┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┓
+┃ Case ID              ┃ Cluster    ┃ Outcome ┃ Latency (ms) ┃
+┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━┩
+│ c_0c422a7a__case_001 │ c_0c422a7a │  PASS   │          0.0 │
+│ c_1e5d0942__case_002 │ c_1e5d0942 │  PASS   │          0.0 │
+│ c_2c881177__case_003 │ c_2c881177 │  PASS   │          0.0 │
+│ c_361535b0__case_004 │ c_361535b0 │  PASS   │          0.0 │
+│ c_9a8a4644__case_005 │ c_9a8a4644 │  PASS   │          0.0 │
+│ c_d30af83a__case_006 │ c_d30af83a │  PASS   │          0.0 │
+│ c_d3f3b631__case_007 │ c_d3f3b631 │  PASS   │          0.0 │
+│ c_e834c13c__case_008 │ c_e834c13c │  PASS   │          0.0 │
+└──────────────────────┴────────────┴─────────┴──────────────┘
+Total: 8 | Passed: 8 | Failed: 0 | Errored: 0
+```
+
+The target is an HTTP URL or a `module:function` callable. Checks cover `exact`, `contains_any`, `not_contains`, `regex`, `json_schema`, `tool_sequence`, `no_tool_loop`, and `judge`. Run reports land in `<evals_dir>/runs/` (override with `--runs-dir`); pass `--compare <previous report>` to print regressions and improvements between runs. The exit code is nonzero when any case fails.
+
+### 5. Calibrate the judge
+
+An LLM judge is only as trustworthy as its agreement with human judgment. `calibrate` samples judge-scored results from a run report and presents each agent output for blind pass/fail labeling in the terminal; judge verdicts stay hidden until the end so they cannot anchor you.
+
+```bash
+traceval calibrate evals/runs/run_<timestamp>.json --sample 8
+```
+
+```text
+Judge Calibration Summary
+┏━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓
+┃ Cluster    ┃ Labeled ┃ Agreement ┃
+┡━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩
+│ c_0c422a7a │       1 │      100% │
+│ c_1e5d0942 │       1 │      100% │
+│ c_2c881177 │       1 │      100% │
+│ c_361535b0 │       1 │      100% │
+│ c_9a8a4644 │       1 │        0% │
+│ c_d30af83a │       1 │      100% │
+│ c_d3f3b631 │       1 │      100% │
+│ c_e834c13c │       1 │      100% │
+└────────────┴─────────┴───────────┘
+Overall agreement: 88% on 8 case(s) | false-pass (judge OK, human not): 1 | false-fail: 0
+WARNING: Judge unreliable (< 80% agreement) for clusters: c_9a8a4644. Review their rubrics before trusting automated scores.
+```
+
+False-pass counts (judge approved, human rejected) are called out because that is the dangerous direction: a lenient judge waves bad outputs into production. Clusters below `--min-agreement` (default 80%) are flagged for rubric review, and the full labels plus stats are written to `calibration.json`.
+
+## Scripting with --json
+
+`ingest`, `analyze`, `generate`, and `run` accept `--json`: human-readable output is suppressed and a single JSON object is printed to stdout. `run` still exits nonzero on failures.
+
+```bash
+traceval analyze traces.db --json | python -m json.tool
+```
+
+## GitHub Action
+
+Gate deploys on your generated eval suite. The action installs traceval, runs the suite, and fails the job on any regression:
+
+```yaml
+jobs:
+  agent-evals:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: theramkm/traceval@v0.2.2
+        with:
+          evals-dir: evals/
+          target: myapp.agent:invoke_agent   # or an HTTP URL
+          judge: fake                        # offline; 'openai' needs an API key
+```
+
+Inputs: `evals-dir` and `target` (required); `judge`, `compare`, `only`, `runs-dir`, `traceval-version`, `python-version` (optional). For a real LLM judge, set `judge: openai` and pass `OPENAI_API_KEY` (or `GEMINI_API_KEY`) via `env:` from your repository secrets.
+
+## Development
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for setup.
+Run the test suite with `make test` and the full gate set with `make lint`.
+
+## Honest Limitations
+
+* **Side-Effect Free**: traceval assertions evaluate input/output matches. It does not attempt to replay side effects (e.g., updating database records) on mock tools.
+* **Text Telemetry**: The canonical model is optimized for text logs. Image or multimodal payloads in traces are logged as references.
+* **Static Visualization**: The coverage report is a portable, single-file HTML page. There is no hosted web service.

traceval-0.2.2/README.md

@@ -0,0 +1,195 @@
+# traceval: Trace-to-Eval Compiler
+
+<p align="center">
+  <img src="https://img.shields.io/badge/Python-3.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Version" />
+  <img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License" />
+</p>
+
+***"Your traces already know how your agent fails. traceval turns them into the test suite you never wrote."***
+
+Teams running LLM agents in production have observability traces, but only a fraction maintain evals. The raw material for good tests, thousands of real traces full of edge cases and errors, sits unused because turning it into a regression suite is manual and tedious.
+
+traceval ingests agent traces from standard sources, normalizes them into a canonical Pydantic model, labels outcomes, clusters task shapes, and compiles the result into a human-editable eval suite: YAML cases, a pytest harness, and judge rubric scaffolds.
+
+![Failure-cluster coverage report generated by traceval analyze](docs/img/report.png)
+
+## Quickstart
+
+```bash
+pip install traceval
+traceval demo
+open traceval-demo/analysis/report.html   # xdg-open on Linux
+```
+
+`traceval demo` runs the entire loop against a built-in demo agent: it generates 200 synthetic traces, ingests them, clusters the failures, compiles an eval suite, and then proves the headline claim by running that suite twice:
+
+```text
+=== Demo complete: healthy agent PASSED, buggy agent FAILED ===
+Failure-cluster report: traceval-demo/analysis/report.html
+Run report: traceval-demo/evals/runs/run_20260702T072029851802Z.json
+Run report: traceval-demo/evals/runs/run_20260702T072030171406Z.json
+
+Re-run any stage manually:
+  traceval ingest traceval-demo/synthetic_traces.jsonl -o traceval-demo/traces.db
+  traceval analyze traceval-demo/traces.db -o traceval-demo/analysis
+  traceval generate traceval-demo/traces.db -o traceval-demo/evals --include-failures
+  traceval run traceval-demo/evals --target traceval.demo.agent:invoke_agent --judge fake
+  traceval calibrate traceval-demo/evals/runs/run_20260702T072030171406Z.json
+```
+
+## How it works
+
+```mermaid
+graph LR
+    A[OTel / Langfuse / LangSmith traces] --> B[Canonical trace DB]
+    B --> C[Label and cluster]
+    C --> D[YAML cases + pytest + rubrics]
+    D --> E[Run, diff, calibrate]
+```
+
+## Features
+
+* Ingests OpenTelemetry GenAI, Langfuse, and LangSmith exports, plus generic JSONL. Malformed lines are logged as warnings instead of crashing the run (tested against corrupt fixtures in `tests/fixtures/`).
+* Labels every trace with a rule-based outcome taxonomy (`success`, `tool_error`, `validation_error`, `loop`, `timeout`, `bad_output`) that you can extend with your own Python rules via `--rules`.
+* Clusters task shapes with Jaccard shingle similarity, fully offline: no embeddings, no API calls. Numeric tokens are normalized, so "order 57978" and "order 12345" land in the same cluster.
+* Deterministic generation: regenerating a suite from the same database is byte-identical, so evals diff cleanly in git.
+* Regression cases are inverted: a failure trace asserts the failure does *not* recur (forbidden error signatures, tool-loop bounds, non-empty output), never that the agent reproduces it.
+* Redacts emails, credit cards, phone numbers, and API tokens before case inputs are written (add your own scrubber with `--redact-hook`).
+* `traceval run` exits nonzero on any failing case and diffs against a previous report with `--compare`, so CI can gate deploys on it.
+* `traceval calibrate` measures judge-vs-human agreement per cluster and flags rubrics the automated judge scores unreliably.
+
+## Walkthrough on your own traces
+
+The command outputs below are real, captured from a run over the demo trace set (regenerate them with `scripts/readme-outputs.sh`).
+
+### 1. Ingest
+
+```bash
+traceval ingest traces.jsonl -o traces.db   # --format auto|otel|langfuse|langsmith|generic
+```
+
+```text
+Ingested 200 traces (209 spans).
+```
+
+Malformed spans do not abort the ingest; warnings are written to `<traces.db>.log`.
+
+### 2. Analyze
+
+```bash
+traceval analyze traces.db -o analysis
+```
+
+```text
+Outcomes: success 60% · tool_error 15% · loop 10% · timeout 8% · validation_error 8%
+Clusters: 8 task clusters found.
+Top failure cluster: "refund stripe -> stripe_lookup -> (tool_error)" (30 traces)
+Report written to analysis/report.html
+```
+
+`analysis/report.html` is the single-file page shown in the screenshot above. Pass `--evals evals/` to overlay eval coverage per cluster, and `--rules my_rules.py` to add your own labeling rules. To view it over HTTP instead of `file://`, `traceval serve analysis` starts a stdlib localhost server and prints the report URL.
+
+### 3. Generate
+
+```bash
+traceval generate traces.db -o evals --include-failures
+```
+
+```text
+Wrote 8 eval cases across 8 clusters → evals/cases/*.yaml
+Wrote judge rubrics → evals/rubrics/*.md
+Wrote pytest harness → evals/test_generated.py, evals/conftest.py
+```
+
+Every case is a reviewable YAML file. Golden cases assert the recorded successful behavior. Regression cases, generated from failure traces, assert the failure does **not** recur: forbidden error tokens (word-boundary matched, filtered against tokens that success traces also use), tool-loop bounds, and non-empty output. A regression case passes for any agent that avoids that failure mode; golden cases carry general bug detection.
+
+### 4. Run
+
+```bash
+traceval run evals --target myapp.agent:invoke_agent --judge fake
+```
+
+```text
+traceval Run Summary
+┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━━┓
+┃ Case ID              ┃ Cluster    ┃ Outcome ┃ Latency (ms) ┃
+┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━━┩
+│ c_0c422a7a__case_001 │ c_0c422a7a │  PASS   │          0.0 │
+│ c_1e5d0942__case_002 │ c_1e5d0942 │  PASS   │          0.0 │
+│ c_2c881177__case_003 │ c_2c881177 │  PASS   │          0.0 │
+│ c_361535b0__case_004 │ c_361535b0 │  PASS   │          0.0 │
+│ c_9a8a4644__case_005 │ c_9a8a4644 │  PASS   │          0.0 │
+│ c_d30af83a__case_006 │ c_d30af83a │  PASS   │          0.0 │
+│ c_d3f3b631__case_007 │ c_d3f3b631 │  PASS   │          0.0 │
+│ c_e834c13c__case_008 │ c_e834c13c │  PASS   │          0.0 │
+└──────────────────────┴────────────┴─────────┴──────────────┘
+Total: 8 | Passed: 8 | Failed: 0 | Errored: 0
+```
+
+The target is an HTTP URL or a `module:function` callable. Checks cover `exact`, `contains_any`, `not_contains`, `regex`, `json_schema`, `tool_sequence`, `no_tool_loop`, and `judge`. Run reports land in `<evals_dir>/runs/` (override with `--runs-dir`); pass `--compare <previous report>` to print regressions and improvements between runs. The exit code is nonzero when any case fails.
+
+### 5. Calibrate the judge
+
+An LLM judge is only as trustworthy as its agreement with human judgment. `calibrate` samples judge-scored results from a run report and presents each agent output for blind pass/fail labeling in the terminal; judge verdicts stay hidden until the end so they cannot anchor you.
+
+```bash
+traceval calibrate evals/runs/run_<timestamp>.json --sample 8
+```
+
+```text
+Judge Calibration Summary
+┏━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━┓
+┃ Cluster    ┃ Labeled ┃ Agreement ┃
+┡━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━┩
+│ c_0c422a7a │       1 │      100% │
+│ c_1e5d0942 │       1 │      100% │
+│ c_2c881177 │       1 │      100% │
+│ c_361535b0 │       1 │      100% │
+│ c_9a8a4644 │       1 │        0% │
+│ c_d30af83a │       1 │      100% │
+│ c_d3f3b631 │       1 │      100% │
+│ c_e834c13c │       1 │      100% │
+└────────────┴─────────┴───────────┘
+Overall agreement: 88% on 8 case(s) | false-pass (judge OK, human not): 1 | false-fail: 0
+WARNING: Judge unreliable (< 80% agreement) for clusters: c_9a8a4644. Review their rubrics before trusting automated scores.
+```
+
+False-pass counts (judge approved, human rejected) are called out because that is the dangerous direction: a lenient judge waves bad outputs into production. Clusters below `--min-agreement` (default 80%) are flagged for rubric review, and the full labels plus stats are written to `calibration.json`.
+
+## Scripting with --json
+
+`ingest`, `analyze`, `generate`, and `run` accept `--json`: human-readable output is suppressed and a single JSON object is printed to stdout. `run` still exits nonzero on failures.
+
+```bash
+traceval analyze traces.db --json | python -m json.tool
+```
+
+## GitHub Action
+
+Gate deploys on your generated eval suite. The action installs traceval, runs the suite, and fails the job on any regression:
+
+```yaml
+jobs:
+  agent-evals:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: theramkm/traceval@v0.2.2
+        with:
+          evals-dir: evals/
+          target: myapp.agent:invoke_agent   # or an HTTP URL
+          judge: fake                        # offline; 'openai' needs an API key
+```
+
+Inputs: `evals-dir` and `target` (required); `judge`, `compare`, `only`, `runs-dir`, `traceval-version`, `python-version` (optional). For a real LLM judge, set `judge: openai` and pass `OPENAI_API_KEY` (or `GEMINI_API_KEY`) via `env:` from your repository secrets.
+
+## Development
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for setup.
+Run the test suite with `make test` and the full gate set with `make lint`.
+
+## Honest Limitations
+
+* **Side-Effect Free**: traceval assertions evaluate input/output matches. It does not attempt to replay side effects (e.g., updating database records) on mock tools.
+* **Text Telemetry**: The canonical model is optimized for text logs. Image or multimodal payloads in traces are logged as references.
+* **Static Visualization**: The coverage report is a portable, single-file HTML page. There is no hosted web service.

traceval-0.2.2/examples/demo.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# traceval e2e quickstart: the whole loop now lives in `traceval demo`
+# (generate traces -> ingest -> analyze -> generate evals -> run healthy
+# agent, must pass -> run buggy agent, must fail).
+set -e
+
+uv run traceval demo -o traceval-demo --force "$@"

traceval-0.2.2/examples/demo_agent/core.py

@@ -0,0 +1,8 @@
+"""Thin wrapper: the demo agent now ships inside the package so
+`traceval demo` works from a plain pip install. Kept so existing
+`--target examples.demo_agent.core:invoke_agent` invocations still work.
+"""
+
+from traceval.demo.agent import invoke_agent, run_agent_logic
+
+__all__ = ["invoke_agent", "run_agent_logic"]

traceval-0.2.2/examples/make_traces.py

@@ -0,0 +1,11 @@
+"""Thin wrapper: the trace generator now ships inside the package
+(traceval.demo.traces) so `traceval demo` works from a plain pip install.
+"""
+
+from pathlib import Path
+
+from traceval.demo.traces import generate_traces_file
+
+if __name__ == "__main__":
+    out = Path(__file__).parent / "synthetic_traces.jsonl"
+    generate_traces_file(out)

{traceval-0.2.0 → traceval-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "traceval"
-version = "0.2.0"
+version = "0.2.2"
 description = "Trace-to-Eval Compiler"
 readme = "README.md"
 requires-python = ">=3.11"

traceval-0.2.2/scripts/readme-outputs.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Regenerates every sample-output block in README.md, plus
+# docs/img/report.png, from real command runs so the docs cannot drift.
+# Run from anywhere; paste the printed sections into README.md verbatim.
+set -e
+
+ROOT=$(cd "$(dirname "$0")/.." && pwd)
+WORK=$(mktemp -d)
+trap 'rm -rf "$WORK"' EXIT
+cd "$WORK"
+
+tv() { uv run --project "$ROOT" traceval "$@"; }
+
+uv run --project "$ROOT" python3 -c "
+from pathlib import Path
+from traceval.demo.traces import generate_traces_file
+generate_traces_file(Path('traces.jsonl'))
+" >/dev/null
+
+echo "=== ingest ==="
+tv ingest traces.jsonl -o traces.db
+
+echo
+echo "=== analyze ==="
+tv analyze traces.db -o analysis
+
+echo
+echo "=== generate ==="
+tv generate traces.db -o evals --include-failures
+
+echo
+echo "=== run (healthy demo agent) ==="
+tv run evals --target traceval.demo.agent:invoke_agent --judge fake || true
+
+echo
+echo "=== calibrate (example labels: 7x pass, 1x fail) ==="
+REPORT=$(ls -t evals/runs/run_*.json | head -1)
+printf 'y\ny\ny\ny\nn\ny\ny\ny\n' | tv calibrate "$REPORT" --sample 8 | tail -16
+
+echo
+echo "=== screenshot -> docs/img/report.png ==="
+# Re-analyze with --evals so the report shows populated eval coverage
+tv analyze traces.db --evals evals -o analysis >/dev/null
+CHROME="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+if [ -x "$CHROME" ]; then
+  "$CHROME" --headless --disable-gpu \
+    --screenshot="$ROOT/docs/img/report.png" \
+    --window-size=1280,860 --hide-scrollbars \
+    "file://$WORK/analysis/report.html" 2>/dev/null
+  echo "wrote $ROOT/docs/img/report.png"
+else
+  echo "Chrome not found; screenshot skipped"
+fi

traceval-0.2.2/src/traceval/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.2"