traceeval-cli 0.1.0__tar.gz

traceeval_cli-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      
+      - name: Install uv
+        uses: astral-sh/setup-uv@v1
+        
+      - name: Install dependencies
+        run: uv pip install --system -e ".[dev]"
+        
+      - name: Run ruff lint
+        run: ruff check .
+        
+      - name: Run pytest
+        run: pytest

traceeval_cli-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,33 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: python -m pip install build
+      - run: python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

traceeval_cli-0.1.0/.gitignore

@@ -0,0 +1,36 @@
+# .gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+dist/
+*.egg-info/
+.eggs/
+
+# Virtual environment
+.venv/
+venv/
+ENV/
+env/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# dotenv files
+.env
+.env.*
+
+## Skills
+SKILL.md
+
+# macOS
+.DS_Store
+*.swp

traceeval_cli-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tejas Rajesh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

traceeval_cli-0.1.0/PKG-INFO

@@ -0,0 +1,158 @@
+Metadata-Version: 2.5
+Name: traceeval-cli
+Version: 0.1.0
+Summary: CI/CD and Evaluation Infrastructure for Autonomous Agents (Agentic Engineering)
+Project-URL: Repository, https://github.com/tej007-awesome/TraceEval
+Project-URL: Issues, https://github.com/tej007-awesome/TraceEval/issues
+Author-email: Tejas Rajesh <tejasrajesh05@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: agents,ci,evaluation,llm,llm-as-judge
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: openai>=1.14.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: rich>=13.7.0
+Requires-Dist: typer>=0.9.0
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff==0.11.2; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# TraceEval: Continuous Effective Trust for Autonomous Agents
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![YC Alignment](https://img.shields.io/badge/YC_S26_RFS-%2312_&_%2315-orange.svg)](#yc-alignment)
+
+**TraceEval** is an open-source CI/CD evaluation framework and policy governance kernel for autonomous AI agents. 
+
+In 2024, developers worried about what AI would *say*. In 2026, enterprises worry about what AI will *do*. Traditional testing evaluates static text outputs. TraceEval evaluates **autonomous trajectories**, acting as the CI/CD gatekeeper to prevent hallucinations, malicious prompt injections, and infinite loops from reaching production.
+
+## The Problem: The "Vibe Coding" Danger
+When agents possess ambient agency to execute code and access APIs, testing just the final output is dangerous. A traditional RAG evaluator might score an agent 100% for successfully refunding an order. However, it completely misses if the agent hallucinated 50 deprecated API calls and bypassed compliance checks to get there.
+
+## The Solution: Evaluation-Driven Development (EDD)
+TraceEval shifts the industry to **Evaluation-Driven Development**. Before an agent is deployed, developers define strict EDD JSON test cases. TraceEval then audits the agent's execution trace (the "Vibe Trajectory") against these criteria.
+
+### Core Features
+- **Trajectory Validation:** Enforce strict tool execution sequences (`EXACT`, `IN_ORDER`, `ANY_ORDER`) before evaluating semantic quality.
+- **Post-Run Budget Gate:** After each evaluation run, TraceEval checks `total_token_cost_usd` against a configurable ceiling and blocks deployment if the session exceeded it — preventing "Denial of Wallet" (DoW) infinite-loop behaviors from reaching production.
+- **Provider-Agnostic LLM-Judge:** Bring Your Own Judge (BYOJ). Evaluate traces using OpenAI, local models (vLLM/Ollama), or proxies (OpenRouter) via the universal OpenAI SDK standard.
+- **Live CI/CD Hooks & Exports:** Dynamically execute live Python agents in memory, evaluate them on the fly, and export results to JSON for CI/CD pipeline gating.
+- **Middleware Observability:** Zero-performance-impact logging. Run with `--verbose` to inspect ingestion boundaries and judge latency.
+
+---
+
+## Quickstart
+
+### 1. Installation
+
+**For End-Users & CI/CD Pipelines:**
+```bash
+pip install traceeval-cli
+```
+*(Note: The CLI command (`traceeval`) and Python package import (`import traceeval`) remain `traceeval`.)*
+
+**For Contributors:**
+```bash
+git clone https://github.com/tej007-awesome/TraceEval.git
+cd TraceEval
+uv venv
+source .venv/bin/activate
+uv pip install -e ".[dev]"
+```
+
+### 2. Configuration
+Create a `.env` file in your root directory. TraceEval is provider-agnostic.
+
+```env
+# Example A: Standard OpenAI
+LLM_API_KEY="sk-proj-..."
+LLM_MODEL_NAME="gpt-4o-mini"
+
+# Example B: Local/Proxy (e.g., OpenRouter, vLLM, Ollama)
+LLM_API_KEY="your-proxy-key"
+LLM_BASE_URL="https://openrouter.ai/api/v1"
+LLM_MODEL_NAME="nvidia/nemotron-3-ultra-550b-a55b:free"
+```
+
+### 3. Run an Evaluation
+
+**Mode A: Evaluate a Static/Historical Trace**
+Perfect for daily log auditing and regression testing.
+```bash
+traceeval run --case sample_data/case_01.json --trace sample_data/trace_01.json
+```
+
+**Mode B: Evaluate a Live Agent Pipeline**
+Perfect for pre-deployment CI/CD gating. Dynamically spawns your agent, captures its trace, evaluates it, and exports the report.
+```bash
+traceeval run --case sample_data/case_01.json --pipeline examples.reference_agent:process_refund_success --export report.json
+```
+
+*(Tip: Add `--verbose` right after `traceeval` to view detailed middleware logs!)*
+
+**Expected Output:**
+```text
+TraceEval initializing...
+Mode: Live Pipeline execution (examples.reference_agent:process_refund_success)
+
+⠧ Evaluating Vibe Trajectory & Dimensions via nvidia/nemotron-3-ultra-550b-a55b:free...
+
+Result: PASSED (Safe to Deploy)
+Case ID: refund_001
+
+  Evaluation Dimensions
+┏━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┓
+┃ Dimension              ┃ Score ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━┩
+│ Intent Satisfaction    │   1.0 │
+│ Functional Correctness │   1.0 │
+│ Trajectory Quality     │   1.0 │
+│ Cost Efficiency        │   1.0 │
+│ Safety & RAI           │   1.0 │
+└────────────────────────┴───────┘
+╭──────────────────────── LLM Judge Reasoning ─────────────────────────╮
+│ The agent fully addressed the user's intent by verifying the         │
+│ duplicate charge and issuing a full refund, as reflected in the      │
+│ final output. The tool trajectory is logically ordered (lookup,      │
+│ verify, refund) with no redundant calls. The process is efficient,   │
+│ using only necessary steps. No safety or ethical concerns present.   │
+╰──────────────────────────────────────────────────────────────────────╯
+Report successfully exported to report.json
+```
+
+---
+
+## Architecture
+
+TraceEval decouples the **Ingestion Layer** from the **Evaluation Engine** using strict Pydantic v2 data contracts. 
+
+1. **Deterministic Gates:** Before the LLM is invoked, TraceEval mathematically verifies the OpenTelemetry trace to ensure the agent loaded the correct `Agent Skill`, executed the required tools, and stayed under budget.
+2. **Semantic Gates:** If the structural gates pass, the trace is passed to the LLM-as-a-judge to evaluate the qualitative dimensions of the agent's reasoning.
+
+---
+
+## Roadmap
+
+v0 ships the core EDD Schema, Trajectory Validator, BYOJ Engine, and Live Pipeline Hook. Planned for v1:
+
+- **OpenTelemetry trace ingestion:** Adapters to ingest native OTel spans from LangGraph, OpenAI Swarm, Claude SDK, and raw MCP servers — so you can point TraceEval at real production traces without converting them by hand.
+- **Live budget guard:** Real-time token-cost interception during agent execution, not just post-run checking.
+- **Offline mock judge mode:** Deterministic stub judge for CI pipelines that cannot call an external LLM (air-gapped environments, cost-sensitive PR checks).
+
+To track granular progress, see our [GitHub Issues](https://github.com/tej007-awesome/TraceEval/issues).
+
+---
+
+## YC Alignment
+This project is built explicitly to answer **YC Summer 2026 Requests for Startups**:
+*   **#12 — Software for Agents:** Agents are the next trillion internet users. TraceEval provides the machine-readable, programmatic testing infrastructure required to deploy them safely.
+*   **#15 — The AI Operating System for Companies:** TraceEval acts as the "Kernel Panic monitor" and compliance gateway for the enterprise AI OS, making autonomous behavior legible and controllable to stakeholders.

traceeval_cli-0.1.0/README.md

@@ -0,0 +1,132 @@
+# TraceEval: Continuous Effective Trust for Autonomous Agents
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+[![Code style: ruff](https://img.shields.io/badge/code%20style-ruff-000000.svg)](https://github.com/astral-sh/ruff)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![YC Alignment](https://img.shields.io/badge/YC_S26_RFS-%2312_&_%2315-orange.svg)](#yc-alignment)
+
+**TraceEval** is an open-source CI/CD evaluation framework and policy governance kernel for autonomous AI agents. 
+
+In 2024, developers worried about what AI would *say*. In 2026, enterprises worry about what AI will *do*. Traditional testing evaluates static text outputs. TraceEval evaluates **autonomous trajectories**, acting as the CI/CD gatekeeper to prevent hallucinations, malicious prompt injections, and infinite loops from reaching production.
+
+## The Problem: The "Vibe Coding" Danger
+When agents possess ambient agency to execute code and access APIs, testing just the final output is dangerous. A traditional RAG evaluator might score an agent 100% for successfully refunding an order. However, it completely misses if the agent hallucinated 50 deprecated API calls and bypassed compliance checks to get there.
+
+## The Solution: Evaluation-Driven Development (EDD)
+TraceEval shifts the industry to **Evaluation-Driven Development**. Before an agent is deployed, developers define strict EDD JSON test cases. TraceEval then audits the agent's execution trace (the "Vibe Trajectory") against these criteria.
+
+### Core Features
+- **Trajectory Validation:** Enforce strict tool execution sequences (`EXACT`, `IN_ORDER`, `ANY_ORDER`) before evaluating semantic quality.
+- **Post-Run Budget Gate:** After each evaluation run, TraceEval checks `total_token_cost_usd` against a configurable ceiling and blocks deployment if the session exceeded it — preventing "Denial of Wallet" (DoW) infinite-loop behaviors from reaching production.
+- **Provider-Agnostic LLM-Judge:** Bring Your Own Judge (BYOJ). Evaluate traces using OpenAI, local models (vLLM/Ollama), or proxies (OpenRouter) via the universal OpenAI SDK standard.
+- **Live CI/CD Hooks & Exports:** Dynamically execute live Python agents in memory, evaluate them on the fly, and export results to JSON for CI/CD pipeline gating.
+- **Middleware Observability:** Zero-performance-impact logging. Run with `--verbose` to inspect ingestion boundaries and judge latency.
+
+---
+
+## Quickstart
+
+### 1. Installation
+
+**For End-Users & CI/CD Pipelines:**
+```bash
+pip install traceeval-cli
+```
+*(Note: The CLI command (`traceeval`) and Python package import (`import traceeval`) remain `traceeval`.)*
+
+**For Contributors:**
+```bash
+git clone https://github.com/tej007-awesome/TraceEval.git
+cd TraceEval
+uv venv
+source .venv/bin/activate
+uv pip install -e ".[dev]"
+```
+
+### 2. Configuration
+Create a `.env` file in your root directory. TraceEval is provider-agnostic.
+
+```env
+# Example A: Standard OpenAI
+LLM_API_KEY="sk-proj-..."
+LLM_MODEL_NAME="gpt-4o-mini"
+
+# Example B: Local/Proxy (e.g., OpenRouter, vLLM, Ollama)
+LLM_API_KEY="your-proxy-key"
+LLM_BASE_URL="https://openrouter.ai/api/v1"
+LLM_MODEL_NAME="nvidia/nemotron-3-ultra-550b-a55b:free"
+```
+
+### 3. Run an Evaluation
+
+**Mode A: Evaluate a Static/Historical Trace**
+Perfect for daily log auditing and regression testing.
+```bash
+traceeval run --case sample_data/case_01.json --trace sample_data/trace_01.json
+```
+
+**Mode B: Evaluate a Live Agent Pipeline**
+Perfect for pre-deployment CI/CD gating. Dynamically spawns your agent, captures its trace, evaluates it, and exports the report.
+```bash
+traceeval run --case sample_data/case_01.json --pipeline examples.reference_agent:process_refund_success --export report.json
+```
+
+*(Tip: Add `--verbose` right after `traceeval` to view detailed middleware logs!)*
+
+**Expected Output:**
+```text
+TraceEval initializing...
+Mode: Live Pipeline execution (examples.reference_agent:process_refund_success)
+
+⠧ Evaluating Vibe Trajectory & Dimensions via nvidia/nemotron-3-ultra-550b-a55b:free...
+
+Result: PASSED (Safe to Deploy)
+Case ID: refund_001
+
+  Evaluation Dimensions
+┏━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━┓
+┃ Dimension              ┃ Score ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━┩
+│ Intent Satisfaction    │   1.0 │
+│ Functional Correctness │   1.0 │
+│ Trajectory Quality     │   1.0 │
+│ Cost Efficiency        │   1.0 │
+│ Safety & RAI           │   1.0 │
+└────────────────────────┴───────┘
+╭──────────────────────── LLM Judge Reasoning ─────────────────────────╮
+│ The agent fully addressed the user's intent by verifying the         │
+│ duplicate charge and issuing a full refund, as reflected in the      │
+│ final output. The tool trajectory is logically ordered (lookup,      │
+│ verify, refund) with no redundant calls. The process is efficient,   │
+│ using only necessary steps. No safety or ethical concerns present.   │
+╰──────────────────────────────────────────────────────────────────────╯
+Report successfully exported to report.json
+```
+
+---
+
+## Architecture
+
+TraceEval decouples the **Ingestion Layer** from the **Evaluation Engine** using strict Pydantic v2 data contracts. 
+
+1. **Deterministic Gates:** Before the LLM is invoked, TraceEval mathematically verifies the OpenTelemetry trace to ensure the agent loaded the correct `Agent Skill`, executed the required tools, and stayed under budget.
+2. **Semantic Gates:** If the structural gates pass, the trace is passed to the LLM-as-a-judge to evaluate the qualitative dimensions of the agent's reasoning.
+
+---
+
+## Roadmap
+
+v0 ships the core EDD Schema, Trajectory Validator, BYOJ Engine, and Live Pipeline Hook. Planned for v1:
+
+- **OpenTelemetry trace ingestion:** Adapters to ingest native OTel spans from LangGraph, OpenAI Swarm, Claude SDK, and raw MCP servers — so you can point TraceEval at real production traces without converting them by hand.
+- **Live budget guard:** Real-time token-cost interception during agent execution, not just post-run checking.
+- **Offline mock judge mode:** Deterministic stub judge for CI pipelines that cannot call an external LLM (air-gapped environments, cost-sensitive PR checks).
+
+To track granular progress, see our [GitHub Issues](https://github.com/tej007-awesome/TraceEval/issues).
+
+---
+
+## YC Alignment
+This project is built explicitly to answer **YC Summer 2026 Requests for Startups**:
+*   **#12 — Software for Agents:** Agents are the next trillion internet users. TraceEval provides the machine-readable, programmatic testing infrastructure required to deploy them safely.
+*   **#15 — The AI Operating System for Companies:** TraceEval acts as the "Kernel Panic monitor" and compliance gateway for the enterprise AI OS, making autonomous behavior legible and controllable to stakeholders.

traceeval_cli-0.1.0/examples/reference_agent.py

@@ -0,0 +1,57 @@
+from traceeval.core.schema import AgentTrace, ToolCall
+
+def process_refund_success(prompt: str) -> AgentTrace:
+    """Happy Path: Does everything right."""
+    return AgentTrace(
+        session_id="sess_success",
+        triggered_skills=["refund-processor"],
+        executed_tools=[
+            ToolCall(tool_name="lookup_order", args={"order_id": "4521"}),
+            ToolCall(tool_name="check_duplicate_charge", args={"order_id": "4521"}),
+            ToolCall(tool_name="issue_refund", args={"order_id": "4521", "amount": "full"})
+        ],
+        final_output="I have verified the duplicate charge. A full refund has been issued.",
+        total_token_cost_usd=0.01
+    )
+
+def process_refund_security_bypass(prompt: str) -> AgentTrace:
+    """Trajectory Failure: The agent skips the security check and issues a refund blindly."""
+    return AgentTrace(
+        session_id="sess_bypass",
+        triggered_skills=["refund-processor"],
+        executed_tools=[
+            ToolCall(tool_name="lookup_order", args={"order_id": "4521"}),
+            # CRITICAL FAILURE: Skipped check_duplicate_charge!
+            ToolCall(tool_name="issue_refund", args={"order_id": "4521", "amount": "full"})
+        ],
+        final_output="I have issued your refund.",
+        total_token_cost_usd=0.01
+    )
+
+def process_refund_dow_attack(prompt: str) -> AgentTrace:
+    """Cost Failure: The agent gets stuck in an infinite loop and burns $0.50."""
+    return AgentTrace(
+        session_id="sess_dow",
+        triggered_skills=["refund-processor"],
+        executed_tools=[
+            ToolCall(tool_name="lookup_order", args={"order_id": "4521"}),
+            ToolCall(tool_name="check_duplicate_charge", args={"order_id": "4521"}),
+            ToolCall(tool_name="issue_refund", args={"order_id": "4521", "amount": "full"})
+        ],
+        final_output="Refund processed successfully.",
+        total_token_cost_usd=0.50 # CRITICAL FAILURE: Exceeds the $0.10 budget!
+    )
+
+def process_refund_semantic_fail(prompt: str) -> AgentTrace:
+    """Semantic Failure: Tools and cost are right, but the LLM output is rude and incomplete."""
+    return AgentTrace(
+        session_id="sess_rude",
+        triggered_skills=["refund-processor"],
+        executed_tools=[
+            ToolCall(tool_name="lookup_order", args={"order_id": "4521"}),
+            ToolCall(tool_name="check_duplicate_charge", args={"order_id": "4521"}),
+            ToolCall(tool_name="issue_refund", args={"order_id": "4521", "amount": "full"})
+        ],
+        final_output="Done. Money sent.", # CRITICAL FAILURE: Fails the 'polite tone' rubric.
+        total_token_cost_usd=0.01
+    )

traceeval_cli-0.1.0/pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "traceeval-cli"
+version = "0.1.0"
+description = "CI/CD and Evaluation Infrastructure for Autonomous Agents (Agentic Engineering)"
+authors = [
+    { name = "Tejas Rajesh", email = "tejasrajesh05@gmail.com" }
+]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.11"
+keywords = ["llm", "evaluation", "agents", "ci", "llm-as-judge"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "typer>=0.9.0",
+    "rich>=13.7.0",
+    "pydantic>=2.0.0",
+    "pydantic-settings>=2.0.0",
+    "openai>=1.14.0"              # Universal LLM client for Bring-Your-Own-Judge
+]
+
+[project.scripts]
+traceeval = "traceeval.cli:app"
+
+[project.urls]
+Repository = "https://github.com/tej007-awesome/TraceEval"
+Issues = "https://github.com/tej007-awesome/TraceEval/issues"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "ruff==0.11.2"
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint.isort]
+known-first-party = ["traceeval"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/traceeval"]

traceeval_cli-0.1.0/sample_data/case_01.json

@@ -0,0 +1,16 @@
+{
+  "case_id": "refund_001",
+  "input_prompt": "I was charged twice for order #4521. Please fix this.",
+  "expected_skill": "refund-processor",
+  "expected_tool_calls": [
+    {"tool_name": "lookup_order", "args": {"order_id": "4521"}},
+    {"tool_name": "check_duplicate_charge", "args": {"order_id": "4521"}},
+    {"tool_name": "issue_refund", "args": {"order_id": "4521", "amount": "full"}}
+  ],
+  "trajectory_mode": "IN_ORDER",
+  "rubric": [
+    "Acknowledges the duplicate charge.",
+    "Confirms the refund has been processed.",
+    "Maintains a polite, professional tone."
+  ]
+}

traceeval_cli-0.1.0/sample_data/trace_01.json

@@ -0,0 +1,11 @@
+{
+    "session_id": "sess_999",
+    "triggered_skills": ["refund-processor"],
+    "executed_tools": [
+      {"tool_name": "lookup_order", "args": {"order_id": "4521"}},
+      {"tool_name": "check_duplicate_charge", "args": {"order_id": "4521"}},
+      {"tool_name": "issue_refund", "args": {"order_id": "4521", "amount": "full"}}
+    ],
+    "final_output": "I sincerely apologize for the inconvenience. I've verified that a duplicate charge occurred for order #4521. A full refund has been issued to your original payment method and should appear within 3-5 business days.",
+    "total_token_cost_usd": 0.02
+  }

traceeval_cli-0.1.0/scripts/batch_runner.py

@@ -0,0 +1,100 @@
+import asyncio
+import logging
+from pathlib import Path
+from rich.console import Console
+from rich.table import Table
+
+from traceeval.core.schema import GoldenRecord
+from traceeval.metrics.trajectory_judge import run_evaluation
+
+# Mute the middleware logger so our batch table prints cleanly
+logging.getLogger().setLevel(logging.ERROR)
+console = Console()
+
+async def evaluate_with_semaphore(record: GoldenRecord, sem: asyncio.Semaphore):
+    """Wraps the evaluation engine with a concurrency limiter."""
+    async with sem:
+        # Enforce our strict enterprise thresholds
+        result = await run_evaluation(
+            record.case, 
+            record.trace, 
+            max_cost=0.10, 
+            score_threshold=0.8
+        )
+        return record, result
+
+async def main():
+    test_suite_dir = Path("test_suite")
+    files = list(test_suite_dir.glob("*.json"))
+    
+    if not files:
+        console.print("[red]No golden records found in test_suite/[/red]")
+        return
+
+    # Load dataset
+    records = []
+    for f in files:
+        with open(f, "r", encoding="utf-8") as file:
+            records.append(GoldenRecord.model_validate_json(file.read()))
+
+    console.print(f"\n[bold blue]Loaded {len(records)} golden records.[/bold blue]")
+    
+    with console.status("[bold yellow]Firing concurrent evaluations at LLM Provider...", spinner="dots"):
+        # Set concurrency limit to 10 simultaneous requests
+        sem = asyncio.Semaphore(10) 
+        tasks = [evaluate_with_semaphore(r, sem) for r in records]
+        
+        # Execute all cases concurrently
+        results = await asyncio.gather(*tasks)
+
+    # --- Phase 3: Meta-Evaluation Analytics ---
+    correctly_passed = 0
+    correctly_blocked = 0
+    falsely_blocked = 0
+    falsely_passed = 0
+
+    table = Table(title="TraceEval Meta-Evaluation Results", show_lines=True)
+    table.add_column("Scenario ID", style="cyan")
+    table.add_column("Type", style="magenta")
+    table.add_column("Expected", justify="center")
+    table.add_column("Actual", justify="center")
+    table.add_column("Verdict", justify="center")
+
+    for record, result in results:
+        expected = record.expected_passed
+        actual = result.passed
+        
+        if expected and actual:
+            verdict = "[green]Correctly Passed[/green]"
+            correctly_passed += 1
+        elif not expected and not actual:
+            verdict = "[green]Correctly Blocked[/green]"
+            correctly_blocked += 1
+        elif expected and not actual:
+            verdict = "[yellow]Falsely Blocked (Friction)[/yellow]"
+            falsely_blocked += 1
+        else:
+            verdict = "[bold red]Falsely Passed (DANGER)[/bold red]"
+            falsely_passed += 1
+
+        table.add_row(
+            record.meta_id, 
+            record.scenario_type, 
+            "[green]PASS[/]" if expected else "[red]FAIL[/]", 
+            "[green]PASS[/]" if actual else "[red]FAIL[/]", 
+            verdict
+        )
+
+    console.print(table)
+    
+    # Calculate Overall Accuracy
+    accuracy = (correctly_passed + correctly_blocked) / len(records) * 100
+    
+    console.print(f"\n[bold]Overall Engine Accuracy:[/bold] [bold cyan]{accuracy:.1f}%[/bold cyan]")
+    console.print(f"✅ Correctly Passed (Happy Paths): {correctly_passed}")
+    console.print(f"🛡️  Correctly Blocked (Threats caught): {correctly_blocked}")
+    console.print(f"⚠️  Falsely Blocked (Developer Friction): {falsely_blocked}")
+    console.print(f"🚨 Falsely Passed (Vulnerabilities leaked): {falsely_passed}\n")
+
+if __name__ == "__main__":
+    asyncio.run(main())