traceval 0.1.1__tar.gz

traceval-0.1.1/.github/workflows/ci.yml

@@ -0,0 +1,83 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv and set up Python
+      uses: astral-sh/setup-uv@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        enable-cache: true
+
+    - name: Run Ruff Check
+      run: uv run ruff check src/ tests/ examples/
+
+    - name: Run Ruff Format Check
+      run: uv run ruff format --check src/ tests/ examples/
+
+    - name: Run Mypy
+      run: uv run mypy src/
+
+    - name: Run Tests with Coverage
+      run: uv run pytest --cov=src/traceval --cov-fail-under=85
+
+  tag-and-release:
+    needs: test
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Install uv and set up Python
+      uses: astral-sh/setup-uv@v5
+      with:
+        python-version: "3.12"
+        enable-cache: true
+
+    - name: Check version and build
+      id: check-ver
+      run: |
+        VERSION=$(grep -m 1 'version =' pyproject.toml | tr -d '"' | tr -d "'" | awk '{print $NF}')
+        TAG="v$VERSION"
+        
+        if git rev-parse "$TAG" >/dev/null 2>&1; then
+          echo "Tag $TAG already exists, skipping release."
+          echo "publish=false" >> $GITHUB_OUTPUT
+        else
+          echo "New version detected. Building and releasing $TAG..."
+          uv build
+          
+          # Configure Git and push tag
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git tag "$TAG"
+          git push origin "$TAG"
+          
+          echo "publish=true" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Publish to PyPI
+      if: steps.check-ver.outputs.publish == 'true'
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        skip-existing: true
+
+

traceval-0.1.1/.gitignore

@@ -0,0 +1,68 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Pytest / coverage
+.cache
+.pytest_cache/
+.coverage
+.coverage.*
+.traceback
+nosetests.xml
+coverage.xml
+htmlcov/
+.htmlcov/
+
+# Mypy / Ruff
+.mypy_cache/
+.ruff_cache/
+
+# Environments
+.env
+.venv/
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# SQLite databases & logs (traceval database runs)
+*.db
+*.log
+demo_analysis/
+demo_evals/
+evals/
+
+# Specifications & user instructions
+spec.md
+
+# OS files
+.DS_Store
+.DS_Store?
+ehthumbs.db
+Icon?
+Thumbs.db

traceval-0.1.1/.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repr: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.2.0
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format

traceval-0.1.1/CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-07-02
+
+### Added
+- **Canonical Model & SQLite Storage**: Structured canonical schema representing `Trace`, `Step`, `ToolCall`, `LLMCall`, and `Outcome`.
+- **Multi-backend Telemetry Ingestion**: Added log adapters for OpenTelemetry GenAI, Langfuse observation dumps, LangSmith logs, and Generic JSONL files.
+- **Rule-based Labeler**: Dynamic outcome labels classifier (`success`, `tool_error`, `validation_error`, `loop`, `timeout`, `bad_output`, `unknown`) with custom python rule plugins.
+- **Agglomerative Clustering**: Signature and task-input Jaccard shingle clustering.
+- **Jinja2 Coverage Report Visualizer**: Ported self-contained single-page HTML report charts.
+- **Pytest Case Compiler**: Emitter of YAML test case configurations, LLM-as-judge scaffolds, and custom redact hook scrubbers.
+- **Scorers & Judges**: Scorer implementations for `exact`, `contains`, `regex`, `json_schema`, `tool_sequence` (order/subset modes), and `judge` (FakeJudge, OpenAICompatJudge with call caps).
+- **FastAPI Demo Agent**: Supporting mock customer service tools and BUGGY mode regressions checks.

traceval-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ramkumar M
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

traceval-0.1.1/PKG-INFO

@@ -0,0 +1,154 @@
+Metadata-Version: 2.4
+Name: traceval
+Version: 0.1.1
+Summary: Trace-to-Eval Compiler
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.24.0
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: jsonschema>=4.17.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.9.0
+Description-Content-Type: text/markdown
+
+# 🚀 traceval: Trace-to-Eval Compiler
+
+<p align="center">
+  <img src="https://img.shields.io/badge/Python-3.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Version" />
+  <img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License" />
+  <img src="https://img.shields.io/badge/Coverage-87%25-green.svg" alt="Coverage" />
+  <a href="https://github.com/astral-sh/uv"><img src="https://img.shields.io/badge/dynamic/json?label=uv&query=%24.version&url=https%3A%2F%2Fraw.githubusercontent.com%2Fastral-sh%2Fuv%2Fmain%2Fcargo.toml&color=de5d43" alt="uv" /></a>
+</p>
+
+***"Your traces already know how your agent fails. traceval turns them into the test suite you never wrote."***
+
+Teams running LLM agents in production have observability traces, but only a fraction maintain robust evals. The raw material for great tests — thousands of real production traces, including edge cases and errors — sits unused because converting them into regression suites is manual and tedious.
+
+**traceval** automates this by ingesting agent traces from standard sources, normalizing them into a canonical Pydantic model, analyzing outcomes/clustering task signatures, and **compiling them into a human-editable eval suite**: pytest files + YAML datasets + judge rubric scaffolds.
+
+---
+
+## 🎨 Architectural Pipeline
+
+```mermaid
+graph LR
+    classDef source fill:#2c3e50,stroke:#34495e,stroke-width:2px,color:#fff;
+    classDef normalize fill:#16a085,stroke:#1abc9c,stroke-width:2px,color:#fff;
+    classDef analyze fill:#2980b9,stroke:#3498db,stroke-width:2px,color:#fff;
+    classDef compile fill:#8e44ad,stroke:#9b59b6,stroke-width:2px,color:#fff;
+    classDef run fill:#d35400,stroke:#e67e22,stroke-width:2px,color:#fff;
+
+    A[OTel / Langfuse / LangSmith] --> B(Canonical Trace DB)
+    B --> C(Outcome Labeler & Jaccard Clusterer)
+    C --> D(YAML cases + Pytest + Rubrics)
+    D --> E(HTTP / Callable Runner & Diff Reports)
+
+    class A source;
+    class B normalize;
+    class C analyze;
+    class D compile;
+    class E run;
+```
+
+---
+
+## ✨ Key Features
+
+* 🔌 **Zero-Configuration Ingest**: Direct compatibility with OpenTelemetry GenAI semantic conventions, Langfuse observations, LangSmith runs, or generic JSONL exports.
+* 🧠 **Smart Outcome Taxonomy**: Automatic categorization of trace outcomes (`success`, `tool_error`, `validation_error`, `loop`, `timeout`, `bad_output`) using rule-based heuristics that you can extend with Python modules.
+* 📊 **Embedding-Free Clustering**: Fast, local Jaccard-similarity shingle grouping that runs 100% offline, keeping your development cycle private and deterministic.
+* 📝 **Clean Code Generation**: Compiles cases into editable YAML files, LLM-as-a-judge rubrics into Markdown checklist scaffolds, and pytest test runs into clean templates.
+* ⚡ **PII Redaction Safeguards**: Automatically scrubs emails, credit cards, phone numbers, and API tokens before writing test inputs.
+* 🛡️ **CI/CD Regression Diff**: Compares execution summaries and scores between runs using exit codes to catch agent failures before deploying.
+
+---
+
+## ⏱️ 90-Second E2E Quickstart
+
+Experience `traceval` regression testing out of the box using our interactive demo script:
+
+```bash
+# Clone & run the demo
+chmod +x examples/demo.sh
+./examples/demo.sh
+```
+
+### Manual Walkthrough
+
+#### 1. Ingest Observability Logs
+```bash
+# Seed 200 synthetic telemetry traces containing successes and failure edge cases
+python3 examples/make_traces.py
+
+# Ingest into SQLite database
+traceval ingest examples/synthetic_traces.jsonl -o traces.db
+```
+
+#### 2. Label & Analyze Traffic Gaps
+```bash
+traceval analyze traces.db -o analysis/
+```
+*Outputs outcome statistics and generates `analysis/report.html` mapping traffic clusters:*
+```text
+Outcomes: success 60% · tool_error 15% · loop 10% · timeout 8% · validation_error 8%
+Clusters: 37 task clusters found.
+Top failure cluster: "500 refund stripe -> stripe_lookup -> (tool_error)" (30 traces)
+Report written to analysis/report.html
+```
+
+#### 3. Compile Cases and Pytest Harness
+```bash
+traceval generate traces.db -o evals/ --include-failures
+```
+*Generates test parameters `evals/cases/` and rubric Markdown checklists `evals/rubrics/`.*
+
+#### 4. Run Evaluations & Detect Regressions
+```bash
+# Run against the healthy agent (100% Pass)
+traceval run evals/ --target examples.demo_agent.agent:invoke_agent --judge fake
+
+# Run against the buggy agent (Detects regressions and exits with status 1)
+BUGGY=true traceval run evals/ --target examples.demo_agent.agent:invoke_agent --judge fake
+```
+
+---
+
+## 🛠️ CLI Command Reference
+
+> [!NOTE]
+> All CLI commands support `--json` to output machine-readable stdout for scripting.
+
+### Ingestion
+```bash
+traceval ingest <path> --format [auto|otel|langfuse|langsmith|generic] -o <traces.db>
+```
+*Ingests telemetry log dumps losslessly. Malformed spans write warnings to `<traces.db>.log`.*
+
+### Analysis
+```bash
+traceval analyze <traces.db> [--rules custom_rules.py] [--evals evals/] -o <analysis_dir/>
+```
+*Runs rule pipelines and Jaccard shingle similarity groupings.*
+
+### Generation
+```bash
+traceval generate <traces.db> -o <evals_dir/> [--per-cluster 3] [--include-failures] [--redact-hook module:fn]
+```
+*Creates regression cases, Markdown LLM-judge checklists, and conftest runners.*
+
+### Runner
+```bash
+traceval run <evals_dir/> --target <url|module:function> [--judge fake|openai] [--compare runs/prev.json]
+```
+*Executes tests, scores output constraints (`exact`, `contains`, `regex`, `json_schema`, `tool_sequence`, `judge`), and logs to project-level `runs/` directory.*
+
+---
+
+## 💡 Honest Limitations
+
+* **Side-Effect Free**: traceval assertions evaluate input/output matches. It does not attempt to replay side effects (e.g., updating database records) on mock tools.
+* **Text Telemetry**: The canonical model is optimized for text logs. Image or multimodal payloads in traces are logged as references.
+* **Static Visualization**: The coverage report is a portable, single-file HTML page. There is no hosted web service.

traceval-0.1.1/README.md

@@ -0,0 +1,138 @@
+# 🚀 traceval: Trace-to-Eval Compiler
+
+<p align="center">
+  <img src="https://img.shields.io/badge/Python-3.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Version" />
+  <img src="https://img.shields.io/badge/License-MIT-yellow.svg" alt="License" />
+  <img src="https://img.shields.io/badge/Coverage-87%25-green.svg" alt="Coverage" />
+  <a href="https://github.com/astral-sh/uv"><img src="https://img.shields.io/badge/dynamic/json?label=uv&query=%24.version&url=https%3A%2F%2Fraw.githubusercontent.com%2Fastral-sh%2Fuv%2Fmain%2Fcargo.toml&color=de5d43" alt="uv" /></a>
+</p>
+
+***"Your traces already know how your agent fails. traceval turns them into the test suite you never wrote."***
+
+Teams running LLM agents in production have observability traces, but only a fraction maintain robust evals. The raw material for great tests — thousands of real production traces, including edge cases and errors — sits unused because converting them into regression suites is manual and tedious.
+
+**traceval** automates this by ingesting agent traces from standard sources, normalizing them into a canonical Pydantic model, analyzing outcomes/clustering task signatures, and **compiling them into a human-editable eval suite**: pytest files + YAML datasets + judge rubric scaffolds.
+
+---
+
+## 🎨 Architectural Pipeline
+
+```mermaid
+graph LR
+    classDef source fill:#2c3e50,stroke:#34495e,stroke-width:2px,color:#fff;
+    classDef normalize fill:#16a085,stroke:#1abc9c,stroke-width:2px,color:#fff;
+    classDef analyze fill:#2980b9,stroke:#3498db,stroke-width:2px,color:#fff;
+    classDef compile fill:#8e44ad,stroke:#9b59b6,stroke-width:2px,color:#fff;
+    classDef run fill:#d35400,stroke:#e67e22,stroke-width:2px,color:#fff;
+
+    A[OTel / Langfuse / LangSmith] --> B(Canonical Trace DB)
+    B --> C(Outcome Labeler & Jaccard Clusterer)
+    C --> D(YAML cases + Pytest + Rubrics)
+    D --> E(HTTP / Callable Runner & Diff Reports)
+
+    class A source;
+    class B normalize;
+    class C analyze;
+    class D compile;
+    class E run;
+```
+
+---
+
+## ✨ Key Features
+
+* 🔌 **Zero-Configuration Ingest**: Direct compatibility with OpenTelemetry GenAI semantic conventions, Langfuse observations, LangSmith runs, or generic JSONL exports.
+* 🧠 **Smart Outcome Taxonomy**: Automatic categorization of trace outcomes (`success`, `tool_error`, `validation_error`, `loop`, `timeout`, `bad_output`) using rule-based heuristics that you can extend with Python modules.
+* 📊 **Embedding-Free Clustering**: Fast, local Jaccard-similarity shingle grouping that runs 100% offline, keeping your development cycle private and deterministic.
+* 📝 **Clean Code Generation**: Compiles cases into editable YAML files, LLM-as-a-judge rubrics into Markdown checklist scaffolds, and pytest test runs into clean templates.
+* ⚡ **PII Redaction Safeguards**: Automatically scrubs emails, credit cards, phone numbers, and API tokens before writing test inputs.
+* 🛡️ **CI/CD Regression Diff**: Compares execution summaries and scores between runs using exit codes to catch agent failures before deploying.
+
+---
+
+## ⏱️ 90-Second E2E Quickstart
+
+Experience `traceval` regression testing out of the box using our interactive demo script:
+
+```bash
+# Clone & run the demo
+chmod +x examples/demo.sh
+./examples/demo.sh
+```
+
+### Manual Walkthrough
+
+#### 1. Ingest Observability Logs
+```bash
+# Seed 200 synthetic telemetry traces containing successes and failure edge cases
+python3 examples/make_traces.py
+
+# Ingest into SQLite database
+traceval ingest examples/synthetic_traces.jsonl -o traces.db
+```
+
+#### 2. Label & Analyze Traffic Gaps
+```bash
+traceval analyze traces.db -o analysis/
+```
+*Outputs outcome statistics and generates `analysis/report.html` mapping traffic clusters:*
+```text
+Outcomes: success 60% · tool_error 15% · loop 10% · timeout 8% · validation_error 8%
+Clusters: 37 task clusters found.
+Top failure cluster: "500 refund stripe -> stripe_lookup -> (tool_error)" (30 traces)
+Report written to analysis/report.html
+```
+
+#### 3. Compile Cases and Pytest Harness
+```bash
+traceval generate traces.db -o evals/ --include-failures
+```
+*Generates test parameters `evals/cases/` and rubric Markdown checklists `evals/rubrics/`.*
+
+#### 4. Run Evaluations & Detect Regressions
+```bash
+# Run against the healthy agent (100% Pass)
+traceval run evals/ --target examples.demo_agent.agent:invoke_agent --judge fake
+
+# Run against the buggy agent (Detects regressions and exits with status 1)
+BUGGY=true traceval run evals/ --target examples.demo_agent.agent:invoke_agent --judge fake
+```
+
+---
+
+## 🛠️ CLI Command Reference
+
+> [!NOTE]
+> All CLI commands support `--json` to output machine-readable stdout for scripting.
+
+### Ingestion
+```bash
+traceval ingest <path> --format [auto|otel|langfuse|langsmith|generic] -o <traces.db>
+```
+*Ingests telemetry log dumps losslessly. Malformed spans write warnings to `<traces.db>.log`.*
+
+### Analysis
+```bash
+traceval analyze <traces.db> [--rules custom_rules.py] [--evals evals/] -o <analysis_dir/>
+```
+*Runs rule pipelines and Jaccard shingle similarity groupings.*
+
+### Generation
+```bash
+traceval generate <traces.db> -o <evals_dir/> [--per-cluster 3] [--include-failures] [--redact-hook module:fn]
+```
+*Creates regression cases, Markdown LLM-judge checklists, and conftest runners.*
+
+### Runner
+```bash
+traceval run <evals_dir/> --target <url|module:function> [--judge fake|openai] [--compare runs/prev.json]
+```
+*Executes tests, scores output constraints (`exact`, `contains`, `regex`, `json_schema`, `tool_sequence`, `judge`), and logs to project-level `runs/` directory.*
+
+---
+
+## 💡 Honest Limitations
+
+* **Side-Effect Free**: traceval assertions evaluate input/output matches. It does not attempt to replay side effects (e.g., updating database records) on mock tools.
+* **Text Telemetry**: The canonical model is optimized for text logs. Image or multimodal payloads in traces are logged as references.
+* **Static Visualization**: The coverage report is a portable, single-file HTML page. There is no hosted web service.

traceval-0.1.1/docs/formats.md

@@ -0,0 +1,126 @@
+# Trace Ingestion Formats & Mappings
+
+This document describes how raw trace records from different observability backends are mapped onto the `traceval` canonical `Trace` Pydantic model.
+
+## Canonical Trace Model
+
+All trace adapters transform incoming logs into the `Trace` schema (`src/traceval/model.py`), which includes:
+- `trace_id` (str)
+- `source` (str: "otel", "langfuse", "langsmith", "generic")
+- `started_at` (datetime)
+- `ended_at` (datetime or None)
+- `task_input` (str, user prompt triggering the trace)
+- `final_output` (str or None, assistant's final response)
+- `steps` (list of Step objects chronologically ordered):
+  - `index` (int)
+  - `kind` ("llm", "tool", "retrieval", "other")
+  - `llm` (LLMCall or None)
+  - `tool` (ToolCall or None)
+  - `raw_attributes` (dict[str, str], lossless metadata escape hatch)
+
+---
+
+## 1. Generic format (`generic`)
+
+A line-by-line JSONL file where each line is a raw JSON string validating directly against our canonical `Trace` model.
+
+### Assumptions & Heuristics
+- Direct structural validation.
+- Lines failing to parse are logged as warnings and skipped.
+
+---
+
+## 2. OpenTelemetry GenAI Conventions (`otel`)
+
+OTel traces are ingested from flat lists of JSON span logs (e.g. OTLP export format), grouped by `trace_id`.
+
+### Span Categorization Rules
+- **Root Span**: Identified by `parent_span_id` being `None` or empty.
+- **LLM Call**: Spans containing GenAI semantic convention keys in their attributes:
+  - `gen_ai.system`
+  - `gen_ai.prompt`
+  - `gen_ai.completion`
+- **Tool Call**: Spans containing:
+  - `gen_ai.tool.name`
+  - `gen_ai.tool.arguments`
+  - Or span name matching `order_lookup`, `stripe_lookup`, or `kb_search`.
+- **Other**: All other spans are categorized as `other`.
+
+### Attribute Translations
+
+| Canonical Field | OTel Span Path |
+| --- | --- |
+| `started_at` | Root span `start_time` (ISO datetime) |
+| `ended_at` | Root span `end_time` (ISO datetime) |
+| `task_input` | Root span `attributes["gen_ai.task_input"]` |
+| `final_output` | Root span `attributes["gen_ai.final_output"]` |
+| `llm.model` | Span `attributes["gen_ai.request.model"]` |
+| `llm.input_messages` | Parsed JSON array from `attributes["gen_ai.prompt"]` |
+| `llm.output_message` | Assistant role with content from `attributes["gen_ai.completion"]` |
+| `llm.prompt_tokens` | `attributes["gen_ai.usage.prompt_tokens"]` |
+| `llm.completion_tokens` | `attributes["gen_ai.usage.completion_tokens"]` |
+| `llm.error` | `attributes["gen_ai.error"]` |
+| `tool.name` | `attributes["gen_ai.tool.name"]` or Span `name` |
+| `tool.arguments_json` | `attributes["gen_ai.tool.arguments"]` |
+| `tool.output` | `attributes["gen_ai.tool.output"]` |
+| `tool.error` | `attributes["gen_ai.tool.error"]` or `attributes["gen_ai.error"]` |
+
+---
+
+## 3. Langfuse Export (`langfuse`)
+
+Langfuse exports traces as JSON objects with nested lists of observations (of types `GENERATION`, `SPAN`, `EVENT`).
+
+### Mapping Translations
+
+| Canonical Field | Langfuse Path |
+| --- | --- |
+| `trace_id` | Trace `id` |
+| `started_at` | Trace `timestamp` |
+| `task_input` | Trace `input` |
+| `final_output` | Trace `output` |
+| `metadata` | Trace `metadata` |
+
+### Observation Mapping
+- **GENERATION** $\rightarrow$ `LLMCall`:
+  - `llm.model` $\leftarrow$ Observation `model`
+  - `llm.input_messages` $\leftarrow$ Observation `input` (parsed list of message objects)
+  - `llm.output_message` $\leftarrow$ Observation `output`
+  - `llm.prompt_tokens` $\leftarrow$ `usage.promptTokens`
+  - `llm.completion_tokens` $\leftarrow$ `usage.completionTokens`
+  - `llm.error` $\leftarrow$ `statusMessage` when `level == "ERROR"`
+- **SPAN** $\rightarrow$ `ToolCall` (if name is order/stripe/kb lookup or `metadata.tool` matches):
+  - `tool.name` $\leftarrow$ Observation `name`
+  - `tool.arguments_json` $\leftarrow$ Observation `input` (serialized to JSON)
+  - `tool.output` $\leftarrow$ Observation `output` (stringified)
+  - `tool.error` $\leftarrow$ `statusMessage` when `level == "ERROR"`
+
+---
+
+## 4. LangSmith Run Export (`langsmith`)
+
+LangSmith exports represent hierarchical run hierarchies grouped by `trace_id`.
+
+### Mapping Translations
+
+| Canonical Field | LangSmith Path |
+| --- | --- |
+| `trace_id` | Run `trace_id` or root run `id` |
+| `started_at` | Root run `start_time` |
+| `ended_at` | Root run `end_time` |
+| `task_input` | Root run `inputs["input"]` or first key's value |
+| `final_output` | Root run `outputs["output"]` or first key's value |
+
+### Child Run Classification
+- **run_type == "llm"** $\rightarrow$ `LLMCall`:
+  - `llm.model` $\leftarrow$ `extra.metadata.ls_model_name`
+  - `llm.input_messages` $\leftarrow$ `inputs.messages` (list mapping)
+  - `llm.output_message` $\leftarrow$ First element of `outputs.generations`
+  - `llm.prompt_tokens` $\leftarrow$ `extra.token_usage.prompt_tokens`
+  - `llm.completion_tokens` $\leftarrow$ `extra.token_usage.completion_tokens`
+  - `llm.error` $\leftarrow$ Run `error` field
+- **run_type == "tool"** $\rightarrow$ `ToolCall`:
+  - `tool.name` $\leftarrow$ Run `name`
+  - `tool.arguments_json` $\leftarrow$ Serialized run `inputs` dict
+  - `tool.output` $\leftarrow$ Run `outputs["output"]` or serialized outputs
+  - `tool.error` $\leftarrow$ Run `error` field

traceval-0.1.1/examples/demo.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# traceval e2e quickstart demo script
+set -e
+
+echo "=== 1. Generate synthetic traces ==="
+python3 examples/make_traces.py
+
+echo -e "\n=== 2. Ingest traces into SQLite ==="
+rm -f demo_traces.db
+uv run python3 src/traceval/cli.py ingest examples/synthetic_traces.jsonl -o demo_traces.db
+
+echo -e "\n=== 3. Analyze traces (labeling, clustering, outcomes) ==="
+uv run python3 src/traceval/cli.py analyze demo_traces.db -o demo_analysis/
+
+echo -e "\n=== 4. Generate eval suite ==="
+rm -rf demo_evals/
+uv run python3 src/traceval/cli.py generate demo_traces.db -o demo_evals/ --include-failures
+
+echo -e "\n=== 5. Run evals against healthy demo agent ==="
+# We add --with fastapi --with uvicorn --with pytest so that all run dependencies are active
+HEALTHY_REPORT=$(uv run --with fastapi --with uvicorn --with pytest python3 src/traceval/cli.py run demo_evals/ --target examples.demo_agent.agent:invoke_agent --judge fake | grep -o 'demo_evals/runs/run_.*\.json' | head -n 1) || true
+
+echo -e "\n=== 6. Run evals against buggy demo agent and compare ==="
+if [ -n "$HEALTHY_REPORT" ]; then
+  # Should report regressions and exit with failure code 1
+  BUGGY=true uv run --with fastapi --with uvicorn --with pytest python3 src/traceval/cli.py run demo_evals/ --target examples.demo_agent.agent:invoke_agent --judge fake --compare "$HEALTHY_REPORT" || echo "✅ E2E regression check passed: traceval correctly detected regressions and exited with failure status!"
+else
+  # Fallback if grep failed to capture path
+  BUGGY=true uv run --with fastapi --with uvicorn --with pytest python3 src/traceval/cli.py run demo_evals/ --target examples.demo_agent.agent:invoke_agent --judge fake || echo "✅ E2E regression check passed: traceval correctly detected regressions!"
+fi