llm-eval-harness 0.1.0__tar.gz

llm_eval_harness-0.1.0/.env.example

@@ -0,0 +1 @@
+OPENRIXER_API_KEY=sk-or-replace-me

llm_eval_harness-0.1.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Report a bug in eval-harness
+title: '[BUG] '
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce:
+1. Run `eval-harness run ...`
+2. See error
+
+**Eval Run ID** (if applicable)
+<!-- Run ID from the database or output -->
+
+**Environment**
+- eval-harness version:
+- Python version:
+- OS:
+
+**Additional context**
+Add any other context here.

llm_eval_harness-0.1.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest a new feature
+title: '[FEAT] '
+labels: enhancement
+assignees: ''
+
+---
+
+**Problem**
+What problem are you trying to solve?
+
+**Solution**
+What do you want to happen?
+
+**Alternatives**
+Any alternative solutions you've considered?
+
+**Additional context**
+Add any other context here.

llm_eval_harness-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[dev]"
+      - name: Lint
+        run: |
+          ruff check src tests
+          ruff format --check src tests
+      - name: Test
+        run: |
+          pytest tests/ -v --cov=src --cov-report=term-missing

llm_eval_harness-0.1.0/.gitignore

@@ -0,0 +1,21 @@
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+.env
+.coverage
+coverage.xml
+htmlcov/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+*.db
+*.db-journal
+*.db-wal
+*.db-shm
+.eval-harness/
+/push.sh

llm_eval_harness-0.1.0/AGENTS.md

@@ -0,0 +1,109 @@
+# AGENTS.md — Eval Harness Project
+
+## Project
+Python CLI tool that evaluates LLM outputs from production logs against a dual-dimension rubric (faithfulness + task completion).
+
+## Tech Stack
+- Python 3.11+
+- hatchling build backend (pyproject.toml)
+- Typer + Rich (CLI + terminal output)
+- Pydantic v2 (data validation)
+- sqlite3 stdlib (sync, WAL mode)
+- httpx (async HTTP for judge API calls)
+- tiktoken (token estimation)
+- pytest + pytest-httpx + ruff
+
+## Project Structure
+```
+eval-harness/
+├── src/
+│   ├── __init__.py
+│   ├── cli.py              # Typer app: run, judges, report, export, cache
+│   ├── models.py           # Pydantic: EvalRecord, EvalResult, EvalRun, JudgeCacheEntry, RubricTemplate
+│   ├── db.py               # SQLite: schema versioning, CRUD, migrations, export
+│   ├── ingest.py           # JSONL + CSV + stdin parser: --sample, --since, lenient parsing
+│   ├── evaluator.py        # LLM-as-judge: async batch, round-robin fallback, response caching
+│   ├── reporter.py         # Rich terminal tables, ASCII histogram, JSON/CSV export
+│   └── judges.py           # OpenRouter free model fetcher/cache
+├── tests/
+│   ├── conftest.py         # Fixtures, mock judge responses, temp DB setup
+│   ├── test_models.py
+│   ├── test_db.py
+│   ├── test_ingest.py      # VCR cassettes for mock LLM responses
+│   ├── test_evaluator.py   # pytest-httpx for mock HTTP
+│   ├── test_reporter.py
+│   ├── test_cli.py         # Typer CliRunner, end-to-end
+│   └── test_judges.py
+├── .github/workflows/ci.yml
+├── pyproject.toml
+├── README.md
+├── .gitignore
+├── .env.example
+├── LICENSE (MIT)
+└── CHANGELOG.md
+```
+
+## Database Schema (v1)
+```sql
+CREATE TABLE schema_version (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL);
+
+CREATE TABLE eval_runs (
+    run_id TEXT PRIMARY KEY, created_at TEXT NOT NULL, config_json TEXT NOT NULL,
+    record_count INTEGER DEFAULT 0, rubric_id TEXT DEFAULT 'faithfulness-v1',
+    judge_model TEXT, status TEXT DEFAULT 'running', completed_at TEXT,
+    mean_score REAL, pass_rate REAL, eval_time_seconds REAL
+);
+
+CREATE TABLE eval_records (
+    record_id TEXT PRIMARY KEY, run_id TEXT NOT NULL REFERENCES eval_runs(run_id),
+    input_text TEXT NOT NULL, output_text TEXT NOT NULL, reference_text TEXT,
+    source_file TEXT, metadata_json TEXT, created_at TEXT NOT NULL
+);
+CREATE INDEX idx_records_run ON eval_records(run_id);
+
+CREATE TABLE eval_results (
+    result_id TEXT PRIMARY KEY, record_id TEXT NOT NULL REFERENCES eval_records(record_id),
+    run_id TEXT NOT NULL REFERENCES eval_runs(run_id), rubric_id TEXT DEFAULT 'faithfulness-v1',
+    rubric_version TEXT DEFAULT '1.0', faithfulness REAL NOT NULL, task_completion REAL NOT NULL,
+    combined_score REAL NOT NULL, pass_fail TEXT NOT NULL, reasoning TEXT DEFAULT '',
+    faithfulness_reasoning TEXT DEFAULT '', task_completion_reasoning TEXT DEFAULT '',
+    judge_model TEXT NOT NULL, judge_fallbacks INTEGER DEFAULT 0, judge_tried TEXT DEFAULT '[]',
+    tokens_estimated INTEGER, evaluated_at TEXT NOT NULL, error TEXT
+);
+CREATE INDEX idx_results_run ON eval_results(run_id);
+
+CREATE TABLE judge_cache (
+    cache_key TEXT PRIMARY KEY, model_id TEXT NOT NULL, rubric_version TEXT NOT NULL,
+    response_json TEXT NOT NULL, created_at TEXT NOT NULL, hits INTEGER DEFAULT 1
+);
+```
+
+## Input Schema (JSONL)
+```json
+{"input": "user prompt", "output": "model response", "reference": "optional ground truth"}
+```
+
+## Judge Output Schema
+```json
+{"faithfulness": 0.0-1.0, "task_completion": 0.0-1.0, "reasoning": "str", "faithfulness_reasoning": "str", "task_completion_reasoning": "str"}
+```
+Combined: 0.5 * faithfulness + 0.5 * task_completion. Pass/fail threshold: 0.7.
+
+## CLI Commands
+- `eval-harness run <file>` — primary: ingest + evaluate + report. Flags: --format jsonl|csv, --input-col, --output-col, --reference-col, --sample N, --since DATE, --limit N, --judge MODEL, --no-fallback, --max-fallbacks N, --pass-threshold FLOAT, --output json|table, --output-file PATH, --dry-run, --resume, --timeout SECONDS, --rpm-limit INT, --yes, --verbose, --quiet, --config PATH
+- `eval-harness judges` — list free judge models. Flags: --refresh, --json
+- `eval-harness report --run-id UUID` — show results. Flags: --output json|table|csv, --output-file PATH
+- `eval-harness export --run-id UUID --format json|csv --output-file PATH`
+- `eval-harness cache [--clear] [--stats]`
+
+Exit codes: 0=all pass, 1=any failures, 2=evaluator error
+
+## API Key
+Read from OPENRIXER_API_KEY env var (not hardcoded).
+
+## Rules
+- TDD: write failing test BEFORE implementation
+- Type hints everywhere
+- Google-style docstrings
+- ruff check + ruff format before every commit
+- Commit after every task

llm_eval_harness-0.1.0/CHANGELOG.md

@@ -0,0 +1,17 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2026-05-29
+
+### Added
+- Typer CLI: `run`, `judges`, `report`, `export`, `cache`
+- Pydantic v2 models: `EvalRecord`, `EvalResult`, `EvalRun`, `JudgeCacheEntry`, `RubricTemplate`, `EvalSummary`
+- SQLite persistence layer with WAL mode and schema migrations
+- JSONL/CSV/stdin ingestion with sampling, since-date filtering, and lenient parsing
+- Async LLM-as-judge evaluator with round-robin fallback and response caching
+- Rich-based reporter with summary table, ASCII histogram, and JSON/CSV export
+- OpenRouter free-model fetcher and on-disk cache

llm_eval_harness-0.1.0/GENERATION_PROMPT.md

@@ -0,0 +1,34 @@
+Generate the complete eval-harness Python CLI project. Follow the AGENTS.md spec in the current directory exactly.
+
+Create ALL files listed in the project structure. Use TDD: write each test file before its corresponding source file.
+
+Files to create:
+1. pyproject.toml (hatchling build, all deps, ruff config, test config)
+2. .github/workflows/ci.yml (ruff + pytest)
+3. README.md (install, quickstart, CI/CD example)
+4. .gitignore
+5. .env.example
+6. LICENSE (MIT)
+7. CHANGELOG.md
+8. src/__init__.py
+9. src/models.py (Pydantic v2: EvalRecord, EvalResult, EvalRun, JudgeCacheEntry, RubricTemplate, BUILTIN_RUBRIC_V1, PassFail/RunStatus enums, EvalSummary)
+10. src/db.py (SQLite WAL mode, schema versioning, full CRUD for all 5 tables, export JSON/CSV)
+11. src/ingest.py (JSONL + CSV + stdin parser: --sample N, --since DATE, lenient parsing, no auto-detect)
+12. src/evaluator.py (async batch eval, round-robin fallback, sqlite response cache, token tracking, markdown-wrapped JSON handling)
+13. src/reporter.py (Rich terminal tables, ASCII histogram, JSON/CSV export, judge usage stats)
+14. src/cli.py (Typer app: run/judges/report/export/cache commands, all flags from AGENTS.md, exit codes 0/1/2, --dry-run, --resume, progress bar for batches > 10)
+15. src/judges.py (OpenRouter model fetch/cache, rank by context length, --refresh, --json)
+16. tests/conftest.py (temp DB per test, sample records, mock judge responses)
+17. tests/test_models.py, test_db.py, test_ingest.py, test_evaluator.py, test_reporter.py, test_cli.py, test_judges.py
+
+Requirements:
+- Python 3.11+, hatchling build backend
+- Dependencies: typer>=0.12, pydantic>=2.0, rich>=13.0, httpx>=0.27, tiktoken>=0.7
+- Dev deps: pytest>=8, pytest-httpx>=0.30, pytest-cov>=5, ruff>=0.5
+- OPENRIXER_API_KEY env var (never hardcoded)
+- TDD for every module: test file before source file, each public function tested
+- Type hints everywhere (mypy-compatible)
+- Google-style docstrings for all public functions
+- The project must work: pip install -e . && eval-harness --help
+- Database: sqlite3 stdlib (NOT aiosqlite), WAL mode, foreign keys, schema migration system
+- Judge API: OpenRouter (openrouter.ai/api/v1/chat/completions), round-robin fallback through cached free models

llm_eval_harness-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Eval Harness Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llm_eval_harness-0.1.0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: llm-eval-harness
+Version: 0.1.0
+Summary: CLI tool that evaluates LLM outputs from production logs against a dual-dimension rubric.
+Project-URL: Homepage, https://github.com/onicarps/eval-harness
+Project-URL: Repository, https://github.com/onicarps/eval-harness
+Project-URL: Issues, https://github.com/onicarps/eval-harness/issues
+Author: llm-eval-harness
+License: MIT
+License-File: LICENSE
+Keywords: cli,evaluation,faithfulness,llm,rubric
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Requires-Dist: tiktoken>=0.7
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest-cov>=5; extra == 'dev'
+Requires-Dist: pytest-httpx>=0.30; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# eval-harness
+
+A Python CLI that evaluates LLM outputs from production logs against a
+dual-dimension rubric (faithfulness + task completion).
+
+## Install
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Quickstart
+
+```bash
+export OPENRIXER_API_KEY=sk-or-...
+eval-harness run path/to/logs.jsonl --judge meta-llama/llama-3.1-8b-instruct:free
+```
+
+Input JSONL schema:
+
+```json
+{"input": "user prompt", "output": "model response", "reference": "optional ground truth"}
+```
+
+## Commands
+
+- `eval-harness run <file>` — ingest, evaluate, and report
+- `eval-harness judges` — list free judge models (cached in `~/.eval-harness/judges.json`)
+- `eval-harness report --run-id UUID` — show a stored run
+- `eval-harness export --run-id UUID --format json|csv --output-file PATH`
+- `eval-harness cache [--stats] [--clear]`
+
+Exit codes: `0` all pass, `1` any failures, `2` evaluator error.
+
+## CI/CD example
+
+```yaml
+- run: pip install eval-harness
+- run: OPENRIXER_API_KEY=${{ secrets.OPENRIXER_API_KEY }} eval-harness run eval/cases.jsonl --pass-threshold 0.7
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+ruff check src tests && ruff format --check src tests
+pytest tests/ -v --cov=src
+```

llm_eval_harness-0.1.0/README.md

@@ -0,0 +1,48 @@
+# eval-harness
+
+A Python CLI that evaluates LLM outputs from production logs against a
+dual-dimension rubric (faithfulness + task completion).
+
+## Install
+
+```bash
+pip install -e ".[dev]"
+```
+
+## Quickstart
+
+```bash
+export OPENRIXER_API_KEY=sk-or-...
+eval-harness run path/to/logs.jsonl --judge meta-llama/llama-3.1-8b-instruct:free
+```
+
+Input JSONL schema:
+
+```json
+{"input": "user prompt", "output": "model response", "reference": "optional ground truth"}
+```
+
+## Commands
+
+- `eval-harness run <file>` — ingest, evaluate, and report
+- `eval-harness judges` — list free judge models (cached in `~/.eval-harness/judges.json`)
+- `eval-harness report --run-id UUID` — show a stored run
+- `eval-harness export --run-id UUID --format json|csv --output-file PATH`
+- `eval-harness cache [--stats] [--clear]`
+
+Exit codes: `0` all pass, `1` any failures, `2` evaluator error.
+
+## CI/CD example
+
+```yaml
+- run: pip install eval-harness
+- run: OPENRIXER_API_KEY=${{ secrets.OPENRIXER_API_KEY }} eval-harness run eval/cases.jsonl --pass-threshold 0.7
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+ruff check src tests && ruff format --check src tests
+pytest tests/ -v --cov=src
+```

llm_eval_harness-0.1.0/pyproject.toml

@@ -0,0 +1,50 @@
+[build-system]
+requires = [ "hatchling",]
+build-backend = "hatchling.build"
+
+[project]
+name = "llm-eval-harness"
+version = "0.1.0"
+description = "CLI tool that evaluates LLM outputs from production logs against a dual-dimension rubric."
+readme = "README.md"
+requires-python = ">=3.11"
+keywords = [ "llm", "evaluation", "rubric", "cli", "faithfulness",]
+classifiers = [ "Development Status :: 4 - Beta", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Topic :: Software Development :: Quality Assurance",]
+dependencies = [ "typer>=0.12", "pydantic>=2.0", "rich>=13.0", "httpx>=0.27", "tiktoken>=0.7",]
+[[project.authors]]
+name = "llm-eval-harness"
+
+[project.license]
+text = "MIT"
+
+[project.optional-dependencies]
+dev = [ "pytest>=8", "pytest-httpx>=0.30", "pytest-cov>=5", "pytest-asyncio>=0.23", "ruff>=0.5",]
+
+[project.scripts]
+eval-harness = "src.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/onicarps/eval-harness"
+Repository = "https://github.com/onicarps/eval-harness"
+Issues = "https://github.com/onicarps/eval-harness/issues"
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = [ "E", "F", "I", "B", "UP", "W",]
+ignore = [ "E501", "B008", "B017", "UP045", "UP007",]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.pytest.ini_options]
+testpaths = [ "tests",]
+addopts = "-ra"
+asyncio_mode = "auto"
+filterwarnings = [ "ignore::DeprecationWarning",]
+
+[tool.hatch.build.targets.wheel]
+packages = [ "src",]

llm_eval_harness-0.1.0/pyproject.toml.bak

@@ -0,0 +1,66 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "eval-harness"
+version = "0.1.0"
+description = "CLI tool that evaluates LLM outputs from production logs against a dual-dimension rubric."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "Eval Harness Contributors" }]
+keywords = ["llm", "evaluation", "rubric", "cli", "faithfulness"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Quality Assurance",
+]
+dependencies = [
+    "typer>=0.12",
+    "pydantic>=2.0",
+    "rich>=13.0",
+    "httpx>=0.27",
+    "tiktoken>=0.7",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8",
+    "pytest-httpx>=0.30",
+    "pytest-cov>=5",
+    "pytest-asyncio>=0.23",
+    "ruff>=0.5",
+]
+
+[project.scripts]
+eval-harness = "src.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/onicarps/eval-harness"
+Repository = "https://github.com/onicarps/eval-harness"
+Issues = "https://github.com/onicarps/eval-harness/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "B", "UP", "W"]
+ignore = ["E501", "B008", "B017", "UP045", "UP007"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-ra"
+asyncio_mode = "auto"
+filterwarnings = ["ignore::DeprecationWarning"]

llm_eval_harness-0.1.0/run_droid.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+cd /home/oni/.hermes/profiles/eval-harness/workspace/eval-harness
+
+# Load env
+eval $(grep -v '^#' /home/oni/.hermes/profiles/eval-harness/.env | grep -v '^$' | sed 's/^/export /')
+
+echo "FACTORY_API_KEY set: ${FACTORY_API_KEY:0:8}..."
+
+# Write prompt to temp file
+cat > /tmp/droid_prompt.txt << 'PROMPT'
+Read the AGENTS.md and GENERATION_PROMPT.md files in the current directory. Then generate the complete eval-harness Python CLI project as specified. Create ALL files listed in the project structure. Use TDD: write each test file before its corresponding source file. Start with pyproject.toml, then src/__init__.py, src/models.py, tests/test_models.py, src/db.py, tests/test_db.py, src/ingest.py, tests/test_ingest.py, src/evaluator.py, tests/test_evaluator.py, src/reporter.py, tests/test_reporter.py, src/cli.py, tests/test_cli.py, src/judges.py, tests/test_judges.py, .github/workflows/ci.yml, README.md, .gitignore, .env.example, LICENSE, CHANGELOG.md. Make sure pip install -e . works and eval-harness --help shows all commands.
+PROMPT
+
+echo "Starting droid..."
+droid exec --auto high -f /tmp/droid_prompt.txt

llm_eval_harness-0.1.0/src/__init__.py

@@ -0,0 +1,3 @@
+"""Eval Harness: CLI tool to evaluate LLM outputs against a dual-dimension rubric."""
+
+__version__ = "0.1.0"