rag-eval-harness 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. rag_eval_harness-0.2.0/.gitignore +24 -0
  2. rag_eval_harness-0.2.0/LICENSE +21 -0
  3. rag_eval_harness-0.2.0/PKG-INFO +161 -0
  4. rag_eval_harness-0.2.0/README.md +121 -0
  5. rag_eval_harness-0.2.0/pyproject.toml +89 -0
  6. rag_eval_harness-0.2.0/src/rag_eval_harness/__init__.py +12 -0
  7. rag_eval_harness-0.2.0/src/rag_eval_harness/adapters/__init__.py +6 -0
  8. rag_eval_harness-0.2.0/src/rag_eval_harness/adapters/http.py +92 -0
  9. rag_eval_harness-0.2.0/src/rag_eval_harness/adapters/traces.py +29 -0
  10. rag_eval_harness-0.2.0/src/rag_eval_harness/agent/__init__.py +5 -0
  11. rag_eval_harness-0.2.0/src/rag_eval_harness/agent/agents.py +180 -0
  12. rag_eval_harness-0.2.0/src/rag_eval_harness/agent/graph.py +171 -0
  13. rag_eval_harness-0.2.0/src/rag_eval_harness/agent/supervisor.py +168 -0
  14. rag_eval_harness-0.2.0/src/rag_eval_harness/agent/trace.py +71 -0
  15. rag_eval_harness-0.2.0/src/rag_eval_harness/agent/triage.py +130 -0
  16. rag_eval_harness-0.2.0/src/rag_eval_harness/api/__init__.py +3 -0
  17. rag_eval_harness-0.2.0/src/rag_eval_harness/api/app.py +92 -0
  18. rag_eval_harness-0.2.0/src/rag_eval_harness/api/auth.py +57 -0
  19. rag_eval_harness-0.2.0/src/rag_eval_harness/api/routes.py +251 -0
  20. rag_eval_harness-0.2.0/src/rag_eval_harness/api/schemas.py +63 -0
  21. rag_eval_harness-0.2.0/src/rag_eval_harness/cli.py +439 -0
  22. rag_eval_harness-0.2.0/src/rag_eval_harness/config.py +44 -0
  23. rag_eval_harness-0.2.0/src/rag_eval_harness/engine.py +154 -0
  24. rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/__init__.py +20 -0
  25. rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/lexical.py +109 -0
  26. rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/protocol.py +12 -0
  27. rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/ragas.py +177 -0
  28. rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/stub.py +66 -0
  29. rag_eval_harness-0.2.0/src/rag_eval_harness/io.py +147 -0
  30. rag_eval_harness-0.2.0/src/rag_eval_harness/py.typed +0 -0
  31. rag_eval_harness-0.2.0/src/rag_eval_harness/redact.py +35 -0
  32. rag_eval_harness-0.2.0/src/rag_eval_harness/regression/__init__.py +19 -0
  33. rag_eval_harness-0.2.0/src/rag_eval_harness/regression/compare.py +237 -0
  34. rag_eval_harness-0.2.0/src/rag_eval_harness/store/__init__.py +5 -0
  35. rag_eval_harness-0.2.0/src/rag_eval_harness/store/models.py +79 -0
  36. rag_eval_harness-0.2.0/src/rag_eval_harness/store/repo.py +272 -0
  37. rag_eval_harness-0.2.0/src/rag_eval_harness/types.py +124 -0
@@ -0,0 +1,24 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .eggs/
6
+ dist/
7
+ build/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .mypy_cache/
11
+ .coverage
12
+ htmlcov/
13
+ *.db
14
+ *.sqlite
15
+ *.sqlite3
16
+ rag-eval-results.json
17
+ .env
18
+ .DS_Store
19
+ node_modules/
20
+ web/dist/
21
+ web/.vite/
22
+ data/
23
+ .idea/
24
+ .vscode/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 John Carl Santos
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,161 @@
1
+ Metadata-Version: 2.5
2
+ Name: rag-eval-harness
3
+ Version: 0.2.0
4
+ Summary: RAGAS with a memory and a diff view. Evaluate RAG pipelines, persist runs, and fail CI on regressions.
5
+ Author-email: John Carl Santos <johncarlsantos30@gmail.com>
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Keywords: ci,evaluation,lexical,llm,rag,ragas,regression
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Environment :: Console
11
+ Classifier: Framework :: FastAPI
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Requires-Python: >=3.11
20
+ Requires-Dist: fastapi>=0.115
21
+ Requires-Dist: httpx>=0.28
22
+ Requires-Dist: pydantic-settings>=2.6
23
+ Requires-Dist: pydantic>=2.10
24
+ Requires-Dist: python-dotenv>=1.0
25
+ Requires-Dist: python-multipart>=0.0.18
26
+ Requires-Dist: sqlalchemy>=2.0.36
27
+ Requires-Dist: typer>=0.15
28
+ Requires-Dist: uvicorn[standard]>=0.32
29
+ Provides-Extra: agent
30
+ Requires-Dist: langgraph>=1.2.2; extra == 'agent'
31
+ Requires-Dist: pydantic-ai-slim[anthropic,openai]>=2.51.0; extra == 'agent'
32
+ Provides-Extra: postgres
33
+ Requires-Dist: psycopg[binary]>=3.2; extra == 'postgres'
34
+ Provides-Extra: ragas
35
+ Requires-Dist: openai>=1.40; extra == 'ragas'
36
+ Requires-Dist: ragas>=0.2; extra == 'ragas'
37
+ Provides-Extra: trace
38
+ Requires-Dist: logfire>=5.1.1; extra == 'trace'
39
+ Description-Content-Type: text/markdown
40
+
41
+ # rag-eval-harness
42
+
43
+ [![CI](https://github.com/JohnCarl-30/rag-eval-harness/actions/workflows/ci.yml/badge.svg)](https://github.com/JohnCarl-30/rag-eval-harness/actions/workflows/ci.yml)
44
+
45
+ RAGAS with a memory and a diff view.
46
+
47
+ **What this proved:** a weaker retriever on a 40-question Nimbus help-center set cut mean **context recall from 0.880 to 0.740**. `rag-eval regress --threshold 0.05` exited **1**. Full write-up: [Nimbus case study](docs/nimbus-case-study.md). 90-second recording: `./scripts/demo.sh`.
48
+
49
+ Chunking the same corpus doubled lexical precision and failed the recall gate. Production stayed on whole articles. [Chunking case study](docs/nimbus-chunking.md). `./scripts/demo-chunking.sh`.
50
+
51
+ Upload a golden RAG dataset, run a pipeline (precomputed traces or HTTP), score with **RAGAS**, **lexical** overlap, or a keyless **stub** evaluator, persist runs, and fail CI when mean metrics drop versus a tagged baseline.
52
+
53
+ This is not a RAG framework. There is no ingestion or retrieval stack. Evaluation orchestration is the product.
54
+
55
+ ## 10-minute path
56
+
57
+ Requires Python 3.11+ ([uv](https://docs.astral.sh/uv/) recommended) and, for the UI, Node 20+.
58
+
59
+ ```bash
60
+ git clone https://github.com/JohnCarl-30/rag-eval-harness.git && cd rag-eval-harness
61
+ uv sync
62
+ ./scripts/demo.sh
63
+ ```
64
+
65
+ You should see `context_recall` FAIL (0.880 → 0.740) and the script exit 0. That red `rag-eval regress` is the product. No API key. Then score dummy traces if you want a green run:
66
+
67
+ ```bash
68
+ uv run rag-eval eval examples/dummy-rag/traces.jsonl --evaluator stub -o /tmp/rag-eval-head.json
69
+ ```
70
+
71
+ Optional. Call a live HTTP SUT (the canned FAQ server):
72
+
73
+ ```bash
74
+ python3 examples/dummy-rag/server.py --port 8080 &
75
+ uv run rag-eval eval examples/dummy-rag/golden.csv \
76
+ --sut-url http://127.0.0.1:8080/query \
77
+ --evaluator stub
78
+ ```
79
+
80
+ UI (dev):
81
+
82
+ ```bash
83
+ uv run rag-eval serve # API on 127.0.0.1:8000, open on loopback (no API key)
84
+ cd web && npm install && npm run dev # Vite on :5173, proxies /api
85
+ ```
86
+
87
+ Open http://127.0.0.1:5173, upload `examples/dummy-rag/traces.jsonl` or `golden.csv`, create a run, tag a baseline, open Diff.
88
+
89
+ Docker:
90
+
91
+ ```bash
92
+ cp .env.example .env # set RAG_EVAL_API_KEY (required: the API binds 0.0.0.0)
93
+ docker compose up --build
94
+ # UI+API: http://127.0.0.1:8000 dummy RAG: http://127.0.0.1:8080/query
95
+ ```
96
+
97
+ ## Install
98
+
99
+ ```bash
100
+ pip install git+https://github.com/JohnCarl-30/rag-eval-harness.git
101
+ pip install 'rag-eval-harness[ragas]' # after a PyPI release, or: pip install '.[ragas]' from a clone
102
+ ```
103
+
104
+ ```bash
105
+ rag-eval eval traces.jsonl --evaluator stub
106
+ rag-eval eval golden.csv --sut-url https://your-rag.example/query --sut-token "$TOKEN"
107
+ rag-eval baseline <run-id>
108
+ rag-eval regress --baseline <run-id-or.json> --head <run-id-or.json> --threshold 0.05
109
+ rag-eval diff --baseline <run-id-or.json> --head <run-id-or.json>
110
+ rag-eval investigate --baseline <run-id-or.json> --head <run-id-or.json> # needs [agent]
111
+ rag-eval investigate ... --mode supervisor --trace trace.json
112
+ rag-eval serve
113
+ ```
114
+
115
+ `regress` exits **1** if any mean drops by more than `--threshold` (default `0.05`), or if head has more errored rows than baseline (means skip errored rows, so a timing-out SUT would otherwise pass). `diff` prints the same means plus the worst per-row drops (rows paired by question) and always exits 0 on a successful compare.
116
+
117
+ `investigate` runs the `regress` gate, then a LangGraph fan-out of Pydantic AI agents explains a failure: one diagnosis per failing stage (retrieval, generation), then a summary. `--mode supervisor` swaps the fixed graph for one agent that picks its own tools. `--trace` writes every agent run as JSON; `--otel` sends spans via Logfire. Same exit codes as `regress`. Install with `pip install 'rag-eval-harness[agent]'`. See [docs/agent.md](docs/agent.md).
118
+
119
+ ## Contracts
120
+
121
+ | Piece | Behavior |
122
+ | --- | --- |
123
+ | Columns | `question` required. Optional `ground_truth`. Traces also need `answer` and `retrieved_contexts` (JSON list or `\|` / newline delimited). Aliases: `user_input`, `reference`, `response`, `contexts`. |
124
+ | HTTP SUT | `POST {"question"}` → `{"answer","retrieved_contexts"}`. Optional bearer. Per-row timeout; a failed row is an error, the run continues. |
125
+ | Metrics | Always faithfulness + answer relevancy. Context precision + context recall when `ground_truth` is present. |
126
+ | Evaluators | `stub` (stable hashes, CI smoke), `lexical` (token overlap vs context, use this for real gates), `ragas` (`OPENAI_API_KEY`, optional `OPENAI_BASE_URL`). |
127
+ | Store | SQLite default (`DATABASE_URL`). Postgres via `postgresql+psycopg://…`. |
128
+ | Jobs | In-process. **100-row cap.** No Redis. |
129
+ | Auth | Open on loopback. `RAG_EVAL_API_KEY` required when binding a non-loopback address. No cross-origin access unless `RAG_EVAL_CORS_ORIGINS` lists origins (comma-separated). |
130
+ | Regression | Tag a run as baseline. Gate on **mean** delta. Per-row diffs are UI-only. |
131
+
132
+ Docs: [adapters](docs/adapters.md) · [metrics](docs/metrics.md) · [CI](docs/ci.md) · [investigator](docs/agent.md) · [storage](docs/storage.md) · [Nimbus case study](docs/nimbus-case-study.md) · [Chunking case study](docs/nimbus-chunking.md) · [Cost and p95](docs/nimbus-cost.md) · [Lexical vs judge](docs/nimbus-judge.md)
133
+
134
+ ## GitHub Action
135
+
136
+ `stub` is keyless CI smoke. Hashes look healthy on junk contexts. Use `lexical` for a real gate.
137
+
138
+ ```yaml
139
+ - uses: JohnCarl-30/rag-eval-harness@v0.2.0
140
+ with:
141
+ traces: tests/golden/traces.jsonl
142
+ evaluator: lexical
143
+ baseline: tests/golden/baseline.json
144
+ threshold: "0.05"
145
+ ```
146
+
147
+ Public consumer: [Relaydesk](https://github.com/JohnCarl-30/relaydesk) fails PRs when Nimbus recall drops. See [docs/ci.md](docs/ci.md).
148
+
149
+ ## Development
150
+
151
+ ```bash
152
+ uv sync --extra agent # without the extra, tests/test_agent.py is skipped
153
+ uv run pytest
154
+ uv run ruff check src tests
155
+ cd web && npm install && npm run build
156
+ docker compose up --build # then ./scripts/smoke.sh
157
+ ```
158
+
159
+ ## License
160
+
161
+ MIT. See [CHANGELOG](CHANGELOG.md) and [CONTRIBUTING](CONTRIBUTING.md).
@@ -0,0 +1,121 @@
1
+ # rag-eval-harness
2
+
3
+ [![CI](https://github.com/JohnCarl-30/rag-eval-harness/actions/workflows/ci.yml/badge.svg)](https://github.com/JohnCarl-30/rag-eval-harness/actions/workflows/ci.yml)
4
+
5
+ RAGAS with a memory and a diff view.
6
+
7
+ **What this proved:** a weaker retriever on a 40-question Nimbus help-center set cut mean **context recall from 0.880 to 0.740**. `rag-eval regress --threshold 0.05` exited **1**. Full write-up: [Nimbus case study](docs/nimbus-case-study.md). 90-second recording: `./scripts/demo.sh`.
8
+
9
+ Chunking the same corpus doubled lexical precision and failed the recall gate. Production stayed on whole articles. [Chunking case study](docs/nimbus-chunking.md). `./scripts/demo-chunking.sh`.
10
+
11
+ Upload a golden RAG dataset, run a pipeline (precomputed traces or HTTP), score with **RAGAS**, **lexical** overlap, or a keyless **stub** evaluator, persist runs, and fail CI when mean metrics drop versus a tagged baseline.
12
+
13
+ This is not a RAG framework. There is no ingestion or retrieval stack. Evaluation orchestration is the product.
14
+
15
+ ## 10-minute path
16
+
17
+ Requires Python 3.11+ ([uv](https://docs.astral.sh/uv/) recommended) and, for the UI, Node 20+.
18
+
19
+ ```bash
20
+ git clone https://github.com/JohnCarl-30/rag-eval-harness.git && cd rag-eval-harness
21
+ uv sync
22
+ ./scripts/demo.sh
23
+ ```
24
+
25
+ You should see `context_recall` FAIL (0.880 → 0.740) and the script exit 0. That red `rag-eval regress` is the product. No API key. Then score dummy traces if you want a green run:
26
+
27
+ ```bash
28
+ uv run rag-eval eval examples/dummy-rag/traces.jsonl --evaluator stub -o /tmp/rag-eval-head.json
29
+ ```
30
+
31
+ Optional. Call a live HTTP SUT (the canned FAQ server):
32
+
33
+ ```bash
34
+ python3 examples/dummy-rag/server.py --port 8080 &
35
+ uv run rag-eval eval examples/dummy-rag/golden.csv \
36
+ --sut-url http://127.0.0.1:8080/query \
37
+ --evaluator stub
38
+ ```
39
+
40
+ UI (dev):
41
+
42
+ ```bash
43
+ uv run rag-eval serve # API on 127.0.0.1:8000, open on loopback (no API key)
44
+ cd web && npm install && npm run dev # Vite on :5173, proxies /api
45
+ ```
46
+
47
+ Open http://127.0.0.1:5173, upload `examples/dummy-rag/traces.jsonl` or `golden.csv`, create a run, tag a baseline, open Diff.
48
+
49
+ Docker:
50
+
51
+ ```bash
52
+ cp .env.example .env # set RAG_EVAL_API_KEY (required: the API binds 0.0.0.0)
53
+ docker compose up --build
54
+ # UI+API: http://127.0.0.1:8000 dummy RAG: http://127.0.0.1:8080/query
55
+ ```
56
+
57
+ ## Install
58
+
59
+ ```bash
60
+ pip install git+https://github.com/JohnCarl-30/rag-eval-harness.git
61
+ pip install 'rag-eval-harness[ragas]' # after a PyPI release, or: pip install '.[ragas]' from a clone
62
+ ```
63
+
64
+ ```bash
65
+ rag-eval eval traces.jsonl --evaluator stub
66
+ rag-eval eval golden.csv --sut-url https://your-rag.example/query --sut-token "$TOKEN"
67
+ rag-eval baseline <run-id>
68
+ rag-eval regress --baseline <run-id-or.json> --head <run-id-or.json> --threshold 0.05
69
+ rag-eval diff --baseline <run-id-or.json> --head <run-id-or.json>
70
+ rag-eval investigate --baseline <run-id-or.json> --head <run-id-or.json> # needs [agent]
71
+ rag-eval investigate ... --mode supervisor --trace trace.json
72
+ rag-eval serve
73
+ ```
74
+
75
+ `regress` exits **1** if any mean drops by more than `--threshold` (default `0.05`), or if head has more errored rows than baseline (means skip errored rows, so a timing-out SUT would otherwise pass). `diff` prints the same means plus the worst per-row drops (rows paired by question) and always exits 0 on a successful compare.
76
+
77
+ `investigate` runs the `regress` gate, then a LangGraph fan-out of Pydantic AI agents explains a failure: one diagnosis per failing stage (retrieval, generation), then a summary. `--mode supervisor` swaps the fixed graph for one agent that picks its own tools. `--trace` writes every agent run as JSON; `--otel` sends spans via Logfire. Same exit codes as `regress`. Install with `pip install 'rag-eval-harness[agent]'`. See [docs/agent.md](docs/agent.md).
78
+
79
+ ## Contracts
80
+
81
+ | Piece | Behavior |
82
+ | --- | --- |
83
+ | Columns | `question` required. Optional `ground_truth`. Traces also need `answer` and `retrieved_contexts` (JSON list or `\|` / newline delimited). Aliases: `user_input`, `reference`, `response`, `contexts`. |
84
+ | HTTP SUT | `POST {"question"}` → `{"answer","retrieved_contexts"}`. Optional bearer. Per-row timeout; a failed row is an error, the run continues. |
85
+ | Metrics | Always faithfulness + answer relevancy. Context precision + context recall when `ground_truth` is present. |
86
+ | Evaluators | `stub` (stable hashes, CI smoke), `lexical` (token overlap vs context, use this for real gates), `ragas` (`OPENAI_API_KEY`, optional `OPENAI_BASE_URL`). |
87
+ | Store | SQLite default (`DATABASE_URL`). Postgres via `postgresql+psycopg://…`. |
88
+ | Jobs | In-process. **100-row cap.** No Redis. |
89
+ | Auth | Open on loopback. `RAG_EVAL_API_KEY` required when binding a non-loopback address. No cross-origin access unless `RAG_EVAL_CORS_ORIGINS` lists origins (comma-separated). |
90
+ | Regression | Tag a run as baseline. Gate on **mean** delta. Per-row diffs are UI-only. |
91
+
92
+ Docs: [adapters](docs/adapters.md) · [metrics](docs/metrics.md) · [CI](docs/ci.md) · [investigator](docs/agent.md) · [storage](docs/storage.md) · [Nimbus case study](docs/nimbus-case-study.md) · [Chunking case study](docs/nimbus-chunking.md) · [Cost and p95](docs/nimbus-cost.md) · [Lexical vs judge](docs/nimbus-judge.md)
93
+
94
+ ## GitHub Action
95
+
96
+ `stub` is keyless CI smoke. Hashes look healthy on junk contexts. Use `lexical` for a real gate.
97
+
98
+ ```yaml
99
+ - uses: JohnCarl-30/rag-eval-harness@v0.2.0
100
+ with:
101
+ traces: tests/golden/traces.jsonl
102
+ evaluator: lexical
103
+ baseline: tests/golden/baseline.json
104
+ threshold: "0.05"
105
+ ```
106
+
107
+ Public consumer: [Relaydesk](https://github.com/JohnCarl-30/relaydesk) fails PRs when Nimbus recall drops. See [docs/ci.md](docs/ci.md).
108
+
109
+ ## Development
110
+
111
+ ```bash
112
+ uv sync --extra agent # without the extra, tests/test_agent.py is skipped
113
+ uv run pytest
114
+ uv run ruff check src tests
115
+ cd web && npm install && npm run build
116
+ docker compose up --build # then ./scripts/smoke.sh
117
+ ```
118
+
119
+ ## License
120
+
121
+ MIT. See [CHANGELOG](CHANGELOG.md) and [CONTRIBUTING](CONTRIBUTING.md).
@@ -0,0 +1,89 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "rag-eval-harness"
7
+ version = "0.2.0"
8
+ description = "RAGAS with a memory and a diff view. Evaluate RAG pipelines, persist runs, and fail CI on regressions."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ requires-python = ">=3.11"
12
+ authors = [
13
+ { name = "John Carl Santos", email = "johncarlsantos30@gmail.com" },
14
+ ]
15
+ keywords = ["rag", "evaluation", "ragas", "llm", "ci", "lexical", "regression"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Environment :: Console",
19
+ "Framework :: FastAPI",
20
+ "Intended Audience :: Developers",
21
+ "License :: OSI Approved :: MIT License",
22
+ "Programming Language :: Python :: 3",
23
+ "Programming Language :: Python :: 3.11",
24
+ "Programming Language :: Python :: 3.12",
25
+ "Programming Language :: Python :: 3.13",
26
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
27
+ ]
28
+ dependencies = [
29
+ "fastapi>=0.115",
30
+ "httpx>=0.28",
31
+ "pydantic-settings>=2.6",
32
+ "pydantic>=2.10",
33
+ "python-dotenv>=1.0",
34
+ "python-multipart>=0.0.18",
35
+ "sqlalchemy>=2.0.36",
36
+ "typer>=0.15",
37
+ "uvicorn[standard]>=0.32",
38
+ ]
39
+
40
+ [project.optional-dependencies]
41
+ ragas = [
42
+ "openai>=1.40",
43
+ "ragas>=0.2",
44
+ ]
45
+ postgres = [
46
+ "psycopg[binary]>=3.2",
47
+ ]
48
+ agent = [
49
+ "langgraph>=1.2.2",
50
+ "pydantic-ai-slim[anthropic,openai]>=2.51.0",
51
+ ]
52
+ trace = [
53
+ "logfire>=5.1.1",
54
+ ]
55
+
56
+ [project.scripts]
57
+ rag-eval = "rag_eval_harness.cli:app"
58
+
59
+ [dependency-groups]
60
+ dev = [
61
+ "pytest>=8.3",
62
+ "pytest-asyncio>=0.24",
63
+ "ruff>=0.8",
64
+ ]
65
+
66
+ [tool.hatch.build.targets.sdist]
67
+ include = [
68
+ "/src",
69
+ "/LICENSE",
70
+ "/README.md",
71
+ ]
72
+
73
+ [tool.ruff]
74
+ line-length = 100
75
+ target-version = "py311"
76
+ src = ["src", "tests"]
77
+
78
+ [tool.ruff.lint]
79
+ select = ["E", "F", "I", "UP", "B"]
80
+
81
+ [tool.ruff.lint.per-file-ignores]
82
+ "src/rag_eval_harness/api/routes.py" = ["B008"]
83
+
84
+ [tool.pytest.ini_options]
85
+ testpaths = ["tests"]
86
+ asyncio_mode = "auto"
87
+ filterwarnings = [
88
+ "ignore::DeprecationWarning",
89
+ ]
@@ -0,0 +1,12 @@
1
+ """RAG eval harness: RAGAS with a memory and a diff view."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from importlib.metadata import PackageNotFoundError, version
6
+
7
+ try:
8
+ __version__ = version("rag-eval-harness")
9
+ except PackageNotFoundError: # pragma: no cover
10
+ __version__ = "0.2.0"
11
+
12
+ __all__ = ["__version__"]
@@ -0,0 +1,6 @@
1
+ from __future__ import annotations
2
+
3
+ from rag_eval_harness.adapters.http import HttpAdapter
4
+ from rag_eval_harness.adapters.traces import TracesAdapter
5
+
6
+ __all__ = ["HttpAdapter", "TracesAdapter"]
@@ -0,0 +1,92 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ import httpx
6
+
7
+ from rag_eval_harness.io import parse_contexts
8
+ from rag_eval_harness.types import EvalRow, is_truthy
9
+
10
+
11
+ def _row_error(exc: BaseException) -> str:
12
+ if isinstance(exc, httpx.TimeoutException):
13
+ return "http adapter: timeout"
14
+ if isinstance(exc, httpx.HTTPStatusError):
15
+ return f"http adapter: HTTP {exc.response.status_code}"
16
+ return f"http adapter: {type(exc).__name__}"
17
+
18
+
19
+ class HttpAdapter:
20
+ """POST {\"question\"} to a user RAG endpoint and collect answer + contexts."""
21
+
22
+ name = "http"
23
+
24
+ def __init__(
25
+ self,
26
+ url: str,
27
+ *,
28
+ token: str | None = None,
29
+ timeout: float = 30.0,
30
+ client: httpx.Client | None = None,
31
+ ) -> None:
32
+ self.url = url
33
+ self.token = token
34
+ self.timeout = timeout
35
+ self._client = client
36
+
37
+ def _headers(self) -> dict[str, str]:
38
+ headers = {"Content-Type": "application/json", "Accept": "application/json"}
39
+ if self.token:
40
+ headers["Authorization"] = f"Bearer {self.token}"
41
+ return headers
42
+
43
+ def _parse_response(self, payload: Any) -> tuple[str | None, list[str], bool]:
44
+ if not isinstance(payload, dict):
45
+ raise ValueError("SUT response must be a JSON object")
46
+ answer = payload.get("answer", payload.get("response"))
47
+ contexts = payload.get("retrieved_contexts", payload.get("contexts"))
48
+ abstained = payload.get("abstained", payload.get("escalated"))
49
+ if answer is None or not str(answer).strip():
50
+ raise ValueError("SUT response missing answer")
51
+ return str(answer), parse_contexts(contexts), is_truthy(abstained)
52
+
53
+ def run(self, rows: list[EvalRow]) -> list[EvalRow]:
54
+ owns_client = self._client is None
55
+ client = self._client or httpx.Client(timeout=self.timeout)
56
+ try:
57
+ out: list[EvalRow] = []
58
+ for row in rows:
59
+ if row.error:
60
+ out.append(row)
61
+ continue
62
+ try:
63
+ response = client.post(
64
+ self.url,
65
+ json={"question": row.question},
66
+ headers=self._headers(),
67
+ )
68
+ response.raise_for_status()
69
+ answer, contexts, abstained = self._parse_response(response.json())
70
+ out.append(
71
+ EvalRow(
72
+ question=row.question,
73
+ answer=answer,
74
+ retrieved_contexts=contexts,
75
+ ground_truth=row.ground_truth,
76
+ abstained=abstained,
77
+ )
78
+ )
79
+ except Exception as exc: # noqa: BLE001 — per-row failure must not abort the run
80
+ out.append(
81
+ EvalRow(
82
+ question=row.question,
83
+ answer=row.answer,
84
+ retrieved_contexts=row.retrieved_contexts,
85
+ ground_truth=row.ground_truth,
86
+ error=_row_error(exc),
87
+ )
88
+ )
89
+ return out
90
+ finally:
91
+ if owns_client:
92
+ client.close()
@@ -0,0 +1,29 @@
1
+ from __future__ import annotations
2
+
3
+ from rag_eval_harness.types import EvalRow
4
+
5
+
6
+ class TracesAdapter:
7
+ """Use precomputed answers and retrieved contexts already on each row."""
8
+
9
+ name = "traces"
10
+
11
+ def run(self, rows: list[EvalRow]) -> list[EvalRow]:
12
+ out: list[EvalRow] = []
13
+ for row in rows:
14
+ if row.error:
15
+ out.append(row)
16
+ continue
17
+ if row.answer is None or not str(row.answer).strip():
18
+ out.append(
19
+ EvalRow(
20
+ question=row.question,
21
+ answer=row.answer,
22
+ retrieved_contexts=row.retrieved_contexts,
23
+ ground_truth=row.ground_truth,
24
+ error="missing answer for traces adapter",
25
+ )
26
+ )
27
+ continue
28
+ out.append(row)
29
+ return out
@@ -0,0 +1,5 @@
1
+ """Regression investigator: a LangGraph fan-out over Pydantic AI agents.
2
+
3
+ `triage` has no extra dependencies. `agents` and `graph` need `rag-eval-harness[agent]`,
4
+ so they are not imported here.
5
+ """