rag-eval-harness 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rag_eval_harness-0.2.0/.gitignore +24 -0
- rag_eval_harness-0.2.0/LICENSE +21 -0
- rag_eval_harness-0.2.0/PKG-INFO +161 -0
- rag_eval_harness-0.2.0/README.md +121 -0
- rag_eval_harness-0.2.0/pyproject.toml +89 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/__init__.py +12 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/adapters/__init__.py +6 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/adapters/http.py +92 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/adapters/traces.py +29 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/agent/__init__.py +5 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/agent/agents.py +180 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/agent/graph.py +171 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/agent/supervisor.py +168 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/agent/trace.py +71 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/agent/triage.py +130 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/api/__init__.py +3 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/api/app.py +92 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/api/auth.py +57 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/api/routes.py +251 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/api/schemas.py +63 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/cli.py +439 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/config.py +44 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/engine.py +154 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/__init__.py +20 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/lexical.py +109 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/protocol.py +12 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/ragas.py +177 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/evaluators/stub.py +66 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/io.py +147 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/py.typed +0 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/redact.py +35 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/regression/__init__.py +19 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/regression/compare.py +237 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/store/__init__.py +5 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/store/models.py +79 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/store/repo.py +272 -0
- rag_eval_harness-0.2.0/src/rag_eval_harness/types.py +124 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
dist/
|
|
7
|
+
build/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.mypy_cache/
|
|
11
|
+
.coverage
|
|
12
|
+
htmlcov/
|
|
13
|
+
*.db
|
|
14
|
+
*.sqlite
|
|
15
|
+
*.sqlite3
|
|
16
|
+
rag-eval-results.json
|
|
17
|
+
.env
|
|
18
|
+
.DS_Store
|
|
19
|
+
node_modules/
|
|
20
|
+
web/dist/
|
|
21
|
+
web/.vite/
|
|
22
|
+
data/
|
|
23
|
+
.idea/
|
|
24
|
+
.vscode/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 John Carl Santos
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: rag-eval-harness
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: RAGAS with a memory and a diff view. Evaluate RAG pipelines, persist runs, and fail CI on regressions.
|
|
5
|
+
Author-email: John Carl Santos <johncarlsantos30@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: ci,evaluation,lexical,llm,rag,ragas,regression
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Framework :: FastAPI
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.11
|
|
20
|
+
Requires-Dist: fastapi>=0.115
|
|
21
|
+
Requires-Dist: httpx>=0.28
|
|
22
|
+
Requires-Dist: pydantic-settings>=2.6
|
|
23
|
+
Requires-Dist: pydantic>=2.10
|
|
24
|
+
Requires-Dist: python-dotenv>=1.0
|
|
25
|
+
Requires-Dist: python-multipart>=0.0.18
|
|
26
|
+
Requires-Dist: sqlalchemy>=2.0.36
|
|
27
|
+
Requires-Dist: typer>=0.15
|
|
28
|
+
Requires-Dist: uvicorn[standard]>=0.32
|
|
29
|
+
Provides-Extra: agent
|
|
30
|
+
Requires-Dist: langgraph>=1.2.2; extra == 'agent'
|
|
31
|
+
Requires-Dist: pydantic-ai-slim[anthropic,openai]>=2.51.0; extra == 'agent'
|
|
32
|
+
Provides-Extra: postgres
|
|
33
|
+
Requires-Dist: psycopg[binary]>=3.2; extra == 'postgres'
|
|
34
|
+
Provides-Extra: ragas
|
|
35
|
+
Requires-Dist: openai>=1.40; extra == 'ragas'
|
|
36
|
+
Requires-Dist: ragas>=0.2; extra == 'ragas'
|
|
37
|
+
Provides-Extra: trace
|
|
38
|
+
Requires-Dist: logfire>=5.1.1; extra == 'trace'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# rag-eval-harness
|
|
42
|
+
|
|
43
|
+
[](https://github.com/JohnCarl-30/rag-eval-harness/actions/workflows/ci.yml)
|
|
44
|
+
|
|
45
|
+
RAGAS with a memory and a diff view.
|
|
46
|
+
|
|
47
|
+
**What this proved:** a weaker retriever on a 40-question Nimbus help-center set cut mean **context recall from 0.880 to 0.740**. `rag-eval regress --threshold 0.05` exited **1**. Full write-up: [Nimbus case study](docs/nimbus-case-study.md). 90-second recording: `./scripts/demo.sh`.
|
|
48
|
+
|
|
49
|
+
Chunking the same corpus doubled lexical precision and failed the recall gate. Production stayed on whole articles. [Chunking case study](docs/nimbus-chunking.md). `./scripts/demo-chunking.sh`.
|
|
50
|
+
|
|
51
|
+
Upload a golden RAG dataset, run a pipeline (precomputed traces or HTTP), score with **RAGAS**, **lexical** overlap, or a keyless **stub** evaluator, persist runs, and fail CI when mean metrics drop versus a tagged baseline.
|
|
52
|
+
|
|
53
|
+
This is not a RAG framework. There is no ingestion or retrieval stack. Evaluation orchestration is the product.
|
|
54
|
+
|
|
55
|
+
## 10-minute path
|
|
56
|
+
|
|
57
|
+
Requires Python 3.11+ ([uv](https://docs.astral.sh/uv/) recommended) and, for the UI, Node 20+.
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
git clone https://github.com/JohnCarl-30/rag-eval-harness.git && cd rag-eval-harness
|
|
61
|
+
uv sync
|
|
62
|
+
./scripts/demo.sh
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
You should see `context_recall` FAIL (0.880 → 0.740) and the script exit 0. That red `rag-eval regress` is the product. No API key. Then score dummy traces if you want a green run:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
uv run rag-eval eval examples/dummy-rag/traces.jsonl --evaluator stub -o /tmp/rag-eval-head.json
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Optional. Call a live HTTP SUT (the canned FAQ server):
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
python3 examples/dummy-rag/server.py --port 8080 &
|
|
75
|
+
uv run rag-eval eval examples/dummy-rag/golden.csv \
|
|
76
|
+
--sut-url http://127.0.0.1:8080/query \
|
|
77
|
+
--evaluator stub
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
UI (dev):
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
uv run rag-eval serve # API on 127.0.0.1:8000, open on loopback (no API key)
|
|
84
|
+
cd web && npm install && npm run dev # Vite on :5173, proxies /api
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Open http://127.0.0.1:5173, upload `examples/dummy-rag/traces.jsonl` or `golden.csv`, create a run, tag a baseline, open Diff.
|
|
88
|
+
|
|
89
|
+
Docker:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
cp .env.example .env # set RAG_EVAL_API_KEY (required: the API binds 0.0.0.0)
|
|
93
|
+
docker compose up --build
|
|
94
|
+
# UI+API: http://127.0.0.1:8000 dummy RAG: http://127.0.0.1:8080/query
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Install
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install git+https://github.com/JohnCarl-30/rag-eval-harness.git
|
|
101
|
+
pip install 'rag-eval-harness[ragas]' # after a PyPI release, or: pip install '.[ragas]' from a clone
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
rag-eval eval traces.jsonl --evaluator stub
|
|
106
|
+
rag-eval eval golden.csv --sut-url https://your-rag.example/query --sut-token "$TOKEN"
|
|
107
|
+
rag-eval baseline <run-id>
|
|
108
|
+
rag-eval regress --baseline <run-id-or.json> --head <run-id-or.json> --threshold 0.05
|
|
109
|
+
rag-eval diff --baseline <run-id-or.json> --head <run-id-or.json>
|
|
110
|
+
rag-eval investigate --baseline <run-id-or.json> --head <run-id-or.json> # needs [agent]
|
|
111
|
+
rag-eval investigate ... --mode supervisor --trace trace.json
|
|
112
|
+
rag-eval serve
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
`regress` exits **1** if any mean drops by more than `--threshold` (default `0.05`), or if head has more errored rows than baseline (means skip errored rows, so a timing-out SUT would otherwise pass). `diff` prints the same means plus the worst per-row drops (rows paired by question) and always exits 0 on a successful compare.
|
|
116
|
+
|
|
117
|
+
`investigate` runs the `regress` gate, then a LangGraph fan-out of Pydantic AI agents explains a failure: one diagnosis per failing stage (retrieval, generation), then a summary. `--mode supervisor` swaps the fixed graph for one agent that picks its own tools. `--trace` writes every agent run as JSON; `--otel` sends spans via Logfire. Same exit codes as `regress`. Install with `pip install 'rag-eval-harness[agent]'`. See [docs/agent.md](docs/agent.md).
|
|
118
|
+
|
|
119
|
+
## Contracts
|
|
120
|
+
|
|
121
|
+
| Piece | Behavior |
|
|
122
|
+
| --- | --- |
|
|
123
|
+
| Columns | `question` required. Optional `ground_truth`. Traces also need `answer` and `retrieved_contexts` (JSON list or `\|` / newline delimited). Aliases: `user_input`, `reference`, `response`, `contexts`. |
|
|
124
|
+
| HTTP SUT | `POST {"question"}` → `{"answer","retrieved_contexts"}`. Optional bearer. Per-row timeout; a failed row is an error, the run continues. |
|
|
125
|
+
| Metrics | Always faithfulness + answer relevancy. Context precision + context recall when `ground_truth` is present. |
|
|
126
|
+
| Evaluators | `stub` (stable hashes, CI smoke), `lexical` (token overlap vs context, use this for real gates), `ragas` (`OPENAI_API_KEY`, optional `OPENAI_BASE_URL`). |
|
|
127
|
+
| Store | SQLite default (`DATABASE_URL`). Postgres via `postgresql+psycopg://…`. |
|
|
128
|
+
| Jobs | In-process. **100-row cap.** No Redis. |
|
|
129
|
+
| Auth | Open on loopback. `RAG_EVAL_API_KEY` required when binding a non-loopback address. No cross-origin access unless `RAG_EVAL_CORS_ORIGINS` lists origins (comma-separated). |
|
|
130
|
+
| Regression | Tag a run as baseline. Gate on **mean** delta. Per-row diffs are UI-only. |
|
|
131
|
+
|
|
132
|
+
Docs: [adapters](docs/adapters.md) · [metrics](docs/metrics.md) · [CI](docs/ci.md) · [investigator](docs/agent.md) · [storage](docs/storage.md) · [Nimbus case study](docs/nimbus-case-study.md) · [Chunking case study](docs/nimbus-chunking.md) · [Cost and p95](docs/nimbus-cost.md) · [Lexical vs judge](docs/nimbus-judge.md)
|
|
133
|
+
|
|
134
|
+
## GitHub Action
|
|
135
|
+
|
|
136
|
+
`stub` is keyless CI smoke. Hashes look healthy on junk contexts. Use `lexical` for a real gate.
|
|
137
|
+
|
|
138
|
+
```yaml
|
|
139
|
+
- uses: JohnCarl-30/rag-eval-harness@v0.2.0
|
|
140
|
+
with:
|
|
141
|
+
traces: tests/golden/traces.jsonl
|
|
142
|
+
evaluator: lexical
|
|
143
|
+
baseline: tests/golden/baseline.json
|
|
144
|
+
threshold: "0.05"
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Public consumer: [Relaydesk](https://github.com/JohnCarl-30/relaydesk) fails PRs when Nimbus recall drops. See [docs/ci.md](docs/ci.md).
|
|
148
|
+
|
|
149
|
+
## Development
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
uv sync --extra agent # without the extra, tests/test_agent.py is skipped
|
|
153
|
+
uv run pytest
|
|
154
|
+
uv run ruff check src tests
|
|
155
|
+
cd web && npm install && npm run build
|
|
156
|
+
docker compose up --build # then ./scripts/smoke.sh
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## License
|
|
160
|
+
|
|
161
|
+
MIT. See [CHANGELOG](CHANGELOG.md) and [CONTRIBUTING](CONTRIBUTING.md).
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# rag-eval-harness
|
|
2
|
+
|
|
3
|
+
[](https://github.com/JohnCarl-30/rag-eval-harness/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
RAGAS with a memory and a diff view.
|
|
6
|
+
|
|
7
|
+
**What this proved:** a weaker retriever on a 40-question Nimbus help-center set cut mean **context recall from 0.880 to 0.740**. `rag-eval regress --threshold 0.05` exited **1**. Full write-up: [Nimbus case study](docs/nimbus-case-study.md). 90-second recording: `./scripts/demo.sh`.
|
|
8
|
+
|
|
9
|
+
Chunking the same corpus doubled lexical precision and failed the recall gate. Production stayed on whole articles. [Chunking case study](docs/nimbus-chunking.md). `./scripts/demo-chunking.sh`.
|
|
10
|
+
|
|
11
|
+
Upload a golden RAG dataset, run a pipeline (precomputed traces or HTTP), score with **RAGAS**, **lexical** overlap, or a keyless **stub** evaluator, persist runs, and fail CI when mean metrics drop versus a tagged baseline.
|
|
12
|
+
|
|
13
|
+
This is not a RAG framework. There is no ingestion or retrieval stack. Evaluation orchestration is the product.
|
|
14
|
+
|
|
15
|
+
## 10-minute path
|
|
16
|
+
|
|
17
|
+
Requires Python 3.11+ ([uv](https://docs.astral.sh/uv/) recommended) and, for the UI, Node 20+.
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
git clone https://github.com/JohnCarl-30/rag-eval-harness.git && cd rag-eval-harness
|
|
21
|
+
uv sync
|
|
22
|
+
./scripts/demo.sh
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
You should see `context_recall` FAIL (0.880 → 0.740) and the script exit 0. That red `rag-eval regress` is the product. No API key. Then score dummy traces if you want a green run:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
uv run rag-eval eval examples/dummy-rag/traces.jsonl --evaluator stub -o /tmp/rag-eval-head.json
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Optional. Call a live HTTP SUT (the canned FAQ server):
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
python3 examples/dummy-rag/server.py --port 8080 &
|
|
35
|
+
uv run rag-eval eval examples/dummy-rag/golden.csv \
|
|
36
|
+
--sut-url http://127.0.0.1:8080/query \
|
|
37
|
+
--evaluator stub
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
UI (dev):
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uv run rag-eval serve # API on 127.0.0.1:8000, open on loopback (no API key)
|
|
44
|
+
cd web && npm install && npm run dev # Vite on :5173, proxies /api
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Open http://127.0.0.1:5173, upload `examples/dummy-rag/traces.jsonl` or `golden.csv`, create a run, tag a baseline, open Diff.
|
|
48
|
+
|
|
49
|
+
Docker:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
cp .env.example .env # set RAG_EVAL_API_KEY (required: the API binds 0.0.0.0)
|
|
53
|
+
docker compose up --build
|
|
54
|
+
# UI+API: http://127.0.0.1:8000 dummy RAG: http://127.0.0.1:8080/query
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install git+https://github.com/JohnCarl-30/rag-eval-harness.git
|
|
61
|
+
pip install 'rag-eval-harness[ragas]' # after a PyPI release, or: pip install '.[ragas]' from a clone
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
rag-eval eval traces.jsonl --evaluator stub
|
|
66
|
+
rag-eval eval golden.csv --sut-url https://your-rag.example/query --sut-token "$TOKEN"
|
|
67
|
+
rag-eval baseline <run-id>
|
|
68
|
+
rag-eval regress --baseline <run-id-or.json> --head <run-id-or.json> --threshold 0.05
|
|
69
|
+
rag-eval diff --baseline <run-id-or.json> --head <run-id-or.json>
|
|
70
|
+
rag-eval investigate --baseline <run-id-or.json> --head <run-id-or.json> # needs [agent]
|
|
71
|
+
rag-eval investigate ... --mode supervisor --trace trace.json
|
|
72
|
+
rag-eval serve
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`regress` exits **1** if any mean drops by more than `--threshold` (default `0.05`), or if head has more errored rows than baseline (means skip errored rows, so a timing-out SUT would otherwise pass). `diff` prints the same means plus the worst per-row drops (rows paired by question) and always exits 0 on a successful compare.
|
|
76
|
+
|
|
77
|
+
`investigate` runs the `regress` gate, then a LangGraph fan-out of Pydantic AI agents explains a failure: one diagnosis per failing stage (retrieval, generation), then a summary. `--mode supervisor` swaps the fixed graph for one agent that picks its own tools. `--trace` writes every agent run as JSON; `--otel` sends spans via Logfire. Same exit codes as `regress`. Install with `pip install 'rag-eval-harness[agent]'`. See [docs/agent.md](docs/agent.md).
|
|
78
|
+
|
|
79
|
+
## Contracts
|
|
80
|
+
|
|
81
|
+
| Piece | Behavior |
|
|
82
|
+
| --- | --- |
|
|
83
|
+
| Columns | `question` required. Optional `ground_truth`. Traces also need `answer` and `retrieved_contexts` (JSON list or `\|` / newline delimited). Aliases: `user_input`, `reference`, `response`, `contexts`. |
|
|
84
|
+
| HTTP SUT | `POST {"question"}` → `{"answer","retrieved_contexts"}`. Optional bearer. Per-row timeout; a failed row is an error, the run continues. |
|
|
85
|
+
| Metrics | Always faithfulness + answer relevancy. Context precision + context recall when `ground_truth` is present. |
|
|
86
|
+
| Evaluators | `stub` (stable hashes, CI smoke), `lexical` (token overlap vs context, use this for real gates), `ragas` (`OPENAI_API_KEY`, optional `OPENAI_BASE_URL`). |
|
|
87
|
+
| Store | SQLite default (`DATABASE_URL`). Postgres via `postgresql+psycopg://…`. |
|
|
88
|
+
| Jobs | In-process. **100-row cap.** No Redis. |
|
|
89
|
+
| Auth | Open on loopback. `RAG_EVAL_API_KEY` required when binding a non-loopback address. No cross-origin access unless `RAG_EVAL_CORS_ORIGINS` lists origins (comma-separated). |
|
|
90
|
+
| Regression | Tag a run as baseline. Gate on **mean** delta. Per-row diffs are UI-only. |
|
|
91
|
+
|
|
92
|
+
Docs: [adapters](docs/adapters.md) · [metrics](docs/metrics.md) · [CI](docs/ci.md) · [investigator](docs/agent.md) · [storage](docs/storage.md) · [Nimbus case study](docs/nimbus-case-study.md) · [Chunking case study](docs/nimbus-chunking.md) · [Cost and p95](docs/nimbus-cost.md) · [Lexical vs judge](docs/nimbus-judge.md)
|
|
93
|
+
|
|
94
|
+
## GitHub Action
|
|
95
|
+
|
|
96
|
+
`stub` is keyless CI smoke. Hashes look healthy on junk contexts. Use `lexical` for a real gate.
|
|
97
|
+
|
|
98
|
+
```yaml
|
|
99
|
+
- uses: JohnCarl-30/rag-eval-harness@v0.2.0
|
|
100
|
+
with:
|
|
101
|
+
traces: tests/golden/traces.jsonl
|
|
102
|
+
evaluator: lexical
|
|
103
|
+
baseline: tests/golden/baseline.json
|
|
104
|
+
threshold: "0.05"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Public consumer: [Relaydesk](https://github.com/JohnCarl-30/relaydesk) fails PRs when Nimbus recall drops. See [docs/ci.md](docs/ci.md).
|
|
108
|
+
|
|
109
|
+
## Development
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
uv sync --extra agent # without the extra, tests/test_agent.py is skipped
|
|
113
|
+
uv run pytest
|
|
114
|
+
uv run ruff check src tests
|
|
115
|
+
cd web && npm install && npm run build
|
|
116
|
+
docker compose up --build # then ./scripts/smoke.sh
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## License
|
|
120
|
+
|
|
121
|
+
MIT. See [CHANGELOG](CHANGELOG.md) and [CONTRIBUTING](CONTRIBUTING.md).
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "rag-eval-harness"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "RAGAS with a memory and a diff view. Evaluate RAG pipelines, persist runs, and fail CI on regressions."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
requires-python = ">=3.11"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "John Carl Santos", email = "johncarlsantos30@gmail.com" },
|
|
14
|
+
]
|
|
15
|
+
keywords = ["rag", "evaluation", "ragas", "llm", "ci", "lexical", "regression"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Framework :: FastAPI",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
]
|
|
28
|
+
dependencies = [
|
|
29
|
+
"fastapi>=0.115",
|
|
30
|
+
"httpx>=0.28",
|
|
31
|
+
"pydantic-settings>=2.6",
|
|
32
|
+
"pydantic>=2.10",
|
|
33
|
+
"python-dotenv>=1.0",
|
|
34
|
+
"python-multipart>=0.0.18",
|
|
35
|
+
"sqlalchemy>=2.0.36",
|
|
36
|
+
"typer>=0.15",
|
|
37
|
+
"uvicorn[standard]>=0.32",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
[project.optional-dependencies]
|
|
41
|
+
ragas = [
|
|
42
|
+
"openai>=1.40",
|
|
43
|
+
"ragas>=0.2",
|
|
44
|
+
]
|
|
45
|
+
postgres = [
|
|
46
|
+
"psycopg[binary]>=3.2",
|
|
47
|
+
]
|
|
48
|
+
agent = [
|
|
49
|
+
"langgraph>=1.2.2",
|
|
50
|
+
"pydantic-ai-slim[anthropic,openai]>=2.51.0",
|
|
51
|
+
]
|
|
52
|
+
trace = [
|
|
53
|
+
"logfire>=5.1.1",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
[project.scripts]
|
|
57
|
+
rag-eval = "rag_eval_harness.cli:app"
|
|
58
|
+
|
|
59
|
+
[dependency-groups]
|
|
60
|
+
dev = [
|
|
61
|
+
"pytest>=8.3",
|
|
62
|
+
"pytest-asyncio>=0.24",
|
|
63
|
+
"ruff>=0.8",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
[tool.hatch.build.targets.sdist]
|
|
67
|
+
include = [
|
|
68
|
+
"/src",
|
|
69
|
+
"/LICENSE",
|
|
70
|
+
"/README.md",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
[tool.ruff]
|
|
74
|
+
line-length = 100
|
|
75
|
+
target-version = "py311"
|
|
76
|
+
src = ["src", "tests"]
|
|
77
|
+
|
|
78
|
+
[tool.ruff.lint]
|
|
79
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
80
|
+
|
|
81
|
+
[tool.ruff.lint.per-file-ignores]
|
|
82
|
+
"src/rag_eval_harness/api/routes.py" = ["B008"]
|
|
83
|
+
|
|
84
|
+
[tool.pytest.ini_options]
|
|
85
|
+
testpaths = ["tests"]
|
|
86
|
+
asyncio_mode = "auto"
|
|
87
|
+
filterwarnings = [
|
|
88
|
+
"ignore::DeprecationWarning",
|
|
89
|
+
]
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""RAG eval harness: RAGAS with a memory and a diff view."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version("rag-eval-harness")
|
|
9
|
+
except PackageNotFoundError: # pragma: no cover
|
|
10
|
+
__version__ = "0.2.0"
|
|
11
|
+
|
|
12
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
|
|
7
|
+
from rag_eval_harness.io import parse_contexts
|
|
8
|
+
from rag_eval_harness.types import EvalRow, is_truthy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _row_error(exc: BaseException) -> str:
|
|
12
|
+
if isinstance(exc, httpx.TimeoutException):
|
|
13
|
+
return "http adapter: timeout"
|
|
14
|
+
if isinstance(exc, httpx.HTTPStatusError):
|
|
15
|
+
return f"http adapter: HTTP {exc.response.status_code}"
|
|
16
|
+
return f"http adapter: {type(exc).__name__}"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class HttpAdapter:
|
|
20
|
+
"""POST {\"question\"} to a user RAG endpoint and collect answer + contexts."""
|
|
21
|
+
|
|
22
|
+
name = "http"
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
url: str,
|
|
27
|
+
*,
|
|
28
|
+
token: str | None = None,
|
|
29
|
+
timeout: float = 30.0,
|
|
30
|
+
client: httpx.Client | None = None,
|
|
31
|
+
) -> None:
|
|
32
|
+
self.url = url
|
|
33
|
+
self.token = token
|
|
34
|
+
self.timeout = timeout
|
|
35
|
+
self._client = client
|
|
36
|
+
|
|
37
|
+
def _headers(self) -> dict[str, str]:
|
|
38
|
+
headers = {"Content-Type": "application/json", "Accept": "application/json"}
|
|
39
|
+
if self.token:
|
|
40
|
+
headers["Authorization"] = f"Bearer {self.token}"
|
|
41
|
+
return headers
|
|
42
|
+
|
|
43
|
+
def _parse_response(self, payload: Any) -> tuple[str | None, list[str], bool]:
|
|
44
|
+
if not isinstance(payload, dict):
|
|
45
|
+
raise ValueError("SUT response must be a JSON object")
|
|
46
|
+
answer = payload.get("answer", payload.get("response"))
|
|
47
|
+
contexts = payload.get("retrieved_contexts", payload.get("contexts"))
|
|
48
|
+
abstained = payload.get("abstained", payload.get("escalated"))
|
|
49
|
+
if answer is None or not str(answer).strip():
|
|
50
|
+
raise ValueError("SUT response missing answer")
|
|
51
|
+
return str(answer), parse_contexts(contexts), is_truthy(abstained)
|
|
52
|
+
|
|
53
|
+
def run(self, rows: list[EvalRow]) -> list[EvalRow]:
|
|
54
|
+
owns_client = self._client is None
|
|
55
|
+
client = self._client or httpx.Client(timeout=self.timeout)
|
|
56
|
+
try:
|
|
57
|
+
out: list[EvalRow] = []
|
|
58
|
+
for row in rows:
|
|
59
|
+
if row.error:
|
|
60
|
+
out.append(row)
|
|
61
|
+
continue
|
|
62
|
+
try:
|
|
63
|
+
response = client.post(
|
|
64
|
+
self.url,
|
|
65
|
+
json={"question": row.question},
|
|
66
|
+
headers=self._headers(),
|
|
67
|
+
)
|
|
68
|
+
response.raise_for_status()
|
|
69
|
+
answer, contexts, abstained = self._parse_response(response.json())
|
|
70
|
+
out.append(
|
|
71
|
+
EvalRow(
|
|
72
|
+
question=row.question,
|
|
73
|
+
answer=answer,
|
|
74
|
+
retrieved_contexts=contexts,
|
|
75
|
+
ground_truth=row.ground_truth,
|
|
76
|
+
abstained=abstained,
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
except Exception as exc: # noqa: BLE001 — per-row failure must not abort the run
|
|
80
|
+
out.append(
|
|
81
|
+
EvalRow(
|
|
82
|
+
question=row.question,
|
|
83
|
+
answer=row.answer,
|
|
84
|
+
retrieved_contexts=row.retrieved_contexts,
|
|
85
|
+
ground_truth=row.ground_truth,
|
|
86
|
+
error=_row_error(exc),
|
|
87
|
+
)
|
|
88
|
+
)
|
|
89
|
+
return out
|
|
90
|
+
finally:
|
|
91
|
+
if owns_client:
|
|
92
|
+
client.close()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from rag_eval_harness.types import EvalRow
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TracesAdapter:
|
|
7
|
+
"""Use precomputed answers and retrieved contexts already on each row."""
|
|
8
|
+
|
|
9
|
+
name = "traces"
|
|
10
|
+
|
|
11
|
+
def run(self, rows: list[EvalRow]) -> list[EvalRow]:
|
|
12
|
+
out: list[EvalRow] = []
|
|
13
|
+
for row in rows:
|
|
14
|
+
if row.error:
|
|
15
|
+
out.append(row)
|
|
16
|
+
continue
|
|
17
|
+
if row.answer is None or not str(row.answer).strip():
|
|
18
|
+
out.append(
|
|
19
|
+
EvalRow(
|
|
20
|
+
question=row.question,
|
|
21
|
+
answer=row.answer,
|
|
22
|
+
retrieved_contexts=row.retrieved_contexts,
|
|
23
|
+
ground_truth=row.ground_truth,
|
|
24
|
+
error="missing answer for traces adapter",
|
|
25
|
+
)
|
|
26
|
+
)
|
|
27
|
+
continue
|
|
28
|
+
out.append(row)
|
|
29
|
+
return out
|