rag-scorer 0.1.0__tar.gz

rag_scorer-0.1.0/.gitignore

@@ -0,0 +1,219 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+# Temporary file for partial code execution
+tempCodeRunnerFile.py
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+.DS_Store

rag_scorer-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Lucas Monteverdi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rag_scorer-0.1.0/PKG-INFO

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: rag-scorer
+Version: 0.1.0
+Summary: A from-scratch CLI evaluation framework for RAG pipelines.
+Project-URL: Homepage, https://github.com/lucasmonteverdi1/evalrag
+Project-URL: Repository, https://github.com/lucasmonteverdi1/evalrag
+Author: Lucas Monteverdi
+License: MIT License
+        
+        Copyright (c) 2026 Lucas Monteverdi
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: evaluation,llm,llm-as-judge,rag,retrieval
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.11
+Requires-Dist: openai>=1.0
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
+# EvalRAG
+
+A from-scratch CLI + library for evaluating **RAG pipelines** — no RAGAS, TruLens, or
+DeepEval wrappers.
+
+Answers *"how do I know my RAG pipeline is actually working well?"* by producing
+automated, reproducible quality scores across three metrics:
+
+- **Faithfulness** — is every claim in the answer grounded in the retrieved chunks?
+- **Answer relevance** — does the answer actually address the question?
+- **Context precision** — did the retriever surface relevant chunks, ranked high?
+
+Every score is traceable to an explicit, inspectable decision (the raw judge output is
+kept). It exits non-zero when a metric falls below its threshold, so it works as a **CI
+gate**.
+
+## Install
+
+```bash
+# From PyPI (the import package is `evalrag`):
+pip install rag-scorer
+
+# Or straight from GitHub, no clone:
+pip install git+https://github.com/lucasmonteverdi1/evalrag.git
+
+# Or with uv, from a clone:
+uv sync --all-groups
+```
+
+You need an [OpenRouter](https://openrouter.ai) API key (one key, any vendor's models):
+
+```bash
+export OPENROUTER_API_KEY="sk-or-..."
+```
+
+## Evaluate your RAG pipeline
+
+### 1. Write an adapter
+
+EvalRAG treats your pipeline as a black box. Implement one method — `run(question)` that
+returns `(retrieved_chunks, generated_answer)`:
+
+```python
+# my_eval.py
+from evalrag.types import Chunk
+
+class MyRAGAdapter:
+    def run(self, question: str) -> tuple[list[Chunk], str]:
+        # Whatever your pipeline is (LangChain, LlamaIndex, raw code — doesn't matter):
+        docs = my_retriever.search(question)
+        answer = my_llm.generate(question, docs)
+        return [Chunk(id=d.id, text=d.text) for d in docs], answer
+
+adapter = MyRAGAdapter()   # module-level instance evalrag will import
+```
+
+### 2. Provide questions
+
+A JSON list of questions to evaluate. `expected_answer` / `source_chunk_id` are optional
+(the latter enables context precision's deterministic, no-LLM path):
+
+```json
+[
+  {"question": "What is the capital of France?", "source_chunk_id": "doc-42"},
+  {"question": "What is the return policy?"}
+]
+```
+
+### 3. Run
+
+```bash
+evalrag --adapter my_eval:adapter --inputs questions.json --out-dir eval-out
+```
+
+This runs your pipeline over each question, scores all three metrics, writes
+`eval-out/report.json` + `eval-out/report.html`, and exits **0** if every metric is at
+or above its threshold, **1** if any falls below, **2** on a config/usage error.
+
+> **Note:** `--adapter` imports and runs the module you name, so only pass a spec you
+> trust (like `pytest --plugin`). Don't derive it from untrusted input.
+
+Don't have questions yet? Generate them from your documents instead of `--inputs`:
+
+```bash
+evalrag --adapter my_eval:adapter --generate documents.json
+# documents.json: [{"id": "doc-42", "text": "..."}, ...]
+```
+
+Try it with no code using the bundled demo adapter:
+
+```bash
+echo '[{"question":"What is the capital of France?"}]' > q.json
+evalrag --adapter evalrag.demo_adapter:demo_adapter --inputs q.json
+```
+
+## Reading the reports
+
+Each run writes two files to `--out-dir`:
+
+**`report.html`** — open in a browser. Top section is a **per-metric score table** plus
+an *overall* number (informational only — gating is per-metric, not on the overall).
+Below that, each evaluated case shows its question, the pipeline's answer, and every
+metric's score + rationale. Expand **"raw judge output"** on any metric to see the
+judge's verbatim reasoning — this is the traceability guarantee: no score is a black box.
+
+**`report.json`** — the same data, machine-readable, for dashboards or diffing across
+runs:
+
+```jsonc
+{
+  "summary": {
+    "n_cases": 2,
+    "per_metric": { "faithfulness": 1.0, "answer_relevance": 1.0, "context_precision": 1.0 },
+    "overall": 1.0            // informational, NOT used for gating
+  },
+  "cases": [
+    {
+      "question": "...",
+      "generated_answer": "...",
+      "metrics": [
+        { "metric": "faithfulness", "score": 1.0, "rationale": "1/1 claims grounded",
+          "raw_judge_output": "..." }   // the judge's full response, kept verbatim
+      ]
+    }
+  ]
+}
+```
+
+**How to read the numbers:** each metric is 0.0–1.0, higher is better. A metric **fails**
+(and the run exits non-zero) when its mean score is below the threshold in
+`configs/thresholds.yaml`. The terminal prints a `[PASS]`/`[FAIL]` line per metric and a
+final `Result: PASS`/`FAIL`.
+
+## Use in CI
+
+`evalrag`'s exit code gates the build:
+
+```yaml
+# .github/workflows/eval.yml
+- run: pip install git+https://github.com/lucasmonteverdi1/evalrag.git
+- run: evalrag --adapter my_eval:adapter --inputs questions.json
+  env:
+    OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+```
+
+Thresholds live in `configs/thresholds.yaml` (override with `--thresholds-config`).
+
+## Use as a library
+
+Every metric is importable and takes `(EvalCase, judge)`:
+
+```python
+from evalrag.scorer.faithfulness import score_faithfulness
+result = score_faithfulness(eval_case, judge)   # -> MetricResult(score, rationale, ...)
+```
+
+## Configuration
+
+- `configs/models.yaml` — judge and system-under-test models (the judge **must** differ
+  from the model being evaluated, to avoid self-preference bias). Secrets come from env
+  vars named here (`api_key_env`), never stored in YAML.
+- `configs/thresholds.yaml` — per-metric pass thresholds.
+- `configs/prompts.yaml` — pinned judge-prompt versions.
+
+## Troubleshooting
+
+**`ModuleNotFoundError: No module named 'evalrag.cli'` during local development.**
+Only affects the editable dev install (`uv sync`), never a `pip install` of the package.
+It happens if you mix `uv pip install/uninstall` with `uv sync`. Reset the environment:
+
+```bash
+rm -rf .venv && uv sync --all-groups
+```
+
+Then use `uv run evalrag ...`. (Don't mix `uv pip` and `uv sync` in the same venv.)
+
+## Design
+
+See [AGENTS.md](AGENTS.md) for the full architecture, metric definitions, and the
+scorer-first build order.

rag_scorer-0.1.0/README.md

@@ -0,0 +1,180 @@
+# EvalRAG
+
+A from-scratch CLI + library for evaluating **RAG pipelines** — no RAGAS, TruLens, or
+DeepEval wrappers.
+
+Answers *"how do I know my RAG pipeline is actually working well?"* by producing
+automated, reproducible quality scores across three metrics:
+
+- **Faithfulness** — is every claim in the answer grounded in the retrieved chunks?
+- **Answer relevance** — does the answer actually address the question?
+- **Context precision** — did the retriever surface relevant chunks, ranked high?
+
+Every score is traceable to an explicit, inspectable decision (the raw judge output is
+kept). It exits non-zero when a metric falls below its threshold, so it works as a **CI
+gate**.
+
+## Install
+
+```bash
+# From PyPI (the import package is `evalrag`):
+pip install rag-scorer
+
+# Or straight from GitHub, no clone:
+pip install git+https://github.com/lucasmonteverdi1/evalrag.git
+
+# Or with uv, from a clone:
+uv sync --all-groups
+```
+
+You need an [OpenRouter](https://openrouter.ai) API key (one key, any vendor's models):
+
+```bash
+export OPENROUTER_API_KEY="sk-or-..."
+```
+
+## Evaluate your RAG pipeline
+
+### 1. Write an adapter
+
+EvalRAG treats your pipeline as a black box. Implement one method — `run(question)` that
+returns `(retrieved_chunks, generated_answer)`:
+
+```python
+# my_eval.py
+from evalrag.types import Chunk
+
+class MyRAGAdapter:
+    def run(self, question: str) -> tuple[list[Chunk], str]:
+        # Whatever your pipeline is (LangChain, LlamaIndex, raw code — doesn't matter):
+        docs = my_retriever.search(question)
+        answer = my_llm.generate(question, docs)
+        return [Chunk(id=d.id, text=d.text) for d in docs], answer
+
+adapter = MyRAGAdapter()   # module-level instance evalrag will import
+```
+
+### 2. Provide questions
+
+A JSON list of questions to evaluate. `expected_answer` / `source_chunk_id` are optional
+(the latter enables context precision's deterministic, no-LLM path):
+
+```json
+[
+  {"question": "What is the capital of France?", "source_chunk_id": "doc-42"},
+  {"question": "What is the return policy?"}
+]
+```
+
+### 3. Run
+
+```bash
+evalrag --adapter my_eval:adapter --inputs questions.json --out-dir eval-out
+```
+
+This runs your pipeline over each question, scores all three metrics, writes
+`eval-out/report.json` + `eval-out/report.html`, and exits **0** if every metric is at
+or above its threshold, **1** if any falls below, **2** on a config/usage error.
+
+> **Note:** `--adapter` imports and runs the module you name, so only pass a spec you
+> trust (like `pytest --plugin`). Don't derive it from untrusted input.
+
+Don't have questions yet? Generate them from your documents instead of `--inputs`:
+
+```bash
+evalrag --adapter my_eval:adapter --generate documents.json
+# documents.json: [{"id": "doc-42", "text": "..."}, ...]
+```
+
+Try it with no code using the bundled demo adapter:
+
+```bash
+echo '[{"question":"What is the capital of France?"}]' > q.json
+evalrag --adapter evalrag.demo_adapter:demo_adapter --inputs q.json
+```
+
+## Reading the reports
+
+Each run writes two files to `--out-dir`:
+
+**`report.html`** — open in a browser. Top section is a **per-metric score table** plus
+an *overall* number (informational only — gating is per-metric, not on the overall).
+Below that, each evaluated case shows its question, the pipeline's answer, and every
+metric's score + rationale. Expand **"raw judge output"** on any metric to see the
+judge's verbatim reasoning — this is the traceability guarantee: no score is a black box.
+
+**`report.json`** — the same data, machine-readable, for dashboards or diffing across
+runs:
+
+```jsonc
+{
+  "summary": {
+    "n_cases": 2,
+    "per_metric": { "faithfulness": 1.0, "answer_relevance": 1.0, "context_precision": 1.0 },
+    "overall": 1.0            // informational, NOT used for gating
+  },
+  "cases": [
+    {
+      "question": "...",
+      "generated_answer": "...",
+      "metrics": [
+        { "metric": "faithfulness", "score": 1.0, "rationale": "1/1 claims grounded",
+          "raw_judge_output": "..." }   // the judge's full response, kept verbatim
+      ]
+    }
+  ]
+}
+```
+
+**How to read the numbers:** each metric is 0.0–1.0, higher is better. A metric **fails**
+(and the run exits non-zero) when its mean score is below the threshold in
+`configs/thresholds.yaml`. The terminal prints a `[PASS]`/`[FAIL]` line per metric and a
+final `Result: PASS`/`FAIL`.
+
+## Use in CI
+
+`evalrag`'s exit code gates the build:
+
+```yaml
+# .github/workflows/eval.yml
+- run: pip install git+https://github.com/lucasmonteverdi1/evalrag.git
+- run: evalrag --adapter my_eval:adapter --inputs questions.json
+  env:
+    OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+```
+
+Thresholds live in `configs/thresholds.yaml` (override with `--thresholds-config`).
+
+## Use as a library
+
+Every metric is importable and takes `(EvalCase, judge)`:
+
+```python
+from evalrag.scorer.faithfulness import score_faithfulness
+result = score_faithfulness(eval_case, judge)   # -> MetricResult(score, rationale, ...)
+```
+
+## Configuration
+
+- `configs/models.yaml` — judge and system-under-test models (the judge **must** differ
+  from the model being evaluated, to avoid self-preference bias). Secrets come from env
+  vars named here (`api_key_env`), never stored in YAML.
+- `configs/thresholds.yaml` — per-metric pass thresholds.
+- `configs/prompts.yaml` — pinned judge-prompt versions.
+
+## Troubleshooting
+
+**`ModuleNotFoundError: No module named 'evalrag.cli'` during local development.**
+Only affects the editable dev install (`uv sync`), never a `pip install` of the package.
+It happens if you mix `uv pip install/uninstall` with `uv sync`. Reset the environment:
+
+```bash
+rm -rf .venv && uv sync --all-groups
+```
+
+Then use `uv run evalrag ...`. (Don't mix `uv pip` and `uv sync` in the same venv.)
+
+## Design
+
+See [AGENTS.md](AGENTS.md) for the full architecture, metric definitions, and the
+scorer-first build order.

rag_scorer-0.1.0/configs/models.yaml

@@ -0,0 +1,25 @@
+judge:
+  provider: openrouter
+  base_url: https://openrouter.ai/api/v1
+  api_key_env: OPENROUTER_API_KEY   # names WHICH env var holds the key; value never in YAML
+  model: google/gemini-2.5-flash
+  temperature: 0                    # Reproducibility
+  max_tokens: 1024
+  timeout_seconds: 30
+  max_retries: 3
+
+# The system under test: the RAG pipeline's answer-generating model — the one being
+# evaluated. MUST differ from the judge (self-preference bias). Not to be confused with
+# evalrag/generator/, which synthesizes evaluation questions.
+system_under_test:
+  provider: openrouter
+  base_url: https://openrouter.ai/api/v1
+  api_key_env: OPENROUTER_API_KEY
+  model: openai/gpt-4o
+  temperature: 0
+  max_tokens: 512
+
+# Response cache — all LLM calls keyed by (provider, model, prompt_version, inputs).
+cache:
+  enabled: true
+  dir: .cache/llm_responses

rag_scorer-0.1.0/configs/prompts.yaml

@@ -0,0 +1,16 @@
+# Pinned prompt versions.
+# The judge client loads prompts from prompts/<metric>_<version>.md.
+# Bump the version string here to switch prompts for all future runs;
+# old runs remain reproducible because the cache key includes the version.
+
+# MVP metrics
+faithfulness: v1
+answer_relevance: v1
+context_precision: v1
+
+# Synthetic data generation (used by the generator, not a metric)
+generation: v1
+
+# Optional metrics (uncomment when implemented)
+# answer_correctness: v1
+# context_recall: v1

rag_scorer-0.1.0/configs/thresholds.yaml

@@ -0,0 +1,16 @@
+# Per-metric pass thresholds (0.0–1.0).
+# The CLI exits non-zero when any enabled metric's aggregate score falls below
+# its threshold. Set a threshold to null to disable gating for that metric.
+
+# MVP metrics
+faithfulness: 0.8
+answer_relevance: 0.7
+context_precision: 0.7
+
+# Optional metrics (not yet implemented; null = not gated)
+answer_correctness: null
+context_recall: null
+
+# When strict_mode is true, ANY individual EvalCase below threshold fails the
+# run (not just the aggregate). Useful for catching catastrophic regressions.
+strict_mode: false