claimlens 1.0.0__tar.gz

claimlens-1.0.0/.env-example

@@ -0,0 +1,4 @@
+EXTRACTOR_API_KEY="sk-1234"
+CHECKER_API_KEY="sk-1234"
+LLM_TIMEOUT=120 # not sure this is a seceret?
+# LLM_MAX_TOKENS=500   # unset = no cap; set it to cut answers short on purpose

claimlens-1.0.0/.gitattributes

@@ -0,0 +1,8 @@
+# Normalize line endings: source stays LF in the repo AND in the working
+# tree, so Windows editors stop flip-flopping and git stops warning
+# "LF will be replaced by CRLF" on every command.
+* text=auto
+*.py text eol=lf
+*.md text eol=lf
+*.toml text eol=lf
+*.json text eol=lf

claimlens-1.0.0/.gitignore

@@ -0,0 +1,44 @@
+# ── Build / Dist ─────────────────────────────────────────────────────────────
+dist/
+build/
+*.egg-info/
+
+# ── Python ───────────────────────────────────────────────────────────────────
+__pycache__/
+*.py[cod]
+*.pyo
+
+# ── Virtual Environments ────────────────────────────────────────────────────
+.venv/
+venv/
+
+# ── IDE ──────────────────────────────────────────────────────────────────────
+# launch.json is shared: it documents the standard workflows. Everything else
+# under .vscode/ is personal.
+.vscode/*
+!.vscode/launch.json
+.idea/
+
+# ── Test / Coverage ─────────────────────────────────────────────────────────
+.pytest_cache/
+.coverage
+htmlcov/
+
+# ── Secrets / Env ────────────────────────────────────────────────────────────
+*.env
+.env*
+!.env-example
+
+# ── OS ───────────────────────────────────────────────────────────────────────
+.DS_Store
+Thumbs.db
+
+# ── Results (local runs) ────────────────────────────────────────────────────
+# Outputs land next to their input, so examples/ and eval_data/ grow their own.
+**/results/
+
+ignore-found-bugs.md
+
+agent.md
+
+eval_cases_reference.md

claimlens-1.0.0/.python-version

@@ -0,0 +1 @@
+3.12

claimlens-1.0.0/.vscode/launch.json

@@ -0,0 +1,224 @@
+{
+  "version": "0.2.0",
+  "inputs": [
+    {
+      "id": "extractorModel",
+      "type": "promptString",
+      "description": "Extractor model — bare id when a base URL is set (e.g. llama3.2:3b, openai/gpt-5.6-luna)",
+      "default": "z-ai/glm-5.3-flash"
+    },
+    {
+      "id": "extractorBaseApi",
+      "type": "promptString",
+      "description": "Extractor base URL — leave blank for LiteLLM provider routing (then prefix the model, e.g. openrouter/...)",
+      "default": "https://openrouter.ai/api/v1"
+    },
+    {
+      "id": "checkerModel",
+      "type": "promptString",
+      "description": "Checker model — bare id when a base URL is set (e.g. llama3.2:3b, openai/gpt-5.6-luna)",
+      "default": "z-ai/glm-5.3-flash"
+    },
+    {
+      "id": "checkerBaseApi",
+      "type": "promptString",
+      "description": "Checker base URL — leave blank for LiteLLM provider routing (then prefix the model, e.g. openrouter/...)",
+      "default": "https://openrouter.ai/api/v1"
+    },
+    {
+      "id": "atomizerModel",
+      "type": "promptString",
+      "description": "Atomizer model — measures whether extracted claims are atomic. Needs ATOMIZER_API_KEY.",
+      "default": "z-ai/glm-5.3-flash"
+    },
+    {
+      "id": "atomizerBaseApi",
+      "type": "promptString",
+      "description": "Atomizer base URL — leave blank for LiteLLM provider routing",
+      "default": "https://openrouter.ai/api/v1"
+    }
+  ],
+  "configurations": [
+    {
+      "name": "01 · extract — kepler22b",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "args": [
+        "extract",
+        "examples/extract/kepler22b.json",
+        "--extractor-model",
+        "${input:extractorModel}",
+        "--extractor-base-api",
+        "${input:extractorBaseApi}"
+      ],
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "presentation": {
+        "group": "1 · Walkthrough",
+        "order": 1
+      }
+    },
+    {
+      "name": "02 · check — kepler22b (extracted)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "args": [
+        "check",
+        "examples/check/kepler22b_extracted.json",
+        "--extractor-model",
+        "openai/gpt-5.6-luna",
+        "--checker-model",
+        "${input:checkerModel}",
+        "--checker-base-api",
+        "${input:checkerBaseApi}"
+      ],
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "presentation": {
+        "group": "1 · Walkthrough",
+        "order": 2
+      }
+    },
+    {
+      "name": "03 · refcheck — kepler22b",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "args": [
+        "refcheck",
+        "examples/refcheck/kepler22b.json",
+        "--extractor-model",
+        "${input:extractorModel}",
+        "--extractor-base-api",
+        "${input:extractorBaseApi}",
+        "--checker-model",
+        "${input:checkerModel}",
+        "--checker-base-api",
+        "${input:checkerBaseApi}",
+        "--debug"
+      ],
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "presentation": {
+        "group": "1 · Walkthrough",
+        "order": 3
+      }
+    },
+    {
+      "name": "04 · faithcheck — kepler22b",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "args": [
+        "faithcheck",
+        "examples/faithcheck/kepler22b.json",
+        "--extractor-model",
+        "${input:extractorModel}",
+        "--extractor-base-api",
+        "${input:extractorBaseApi}",
+        "--checker-model",
+        "${input:checkerModel}",
+        "--checker-base-api",
+        "${input:checkerBaseApi}",
+        "--runs",
+        "2"
+      ],
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "presentation": {
+        "group": "1 · Walkthrough",
+        "order": 4
+      }
+    },
+    {
+      "name": "05 · ragcheck — kepler22b",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "args": [
+        "ragcheck",
+        "examples/ragcheck/kepler22b.json",
+        "--extractor-model",
+        "${input:extractorModel}",
+        "--extractor-base-api",
+        "${input:extractorBaseApi}",
+        "--checker-model",
+        "${input:checkerModel}",
+        "--checker-base-api",
+        "${input:checkerBaseApi}",
+        "--runs",
+        "2"
+      ],
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "presentation": {
+        "group": "1 · Walkthrough",
+        "order": 5
+      }
+    },
+    {
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "name": "01 · eval checker — msmarco_gpt4_5‚",
+      "args": [
+        "eval",
+        "checker",
+        "eval_data/msmarco/msmarco_gpt4_25.json",
+        "--checker-model",
+        "${input:checkerModel}",
+        "--checker-base-api",
+        "${input:checkerBaseApi}",
+        "--runs",
+        "1"
+      ],
+      "presentation": {
+        "group": "2 · Evaluation",
+        "order": 1
+      }
+    },
+    {
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/.venv/bin/claimlens",
+      "python": "${workspaceFolder}/.venv/bin/python",
+      "cwd": "${workspaceFolder}",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "name": "02 · eval extractor — msmarco_gpt4_25",
+      "args": [
+        "eval",
+        "extractor",
+        "eval_data/msmarco/msmarco_gpt4_5.json",
+        "--extractor-model",
+        "${input:extractorModel}",
+        "--extractor-base-api",
+        "${input:extractorBaseApi}",
+        "--checker-model",
+        "${input:checkerModel}",
+        "--checker-base-api",
+        "${input:checkerBaseApi}"
+      ],
+      "presentation": {
+        "group": "2 · Evaluation",
+        "order": 2
+      }
+    }
+  ]
+}

claimlens-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Arthur Galas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

claimlens-1.0.0/PKG-INFO

@@ -0,0 +1,342 @@
+Metadata-Version: 2.5
+Name: claimlens
+Version: 1.0.0
+Summary: Claim-level evaluation for LLM outputs: decompose text into atomic claims, then verify every claim against a reference.
+Project-URL: Homepage, https://github.com/Arthur-g-p/claimlens
+Project-URL: Repository, https://github.com/Arthur-g-p/claimlens
+Project-URL: Issues, https://github.com/Arthur-g-p/claimlens/issues
+Author: Arthur Galas
+License-Expression: MIT
+License-File: LICENSE
+Keywords: claim-verification,evaluation,faithfulness,hallucination,llm,rag
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: openai>=1.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: tqdm>=4.60
+Requires-Dist: typer>=0.9.0
+Provides-Extra: litellm
+Requires-Dist: litellm>=1.0.0; extra == 'litellm'
+Provides-Extra: test
+Requires-Dist: coverage>=7.0; extra == 'test'
+Requires-Dist: litellm>=1.0.0; extra == 'test'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
+Requires-Dist: pytest>=8.0; extra == 'test'
+Description-Content-Type: text/markdown
+
+# ClaimLens
+
+Claim-level evaluation for LLM outputs: decompose text into atomic claims,
+then verify every claim against a reference.
+
+Instead of asking a judge LLM "rate this answer 1-10", ClaimLens
+extracts atomic `(subject, predicate, object)` claims from a response with
+an extractor LLM, then classifies each claim against a reference with a
+checker LLM: Entailment, Contradiction, or Neutral. Every score the toolkit
+produces is an aggregate over per-claim verdicts you can read, audit, and
+disagree with.
+
+## Commands
+
+| Command | What it does |
+| --- | --- |
+| `ragcheck` | RAGChecker-style RAG evaluation: 2 extractions + 4 checking directions, one self-contained JSON report (precision, recall, F1, faithfulness, and claim-level detail) |
+| `faithcheck` | Faithfulness checking **without ground truth**: response claims vs the retrieved context. Works on live traffic. |
+| `refcheck` | Classic reference checking: extraction + checking in one pass |
+| `extract` / `check` / `atomize` | The individual building blocks, runnable standalone — walked through in order under [examples/](examples/README.md) |
+| `eval extractor` / `eval checker` | Meta-evaluation: measure the extractor and checker themselves against labeled data. Run `eval checker` first — `eval extractor` uses a checker, so an unqualified one makes its numbers partial |
+
+## Why this instead of a single judge score
+
+- **Auditable**: every metric decomposes into claim-level verdicts with
+  explanations. You can see exactly which fact failed and why.
+- **Variance is first-class**: LLM-based metrics are stochastic. Pass
+  `--runs N` to `ragcheck`, `faithcheck`, `eval extractor` or
+  `eval checker` and get `mean +/- std [min, max]` per metric across N
+  full repetitions, plus each run's complete report. A single-run score
+  overstates your certainty.
+- **The evaluator is itself evaluated**: `eval extractor` and
+  `eval checker` measure the measurement tool against ground-truth
+  labels, so you know how much to trust the numbers before you compare
+  systems with them.
+- **Honest failure handling**: per-item errors (context too long, content
+  policy, parse failure) are recorded as values next to the affected claim,
+  never silently zeroed. A claim the checker failed to judge leaves both
+  sides of the ratio rather than counting against the system under test,
+  and a metric with an empty denominator is `null`, never `0.0`.
+  Abstentions ("I don't know") are first-class labeled outcomes: judged
+  justified or unjustified, their cause apportioned between retriever
+  and generator, and unanswerable questions (`"gt_answer": ""`) expose
+  unwarranted answers — while the metrics stay standard (a refusal is
+  non-delivery; see docs/ragchecker.md#abstention).
+- **Any OpenAI-compatible endpoint**: point the extractor and checker at
+  different models, providers, or local servers independently.
+
+## Installation
+
+Requires Python 3.12+. The project uses [uv](https://docs.astral.sh/uv/) for
+dependency management — install it first if you don't have it:
+
+```bash
+# macOS / Linux
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+```bash
+# macOS (Homebrew alternative)
+brew install uv
+```
+
+```powershell
+# Windows (PowerShell)
+powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
+```
+
+Then clone and install into a virtual environment:
+
+```bash
+git clone https://github.com/Arthur-g-p/claimlens.git
+cd claimlens
+uv venv
+uv pip install -e .
+```
+
+`uv venv` reads `.python-version` and downloads CPython 3.12 if it isn't
+already present. It does not touch your system Python.
+
+Activate the environment before using the `claimlens` command:
+
+```bash
+source .venv/bin/activate     # macOS / Linux
+```
+
+```powershell
+.venv\Scripts\activate        # Windows
+```
+
+Activation is per-shell — you need it again in every new terminal. To skip it,
+prefix commands with `uv run` instead (`uv run claimlens --help`).
+
+To also install the test dependencies (pytest, pytest-asyncio, coverage):
+
+```bash
+uv pip install -e ".[test]"
+```
+
+LiteLLM routing (a provider prefix on the model name, no base URL) is an
+optional extra. The default install talks to any OpenAI-compatible endpoint
+directly and stays small. To add the routing path:
+
+```bash
+uv pip install -e ".[litellm]"
+```
+
+From PyPI that is `pip install "claimlens[litellm]"`.
+
+Verify the install:
+
+```bash
+claimlens --help
+```
+
+## Quick start
+
+Copy the environment template and fill in your API keys:
+
+```bash
+cp .env-example .env
+```
+
+```
+EXTRACTOR_API_KEY=...
+CHECKER_API_KEY=...
+```
+
+Input is a JSON list of items. For `ragcheck`, each item needs `response`,
+`gt_answer`, and `retrieved_context`:
+
+```json
+[
+  {
+    "question": "Who wrote Faust?",
+    "response": "Faust was written by Goethe in the 19th century.",
+    "gt_answer": "Johann Wolfgang von Goethe wrote Faust.",
+    "retrieved_context": ["Goethe published Faust, Part One in 1808. ..."]
+  }
+]
+```
+
+The input file is a **positional** argument on every command — pass the path
+bare, with no flag in front of it.
+
+You do not need to prepare data to start. `examples/` ships a five-step
+walkthrough over one dataset, where each step needs a little more than the last,
+plus `eval_data/` with human-labelled data for the `eval` commands.
+
+Start at step 01 — decompose responses into claims:
+
+```bash
+claimlens extract examples/extract/kepler22b.json --extractor-model gpt-4o-mini
+```
+
+Run the full RAG pipeline (step 05), which needs `gt_answer` and
+`retrieved_context` as well:
+
+```bash
+claimlens ragcheck examples/ragcheck/kepler22b.json --extractor-model gpt-4o-mini --checker-model gpt-4o-mini
+```
+
+Repeat the whole experiment five times and report variance (5x LLM cost):
+
+```bash
+claimlens ragcheck examples/ragcheck/kepler22b.json --extractor-model gpt-4o-mini --checker-model gpt-4o-mini --runs 5
+```
+
+Check faithfulness with no ground truth at all (step 04):
+
+```bash
+claimlens faithcheck examples/faithcheck/kepler22b.json --extractor-model gpt-4o-mini --checker-model gpt-4o-mini
+```
+
+The model flags have short aliases (`-e`, `-c`, `-m`), but the long names are
+spelled the same way in every command, so those are the ones worth learning.
+[examples/README.md](examples/README.md) lists which fields and which arguments
+each step needs.
+
+The pipelines and evals (`ragcheck`, `faithcheck`, `refcheck`, `eval
+extractor`, `eval checker`) write two JSON files. The record holds everything: an `_args` block (what
+you asked for — every flag of the invocation), a `_meta` block (what the run
+turned out to be — counts, timings, derived keys), `metrics` where the command
+computes any, every count the console prints, and the complete per-item
+claim record — `--runs N` adds entries, it never reshapes. The
+`_findings.json` sibling is the review queue: only what went wrong, keyed by
+the console's own branch names, derived from the record. The building
+blocks (`extract`, `check`, `atomize`) instead emit the item list itself,
+enriched in place, so each one's output is the next one's input.
+
+`ragcheck` writes a third file next to those two: `{report_stem}.html`, a
+self-contained viewer of the same record. One screen for the run — the main
+metrics with their spread over `--runs`, then every item as a row of dots,
+one per response claim, lime when a chunk grounds it — and one screen per
+item: GT claims, retrieved chunks and response claims in three columns with
+the verdicts drawn as lines between them, hover for the checker's
+explanation. It opens with a double-click, needs no server and loads nothing
+from the network. `--no-html` skips it.
+`claimlens --help` lists all commands and flags.
+
+## Use it from Python
+
+The library is the same five verbs as the CLI, with the same keyword names
+as its flags. Keys come from `EXTRACTOR_API_KEY` and `CHECKER_API_KEY` in
+the environment or a `.env`, never as arguments.
+
+```python
+import json, claimlens
+
+items = json.load(open("data.json"))
+
+record, findings = claimlens.ragcheck(items, extractor_model="gpt-4o-mini", checker_model="gpt-4o-mini")
+record["metrics"]["faithfulness"]
+findings["hallucination"]          # the review queue, the same as the _findings.json file
+
+items = claimlens.extract(items, extractor_model="gpt-4o-mini")
+```
+
+`ragcheck`, `faithcheck` and `refcheck` return the record and the findings,
+the two documents the CLI writes. `claimlens.render_html(record, findings)`
+returns the viewer page for a `ragcheck` record as a string, the same page
+the CLI writes as `{report_stem}.html`. `extract` and `check` return the enriched
+item list. Extra keyword arguments go to the pipeline (`concurrency`,
+`joint`, `extractor_base_url`, `runs`, ...). Output is silent until you ask
+for it: `claimlens.enable_logging()` turns the console output on, and
+`verbosity="compact"` or `"silent"` per call sets how much.
+
+**Inside a running event loop** (a Jupyter cell, an async server) the sync
+verbs refuse with a message that says so. Use the pipeline classes with
+`await`. It is the same code:
+
+```python
+from claimlens.pipelines.ragchecker import RagCheckerPipeline
+
+pipeline = RagCheckerPipeline(extractor_model="gpt-4o-mini", checker_model="gpt-4o-mini")
+await pipeline.run(items)
+record, findings = pipeline.last_report, pipeline.last_findings
+```
+
+**One response, in real time.** The async form first, because the caller
+is usually a server:
+
+```python
+entry = await claimlens.acheck_faithfulness(response, chunks, extractor_model="gpt-4o-mini", checker_model="gpt-4o-mini")
+entry = claimlens.check_faithfulness(response, chunks, extractor_model="gpt-4o-mini", checker_model="gpt-4o-mini")
+```
+
+The public API is what `claimlens/__init__.py` exports. Everything under
+`claimlens.pipelines`, `.services` and `.workers` is importable but may
+change between minor versions.
+
+## Using other providers and local models
+
+The extractor and checker are configured independently, and there are two ways
+to reach a non-OpenAI endpoint.
+
+**Direct endpoint** — pass a base URL with `--extractor-base-api` (or
+`--checker-base-api`). The request goes straight out over the OpenAI SDK, and
+the model name must **not** carry a provider prefix:
+
+```bash
+claimlens extract examples/extract/kepler22b.json \
+  --extractor-base-api https://openrouter.ai/api/v1 \
+  --extractor-model openai/gpt-5.6-luna
+```
+
+The same flag points at any OpenAI-compatible server, including a local one
+(`--extractor-base-api http://localhost:11434/v1`).
+
+**LiteLLM routing** (optional extra, `claimlens[litellm]`) — omit the base URL and prefix the model with its provider
+instead. LiteLLM resolves the endpoint:
+
+```bash
+claimlens extract examples/extract/kepler22b.json --extractor-model openrouter/openai/gpt-5.6-luna
+```
+
+Prefer the direct endpoint for very new models: LiteLLM routing depends on the
+installed LiteLLM release knowing the model, while a base URL bypasses that
+lookup entirely. Either way the key comes from `EXTRACTOR_API_KEY` /
+`CHECKER_API_KEY`.
+
+## Documentation
+
+- [examples/README.md](examples/README.md) — the five-step walkthrough:
+  what each command needs, what it produces, and a worked dataset with
+  deliberate errors planted in it
+- [eval_data/msmarco/README.md](eval_data/msmarco/README.md) — provenance of
+  the human-labelled evaluation data, what was repaired in it, and what it can
+  and cannot measure
+- [architecture.md](architecture.md) — the full design document (layer
+  model, data contract, error propagation)
+- [docs/request_strategies.md](docs/request_strategies.md) — how requests
+  reach the model: capability discovery, retry behaviour, timeouts, and the
+  settings you can tune
+- [docs/ragchecker.md](docs/ragchecker.md) — the RAG evaluation pipeline
+- [docs/faithfulness.md](docs/faithfulness.md) — ground-truth-free
+  faithfulness checking
+
+## Scientific lineage
+
+The methodology follows RefChecker (Amazon Science) and RAGChecker
+(claim-level RAG evaluation), with modernized components: explicit
+abstention handling, a per-item error taxonomy instead of silent nulls,
+claim deduplication, and variance reporting. It is an independent
+reimplementation, not a fork.
+
+## License
+
+MIT