radfact-lite 0.1.0__tar.gz

radfact_lite-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Luca Lumetti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

radfact_lite-0.1.0/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.4
+Name: radfact-lite
+Version: 0.1.0
+Summary: Lightweight text-only RadFact metric with OpenAI and Ollama backends
+Author-email: Luca Lumetti <lumetti.luca@gmail.com>
+License-Expression: MIT
+Project-URL: Repository, https://github.com/LucaLumetti/radfact_lite
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: openai>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# radfact-lite
+
+Lightweight, text-only [RadFact](https://github.com/microsoft/RadFact) with OpenAI and Ollama backends.
+
+Parses reports into phrases, optionally filters negative findings, then scores bidirectional LLM entailment as logical precision / recall / F1.
+
+## Install
+
+```bash
+pip install radfact-lite
+```
+
+## Usage
+
+```python
+from radfact_lite import ModelConfig, PipelineModels, RadFactLitePipeline
+
+cfg = ModelConfig(model="gpt-4o-mini", provider="openai")  # reads OPENAI_API_KEY
+models = PipelineModels(parse_model=cfg, entailment_model=cfg, filtering_model=cfg)
+pipeline = RadFactLitePipeline(models)
+
+aggregate, per_sample = pipeline.compute_radfact(
+    candidates_by_id={"id1": "No pleural effusion. Mild bibasilar atelectasis."},
+    references_by_id={"id1": "Mild bibasilar atelectasis. No pneumothorax."},
+    filter_negatives=True,
+)
+print(aggregate.logical_f1)
+```
+
+For Ollama: `ModelConfig(model="llama3.1", provider="ollama")` (default base URL `http://localhost:11434/v1`).
+
+Pass pre-parsed phrase lists instead of narrative text with `is_narrative_text=False`.
+
+## New modalities
+
+Add prompt files under `src/prompts/{report_to_phrases,negative_filtering,entailment}/<modality>/` and a value to `ReportType` in `src/rf_types.py`.

radfact_lite-0.1.0/README.md

@@ -0,0 +1,36 @@
+# radfact-lite
+
+Lightweight, text-only [RadFact](https://github.com/microsoft/RadFact) with OpenAI and Ollama backends.
+
+Parses reports into phrases, optionally filters negative findings, then scores bidirectional LLM entailment as logical precision / recall / F1.
+
+## Install
+
+```bash
+pip install radfact-lite
+```
+
+## Usage
+
+```python
+from radfact_lite import ModelConfig, PipelineModels, RadFactLitePipeline
+
+cfg = ModelConfig(model="gpt-4o-mini", provider="openai")  # reads OPENAI_API_KEY
+models = PipelineModels(parse_model=cfg, entailment_model=cfg, filtering_model=cfg)
+pipeline = RadFactLitePipeline(models)
+
+aggregate, per_sample = pipeline.compute_radfact(
+    candidates_by_id={"id1": "No pleural effusion. Mild bibasilar atelectasis."},
+    references_by_id={"id1": "Mild bibasilar atelectasis. No pneumothorax."},
+    filter_negatives=True,
+)
+print(aggregate.logical_f1)
+```
+
+For Ollama: `ModelConfig(model="llama3.1", provider="ollama")` (default base URL `http://localhost:11434/v1`).
+
+Pass pre-parsed phrase lists instead of narrative text with `is_narrative_text=False`.
+
+## New modalities
+
+Add prompt files under `src/prompts/{report_to_phrases,negative_filtering,entailment}/<modality>/` and a value to `ReportType` in `src/rf_types.py`.

radfact_lite-0.1.0/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "radfact-lite"
+version = "0.1.0"
+description = "Lightweight text-only RadFact metric with OpenAI and Ollama backends"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Luca Lumetti", email = "lumetti.luca@gmail.com" }]
+requires-python = ">=3.10"
+dependencies = ["openai>=1.0.0"]
+
+[project.urls]
+Repository = "https://github.com/LucaLumetti/radfact_lite"
+
+[project.optional-dependencies]
+dev = ["pytest"]
+
+[tool.setuptools]
+package-dir = {"radfact_lite" = "src"}
+packages = ["radfact_lite", "radfact_lite.prompts"]
+
+[tool.setuptools.package-data]
+"radfact_lite.prompts" = ["**/*.txt", "**/*.json"]

radfact_lite-0.1.0/radfact_lite.egg-info/PKG-INFO

@@ -0,0 +1,51 @@
+Metadata-Version: 2.4
+Name: radfact-lite
+Version: 0.1.0
+Summary: Lightweight text-only RadFact metric with OpenAI and Ollama backends
+Author-email: Luca Lumetti <lumetti.luca@gmail.com>
+License-Expression: MIT
+Project-URL: Repository, https://github.com/LucaLumetti/radfact_lite
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: openai>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# radfact-lite
+
+Lightweight, text-only [RadFact](https://github.com/microsoft/RadFact) with OpenAI and Ollama backends.
+
+Parses reports into phrases, optionally filters negative findings, then scores bidirectional LLM entailment as logical precision / recall / F1.
+
+## Install
+
+```bash
+pip install radfact-lite
+```
+
+## Usage
+
+```python
+from radfact_lite import ModelConfig, PipelineModels, RadFactLitePipeline
+
+cfg = ModelConfig(model="gpt-4o-mini", provider="openai")  # reads OPENAI_API_KEY
+models = PipelineModels(parse_model=cfg, entailment_model=cfg, filtering_model=cfg)
+pipeline = RadFactLitePipeline(models)
+
+aggregate, per_sample = pipeline.compute_radfact(
+    candidates_by_id={"id1": "No pleural effusion. Mild bibasilar atelectasis."},
+    references_by_id={"id1": "Mild bibasilar atelectasis. No pneumothorax."},
+    filter_negatives=True,
+)
+print(aggregate.logical_f1)
+```
+
+For Ollama: `ModelConfig(model="llama3.1", provider="ollama")` (default base URL `http://localhost:11434/v1`).
+
+Pass pre-parsed phrase lists instead of narrative text with `is_narrative_text=False`.
+
+## New modalities
+
+Add prompt files under `src/prompts/{report_to_phrases,negative_filtering,entailment}/<modality>/` and a value to `ReportType` in `src/rf_types.py`.

radfact_lite-0.1.0/radfact_lite.egg-info/SOURCES.txt

@@ -0,0 +1,29 @@
+LICENSE
+README.md
+pyproject.toml
+radfact_lite.egg-info/PKG-INFO
+radfact_lite.egg-info/SOURCES.txt
+radfact_lite.egg-info/dependency_links.txt
+radfact_lite.egg-info/requires.txt
+radfact_lite.egg-info/top_level.txt
+src/__init__.py
+src/clients.py
+src/entailment.py
+src/metric.py
+src/negative_filter.py
+src/phrase_parser.py
+src/pipeline.py
+src/rf_types.py
+src/prompts/__init__.py
+src/prompts/entailment/toothfairy/few_shot_examples.json
+src/prompts/entailment/toothfairy/system_message_ev_singlephrase.txt
+src/prompts/negative_filtering/toothfairy/few_shot_examples.json
+src/prompts/negative_filtering/toothfairy/system_message.txt
+src/prompts/report_to_phrases/toothfairy/few_shot_examples.json
+src/prompts/report_to_phrases/toothfairy/system_message.txt
+tests/test_clients.py
+tests/test_entailment.py
+tests/test_metric.py
+tests/test_negative_filter.py
+tests/test_phrase_parser.py
+tests/test_pipeline_flow.py

radfact_lite-0.1.0/radfact_lite.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

radfact_lite-0.1.0/radfact_lite.egg-info/requires.txt

@@ -0,0 +1,4 @@
+openai>=1.0.0
+
+[dev]
+pytest

radfact_lite-0.1.0/radfact_lite.egg-info/top_level.txt

@@ -0,0 +1 @@
+radfact_lite

radfact_lite-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

radfact_lite-0.1.0/src/__init__.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from .clients import JSONClient, Message, OllamaJSONClient, OpenAIJSONClient, build_json_client
+from .negative_filter import filter_positive_findings
+from .pipeline import RadFactLitePipeline
+from .rf_types import (
+    EntailmentDecision,
+    ModelConfig,
+    PipelineModels,
+    RadFactAggregateResult,
+    RadFactSampleResult,
+    ReportType,
+)
+
+
+def compute_radfact_text(
+    candidates_by_id: dict[str, list[str] | str],
+    references_by_id: dict[str, list[str] | str],
+    models: PipelineModels,
+    report_type: ReportType = ReportType.TOOTHFAIRY,
+    is_narrative_text: bool = True,
+    filter_negatives: bool = False,
+) -> dict[str, float | int]:
+    pipeline = RadFactLitePipeline(models=models, report_type=report_type)
+    return pipeline.compute_radfact_dict(
+        candidates_by_id=candidates_by_id,
+        references_by_id=references_by_id,
+        is_narrative_text=is_narrative_text,
+        filter_negatives=filter_negatives,
+    )
+
+
+__all__ = [
+    "EntailmentDecision",
+    "JSONClient",
+    "Message",
+    "ModelConfig",
+    "OllamaJSONClient",
+    "OpenAIJSONClient",
+    "PipelineModels",
+    "RadFactAggregateResult",
+    "RadFactLitePipeline",
+    "RadFactSampleResult",
+    "ReportType",
+    "build_json_client",
+    "compute_radfact_text",
+    "filter_positive_findings",
+]

radfact_lite-0.1.0/src/clients.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from openai import OpenAI
+
+from .rf_types import ModelConfig
+
+OLLAMA_DEFAULT_BASE_URL = "http://localhost:11434/v1"
+
+
+@dataclass(frozen=True)
+class Message:
+    role: str
+    content: str
+
+
+class JSONClient(Protocol):
+    def complete_json(self, messages: list[Message], response_format: dict[str, Any]) -> dict[str, Any]:
+        ...
+
+
+class _ChatJSONClient:
+    def __init__(self, model: str, **client_kwargs: Any) -> None:
+        self.model = model
+        self.client = OpenAI(**client_kwargs)
+
+    def complete_json(self, messages: list[Message], response_format: dict[str, Any]) -> dict[str, Any]:
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[{"role": m.role, "content": m.content} for m in messages],
+            response_format=response_format,
+            temperature=0,
+        )
+        content = response.choices[0].message.content
+        if content is None:
+            raise ValueError("Empty response content from model")
+        try:
+            return json.loads(content)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"Invalid JSON response from model: {content}") from exc
+
+
+class OpenAIJSONClient(_ChatJSONClient):
+    def __init__(self, config: ModelConfig) -> None:
+        api_key = os.environ.get(config.api_key_env_var)
+        if not api_key:
+            raise ValueError(f"Missing API key in env var '{config.api_key_env_var}'")
+        super().__init__(
+            config.model,
+            api_key=api_key,
+            base_url=config.base_url,
+            timeout=config.timeout,
+            max_retries=config.max_retries,
+        )
+
+
+class OllamaJSONClient(_ChatJSONClient):
+    def __init__(self, config: ModelConfig) -> None:
+        super().__init__(
+            config.model,
+            api_key="ollama",
+            base_url=config.base_url or OLLAMA_DEFAULT_BASE_URL,
+            timeout=config.timeout,
+            max_retries=config.max_retries,
+        )
+
+
+def build_json_client(config: ModelConfig) -> JSONClient:
+    if config.provider == "openai":
+        return OpenAIJSONClient(config)
+    if config.provider == "ollama":
+        return OllamaJSONClient(config)
+    raise ValueError(f"Unsupported provider '{config.provider}'")

radfact_lite-0.1.0/src/entailment.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import json
+import re
+
+from .clients import JSONClient, Message
+from .prompts import load_few_shot_json, load_system_prompt
+from .rf_types import EntailmentDecision, ReportType
+
+_ENTAILMENT_SCHEMA = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "evidenced_phrase",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {
+                "phrase": {"type": "string"},
+                "evidence": {"type": "array", "items": {"type": "string"}},
+                "status": {"type": "string", "enum": ["entailment", "not_entailment"]},
+            },
+            "required": ["phrase", "evidence", "status"],
+            "additionalProperties": False,
+        },
+    },
+}
+
+
+def normalize_text(text: str) -> str:
+    text = text.lower().strip()
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r"[\-–—]", "", text)
+    text = re.sub(r"\s*(?<=[\.\:\!\?])", "", text)
+    text = re.sub(r"[\.\:\!\?]$", "", text)
+    return text
+
+
+def assess_entailment(
+    client: JSONClient,
+    reference_phrases: list[str],
+    hypothesis_phrase: str,
+    report_type: ReportType,
+) -> EntailmentDecision:
+    system = load_system_prompt("entailment", report_type, "system_message_ev_singlephrase.txt")
+    few_shots = load_few_shot_json("entailment", report_type)
+    messages = [Message(role="system", content=system)]
+
+    for shot in few_shots:
+        for pair in _expand_single_phrase_examples(shot):
+            messages.append(Message(role="user", content=json.dumps(pair["input"], ensure_ascii=False)))
+            messages.append(Message(role="assistant", content=json.dumps(pair["output"], ensure_ascii=False)))
+
+    query = {"reference": reference_phrases, "hypothesis": hypothesis_phrase}
+    messages.append(Message(role="user", content=json.dumps(query, ensure_ascii=False)))
+    out = client.complete_json(messages=messages, response_format=_ENTAILMENT_SCHEMA)
+    phrase = out["phrase"]
+    if normalize_text(phrase) != normalize_text(hypothesis_phrase):
+        phrase = hypothesis_phrase
+    return EntailmentDecision(phrase=phrase, status=out["status"], evidence=out["evidence"])
+
+
+def _expand_single_phrase_examples(example: dict) -> list[dict]:
+    result = []
+    phrases_a = example["input"]["phrases_A"]
+    phrases_b = example["input"]["phrases_B"]
+    for evidenced in example["output"]["phrases_A_evidenced"]:
+        result.append({"input": {"reference": phrases_b, "hypothesis": evidenced["phrase"]}, "output": _binary_output(evidenced)})
+    for evidenced in example["output"]["phrases_B_evidenced"]:
+        result.append({"input": {"reference": phrases_a, "hypothesis": evidenced["phrase"]}, "output": _binary_output(evidenced)})
+    return result
+
+
+def _binary_output(evidenced_phrase: dict) -> dict:
+    return {
+        "phrase": evidenced_phrase["phrase"],
+        "status": "entailment" if evidenced_phrase["status"] == "entailment" else "not_entailment",
+        "evidence": evidenced_phrase.get("evidence", []),
+    }

radfact_lite-0.1.0/src/metric.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from .rf_types import RadFactAggregateResult, RadFactSampleResult
+
+
+def divide_or_zero(numerator: int, denominator: int) -> float:
+    return 0.0 if denominator == 0 else numerator / denominator
+
+
+def f1(precision: float, recall: float) -> float:
+    return 0.0 if (precision + recall) == 0 else 2 * (precision * recall) / (precision + recall)
+
+
+def sample_result(
+    sample_id: str,
+    entailed_candidate_count: int,
+    entailed_reference_count: int,
+    candidate_count: int,
+    reference_count: int,
+) -> RadFactSampleResult:
+    precision = divide_or_zero(entailed_candidate_count, candidate_count)
+    recall = divide_or_zero(entailed_reference_count, reference_count)
+    return RadFactSampleResult(
+        sample_id=sample_id,
+        logical_precision=precision,
+        logical_recall=recall,
+        logical_f1=f1(precision, recall),
+        entailed_candidate_count=entailed_candidate_count,
+        entailed_reference_count=entailed_reference_count,
+        candidate_count=candidate_count,
+        reference_count=reference_count,
+    )
+
+
+def aggregate_results(results: list[RadFactSampleResult], num_llm_failures: int) -> RadFactAggregateResult:
+    if not results:
+        return RadFactAggregateResult(0.0, 0.0, 0.0, 0, num_llm_failures)
+    mean_precision = sum(x.logical_precision for x in results) / len(results)
+    mean_recall = sum(x.logical_recall for x in results) / len(results)
+    return RadFactAggregateResult(mean_precision, mean_recall, f1(mean_precision, mean_recall), len(results), num_llm_failures)

radfact_lite-0.1.0/src/negative_filter.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import json
+
+from .clients import JSONClient, Message
+from .prompts import load_few_shot_json, load_system_prompt
+from .rf_types import ReportType
+
+_FILTER_SCHEMA = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "positive_phrases",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {"phrases": {"type": "array", "items": {"type": "string"}}},
+            "required": ["phrases"],
+            "additionalProperties": False,
+        },
+    },
+}
+
+
+def filter_positive_findings(
+    client: JSONClient,
+    phrases: list[str],
+    report_type: ReportType = ReportType.TOOTHFAIRY,
+) -> list[str]:
+    system = load_system_prompt("negative_filtering", report_type, "system_message.txt")
+    few_shots = load_few_shot_json("negative_filtering", report_type)
+    messages = [Message(role="system", content=system)]
+    for shot in few_shots:
+        messages.append(Message(role="user", content=json.dumps(shot["input"], ensure_ascii=False)))
+        messages.append(Message(role="assistant", content=json.dumps(shot["output"], ensure_ascii=False)))
+    messages.append(Message(role="user", content=json.dumps(phrases, ensure_ascii=False)))
+    parsed = client.complete_json(messages=messages, response_format=_FILTER_SCHEMA)
+
+    original_set = set(phrases)
+    return [p for p in parsed["phrases"] if p in original_set]

radfact_lite-0.1.0/src/phrase_parser.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import json
+
+from .clients import JSONClient, Message
+from .prompts import load_few_shot_json, load_system_prompt
+from .rf_types import ReportType
+
+_PARSE_SCHEMA = {
+    "type": "json_schema",
+    "json_schema": {
+        "name": "parsed_report",
+        "strict": True,
+        "schema": {
+            "type": "object",
+            "properties": {
+                "sentence_list": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "orig": {"type": "string"},
+                            "new": {"type": "array", "items": {"type": "string"}},
+                        },
+                        "required": ["orig", "new"],
+                        "additionalProperties": False,
+                    },
+                }
+            },
+            "required": ["sentence_list"],
+            "additionalProperties": False,
+        },
+    },
+}
+
+
+def parse_report_to_phrases(client: JSONClient, report_text: str, report_type: ReportType) -> list[str]:
+    system = load_system_prompt("report_to_phrases", report_type, "system_message.txt")
+    few_shots = load_few_shot_json("report_to_phrases", report_type)
+    messages = [Message(role="system", content=system)]
+    for shot in few_shots:
+        messages.append(Message(role="user", content=shot["findings_text"]))
+        messages.append(Message(role="assistant", content=json.dumps(shot["parsed_report"], ensure_ascii=False)))
+    messages.append(Message(role="user", content=report_text))
+    parsed = client.complete_json(messages=messages, response_format=_PARSE_SCHEMA)
+
+    return [clean for sentence in parsed["sentence_list"] for phrase in sentence["new"] if (clean := phrase.strip())]

radfact_lite-0.1.0/src/pipeline.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import asdict
+
+from .clients import build_json_client
+from .entailment import assess_entailment
+from .metric import aggregate_results, sample_result
+from .negative_filter import filter_positive_findings
+from .phrase_parser import parse_report_to_phrases
+from .rf_types import PipelineModels, RadFactAggregateResult, RadFactSampleResult, ReportType
+
+
+class RadFactLitePipeline:
+    def __init__(self, models: PipelineModels, report_type: ReportType = ReportType.TOOTHFAIRY) -> None:
+        self.report_type = report_type
+        self._parse_client = build_json_client(models.parse_model)
+        self._entailment_client = build_json_client(models.entailment_model)
+        self._filtering_client = build_json_client(models.filtering_model) if models.filtering_model else None
+
+    def parse_reports(self, reports_by_id: dict[str, str]) -> dict[str, list[str]]:
+        return {k: parse_report_to_phrases(self._parse_client, v, self.report_type) for k, v in reports_by_id.items()}
+
+    def filter_negatives(self, phrases_by_id: dict[str, list[str]]) -> dict[str, list[str]]:
+        if self._filtering_client is None:
+            raise ValueError("No filtering model configured")
+        return {k: filter_positive_findings(self._filtering_client, v, self.report_type) for k, v in phrases_by_id.items()}
+
+    def compute_radfact(
+        self,
+        candidates_by_id: dict[str, list[str] | str],
+        references_by_id: dict[str, list[str] | str],
+        is_narrative_text: bool = True,
+        filter_negatives: bool = False,
+    ) -> tuple[RadFactAggregateResult, list[RadFactSampleResult]]:
+        common = sorted(set(candidates_by_id.keys()).intersection(references_by_id.keys()))
+        if is_narrative_text:
+            parsed_candidates = self.parse_reports({k: str(candidates_by_id[k]) for k in common})
+            parsed_references = self.parse_reports({k: str(references_by_id[k]) for k in common})
+        else:
+            parsed_candidates = {k: list(candidates_by_id[k]) for k in common}  # type: ignore[arg-type]
+            parsed_references = {k: list(references_by_id[k]) for k in common}  # type: ignore[arg-type]
+
+        if filter_negatives:
+            parsed_candidates = self.filter_negatives(parsed_candidates)
+            parsed_references = self.filter_negatives(parsed_references)
+
+        results: list[RadFactSampleResult] = []
+        llm_failures = 0
+        for sample_id in common:
+            candidate_phrases = parsed_candidates[sample_id]
+            reference_phrases = parsed_references[sample_id]
+            entailed_candidate = 0
+            entailed_reference = 0
+
+            for phrase in candidate_phrases:
+                try:
+                    if assess_entailment(self._entailment_client, reference_phrases, phrase, self.report_type).status == "entailment":
+                        entailed_candidate += 1
+                except Exception:
+                    llm_failures += 1
+            for phrase in reference_phrases:
+                try:
+                    if assess_entailment(self._entailment_client, candidate_phrases, phrase, self.report_type).status == "entailment":
+                        entailed_reference += 1
+                except Exception:
+                    llm_failures += 1
+
+            results.append(sample_result(sample_id, entailed_candidate, entailed_reference, len(candidate_phrases), len(reference_phrases)))
+
+        return aggregate_results(results, llm_failures), results
+
+    def compute_radfact_dict(self, *args: object, **kwargs: object) -> dict[str, float | int]:
+        aggregate, _ = self.compute_radfact(*args, **kwargs)  # type: ignore[misc]
+        return asdict(aggregate)

radfact_lite-0.1.0/src/prompts/__init__.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from ..rf_types import ReportType
+
+_ROOT = Path(__file__).resolve().parent
+
+
+def _modality_dir(task: str, report_type: ReportType) -> Path:
+    path = _ROOT / task / report_type.value
+    if not path.is_dir():
+        available = sorted(p.name for p in (_ROOT / task).iterdir() if p.is_dir())
+        raise ValueError(f"No prompts for task='{task}', modality='{report_type.value}'. Available: {available}")
+    return path
+
+
+def load_system_prompt(task: str, report_type: ReportType, filename: str) -> str:
+    return (_modality_dir(task, report_type) / filename).read_text(encoding="utf-8")
+
+
+def load_few_shot_json(task: str, report_type: ReportType) -> list[dict[str, Any]]:
+    return json.loads((_modality_dir(task, report_type) / "few_shot_examples.json").read_text(encoding="utf-8"))

radfact_lite-0.1.0/src/prompts/entailment/toothfairy/few_shot_examples.json

@@ -0,0 +1,17 @@
+[
+  {
+    "input": {
+      "phrases_A": ["Complete opacification of the right maxillary sinus is consistent with inflammatory material as a sequela of sinusitis."],
+      "phrases_B": ["The right maxillary sinus is completely airless on imaging.", "The right max. sinus is consistent with inflammatory changes from sinusitis."]
+    },
+    "output": {
+      "phrases_A_evidenced": [
+        {"phrase": "Complete opacification of the right maxillary sinus is consistent with inflammatory material as a sequela of sinusitis.", "status": "entailment", "evidence": ["The right maxillary sinus is completely airless on imaging.", "The right max. sinus is consistent with inflammatory changes from sinusitis."]}
+      ],
+      "phrases_B_evidenced": [
+        {"phrase": "The right maxillary sinus is completely airless on imaging.", "status": "entailment", "evidence": ["Complete opacification of the right maxillary sinus is consistent with inflammatory material as a sequela of sinusitis."]},
+        {"phrase": "The right max. sinus is consistent with inflammatory changes from sinusitis.", "status": "entailment", "evidence": ["Complete opacification of the right maxillary sinus is consistent with inflammatory material as a sequela of sinusitis."]}
+      ]
+    }
+  }
+]

radfact_lite-0.1.0/src/prompts/entailment/toothfairy/system_message_ev_singlephrase.txt

@@ -0,0 +1,6 @@
+You are an AI radiology assistant. Determine whether the hypothesis phrase is entailed by the reference phrase list.
+
+Return JSON with:
+- phrase: original hypothesis
+- status: entailment or not_entailment
+- evidence: supporting reference phrases

radfact_lite-0.1.0/src/prompts/negative_filtering/toothfairy/few_shot_examples.json

@@ -0,0 +1,10 @@
+[
+  {
+    "input": ["No pneumothorax.", "Mild right pleural effusion.", "Heart size is normal."],
+    "output": {"phrases": ["Mild right pleural effusion."]}
+  },
+  {
+    "input": ["All teeth are present.", "Implant is correctly osseointegrated.", "Sinus are present in the scan.", "Prosthetic crown on tooth 31."],
+    "output": {"phrases": ["Prosthetic crown on tooth 31."]}
+  }
+]

radfact_lite-0.1.0/src/prompts/negative_filtering/toothfairy/system_message.txt

@@ -0,0 +1,3 @@
+You are an AI radiology assistant processing CBCT findings phrases related to the maxillofacial field.
+Filter out phrases that are negative or normal findings.
+Keep only positive findings (including mild abnormalities).

radfact_lite-0.1.0/src/prompts/report_to_phrases/toothfairy/few_shot_examples.json

@@ -0,0 +1,13 @@
+[
+  {
+    "example_id": "few_shot_1",
+    "findings_text": "Absence from the dental arch of teeth 4.7 and 4.8. Presence of an endosseous implant in position 4.6, apparently correctly osseointegrated. Prosthetic crown on the endosseous implant in position 4.6 and on teeth 3.4, 3.5, and 3.6.",
+    "parsed_report": {
+      "id": "few_shot_1",
+      "sentence_list": [
+        {"orig": "Absence from the dental arch of teeth 4.7 and 4.8.", "new": ["Absence of tooth 4.7.", "Absence of tooth 4.8."]},
+        {"orig": "Presence of an endosseous implant in position 4.6, apparently correctly osseointegrated.", "new": ["Presence of an endosseous implant in position 4.6.", "The endosseous implant in position 4.6 is apparently correctly osseointegrated."]}
+      ]
+    }
+  }
+]

radfact_lite-0.1.0/src/prompts/report_to_phrases/toothfairy/system_message.txt

@@ -0,0 +1,10 @@
+You are an AI radiology assistant. You are helping process CBCT reports focussed on the maxillofacial region.
+Please extract phrases from the report which refer to findings or anatomies visible on maxillofacial CBCT, including absence findings.
+
+Rules:
+- Split multi-finding sentences into separate phrases.
+- Exclude recommendations and non-visual speculation.
+- Exclude scan quality comments.
+- Keep change statements.
+
+The objective is to extract phrases that can be directly verified on the CBCT.

radfact_lite-0.1.0/src/rf_types.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Literal
+
+
+class ReportType(str, Enum):
+    TOOTHFAIRY = "toothfairy"
+
+
+@dataclass(frozen=True)
+class ModelConfig:
+    model: str
+    provider: Literal["openai", "ollama"] = "openai"
+    api_key_env_var: str = "OPENAI_API_KEY"
+    base_url: str | None = None
+    timeout: float = 60.0
+    max_retries: int = 3
+
+
+@dataclass(frozen=True)
+class PipelineModels:
+    parse_model: ModelConfig
+    entailment_model: ModelConfig
+    filtering_model: ModelConfig | None = None
+
+
+@dataclass(frozen=True)
+class EntailmentDecision:
+    phrase: str
+    status: str
+    evidence: list[str]
+
+
+@dataclass(frozen=True)
+class RadFactSampleResult:
+    sample_id: str
+    logical_precision: float
+    logical_recall: float
+    logical_f1: float
+    entailed_candidate_count: int
+    entailed_reference_count: int
+    candidate_count: int
+    reference_count: int
+
+
+@dataclass(frozen=True)
+class RadFactAggregateResult:
+    logical_precision: float
+    logical_recall: float
+    logical_f1: float
+    num_samples: int
+    num_llm_failures: int

radfact_lite-0.1.0/tests/test_clients.py

@@ -0,0 +1,62 @@
+import pytest
+
+from radfact_lite.clients import Message, OllamaJSONClient, OpenAIJSONClient, build_json_client
+from radfact_lite.rf_types import ModelConfig
+
+
+class _FakeMessage:
+    def __init__(self, content):  # type: ignore[no-untyped-def]
+        self.content = content
+
+
+class _FakeChoice:
+    def __init__(self, content):  # type: ignore[no-untyped-def]
+        self.message = _FakeMessage(content)
+
+
+class _FakeResponse:
+    def __init__(self, content):  # type: ignore[no-untyped-def]
+        self.choices = [_FakeChoice(content)]
+
+
+def _fake_openai(content):  # type: ignore[no-untyped-def]
+    class FakeOpenAI:
+        def __init__(self, **kwargs):  # type: ignore[no-untyped-def]
+            self.chat = type("Chat", (), {})()
+            self.chat.completions = type("Completions", (), {})()
+            self.chat.completions.create = lambda **create_kwargs: _FakeResponse(content)
+
+    return FakeOpenAI
+
+
+def test_factory_returns_openai_client(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.clients.OpenAI", _fake_openai("{}"))
+    monkeypatch.setenv("OPENAI_API_KEY", "key")
+    client = build_json_client(ModelConfig(model="gpt-4o-mini", provider="openai"))
+    assert isinstance(client, OpenAIJSONClient)
+
+
+def test_factory_returns_ollama_client(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.clients.OpenAI", _fake_openai("{}"))
+    client = build_json_client(ModelConfig(model="llama3.1", provider="ollama"))
+    assert isinstance(client, OllamaJSONClient)
+
+
+def test_openai_client_requires_api_key(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    with pytest.raises(ValueError, match="Missing API key"):
+        OpenAIJSONClient(ModelConfig(model="gpt-4o-mini"))
+
+
+def test_ollama_client_parses_valid_json(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.clients.OpenAI", _fake_openai('{"ok": true}'))
+    client = OllamaJSONClient(ModelConfig(model="llama3.1", provider="ollama"))
+    parsed = client.complete_json([Message(role="user", content="hello")], {"type": "json_object"})
+    assert parsed == {"ok": True}
+
+
+def test_ollama_client_raises_on_invalid_json(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.clients.OpenAI", _fake_openai("not-json"))
+    client = OllamaJSONClient(ModelConfig(model="llama3.1", provider="ollama"))
+    with pytest.raises(ValueError, match="Invalid JSON response"):
+        client.complete_json([Message(role="user", content="hello")], {"type": "json_object"})

radfact_lite-0.1.0/tests/test_entailment.py

@@ -0,0 +1,56 @@
+import json
+
+from radfact_lite.entailment import _binary_output, _expand_single_phrase_examples, assess_entailment
+from radfact_lite.rf_types import ReportType
+
+
+class CapturingClient:
+    def __init__(self, response):  # type: ignore[no-untyped-def]
+        self.response = response
+        self.messages = []
+
+    def complete_json(self, messages: list, response_format: dict) -> dict:  # type: ignore[no-untyped-def]
+        del response_format
+        self.messages = messages
+        return self.response
+
+
+def test_binary_output_maps_non_entailment_states() -> None:
+    neutral = _binary_output({"phrase": "a", "status": "neutral", "evidence": []})
+    contradiction = _binary_output({"phrase": "a", "status": "contradiction", "evidence": ["x"]})
+    assert neutral["status"] == "not_entailment"
+    assert contradiction["status"] == "not_entailment"
+
+
+def test_expand_single_phrase_examples_bidirectional_count() -> None:
+    example = {
+        "input": {"phrases_A": ["a1", "a2"], "phrases_B": ["b1"]},
+        "output": {
+            "phrases_A_evidenced": [
+                {"phrase": "a1", "status": "entailment", "evidence": ["b1"]},
+                {"phrase": "a2", "status": "neutral", "evidence": []},
+            ],
+            "phrases_B_evidenced": [{"phrase": "b1", "status": "entailment", "evidence": ["a1"]}],
+        },
+    }
+    expanded = _expand_single_phrase_examples(example)
+    assert len(expanded) == 3
+
+
+def test_assess_entailment_rewrites_phrase_back_to_hypothesis(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.entailment.load_system_prompt", lambda *args, **kwargs: "sys")
+    monkeypatch.setattr("radfact_lite.entailment.load_few_shot_json", lambda *args, **kwargs: [])
+
+    client = CapturingClient(response={"phrase": "rewritten", "status": "entailment", "evidence": ["ref"]})
+    decision = assess_entailment(client, ["ref"], "original hypothesis", ReportType.TOOTHFAIRY)  # type: ignore[arg-type]
+    assert decision.phrase == "original hypothesis"
+
+
+def test_assess_entailment_sends_expected_query(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.entailment.load_system_prompt", lambda *args, **kwargs: "sys")
+    monkeypatch.setattr("radfact_lite.entailment.load_few_shot_json", lambda *args, **kwargs: [])
+
+    client = CapturingClient(response={"phrase": "h", "status": "entailment", "evidence": ["r"]})
+    assess_entailment(client, ["r1", "r2"], "h", ReportType.TOOTHFAIRY)  # type: ignore[arg-type]
+    payload = json.loads(client.messages[-1].content)
+    assert payload == {"reference": ["r1", "r2"], "hypothesis": "h"}

radfact_lite-0.1.0/tests/test_metric.py

@@ -0,0 +1,25 @@
+from radfact_lite.metric import aggregate_results, f1, sample_result
+
+
+def test_sample_result_and_f1() -> None:
+    result = sample_result(
+        sample_id="1",
+        entailed_candidate_count=3,
+        entailed_reference_count=2,
+        candidate_count=4,
+        reference_count=4,
+    )
+    assert result.logical_precision == 0.75
+    assert result.logical_recall == 0.5
+    assert result.logical_f1 == f1(0.75, 0.5)
+
+
+def test_aggregate_results() -> None:
+    r1 = sample_result("a", 2, 1, 2, 2)
+    r2 = sample_result("b", 1, 2, 2, 2)
+    aggregate = aggregate_results([r1, r2], num_llm_failures=3)
+    assert aggregate.logical_precision == 0.75
+    assert aggregate.logical_recall == 0.75
+    assert aggregate.logical_f1 == 0.75
+    assert aggregate.num_samples == 2
+    assert aggregate.num_llm_failures == 3

radfact_lite-0.1.0/tests/test_negative_filter.py

@@ -0,0 +1,36 @@
+import pytest
+
+from radfact_lite.negative_filter import filter_positive_findings
+
+
+class StubClient:
+    def __init__(self, output_phrases):  # type: ignore[no-untyped-def]
+        self.output_phrases = output_phrases
+
+    def complete_json(self, messages: list, response_format: dict) -> dict:  # type: ignore[no-untyped-def]
+        del messages, response_format
+        return {"phrases": self.output_phrases}
+
+
+def test_filter_positive_findings_drops_hallucinated_output(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.negative_filter.load_system_prompt", lambda *args, **kwargs: "sys")
+    monkeypatch.setattr("radfact_lite.negative_filter.load_few_shot_json", lambda *args, **kwargs: [])
+
+    client = StubClient(["Mild right pleural effusion.", "Hallucinated phrase"])
+    filtered = filter_positive_findings(client, ["No pneumothorax.", "Mild right pleural effusion."])  # type: ignore[arg-type]
+    assert filtered == ["Mild right pleural effusion."]
+
+
+def test_filter_positive_findings_keeps_subset_order(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.negative_filter.load_system_prompt", lambda *args, **kwargs: "sys")
+    monkeypatch.setattr("radfact_lite.negative_filter.load_few_shot_json", lambda *args, **kwargs: [])
+
+    client = StubClient(["b", "a"])
+    filtered = filter_positive_findings(client, ["a", "b", "c"])  # type: ignore[arg-type]
+    assert filtered == ["b", "a"]
+
+
+def test_filter_positive_findings_surfaces_missing_modality_error(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.negative_filter.load_system_prompt", lambda *args, **kwargs: (_ for _ in ()).throw(ValueError("missing prompts")))
+    with pytest.raises(ValueError, match="missing prompts"):
+        filter_positive_findings(StubClient([]), ["a"])  # type: ignore[arg-type]

radfact_lite-0.1.0/tests/test_phrase_parser.py

@@ -0,0 +1,41 @@
+from radfact_lite.phrase_parser import parse_report_to_phrases
+from radfact_lite.rf_types import ReportType
+
+
+class StubClient:
+    def complete_json(self, messages: list, response_format: dict) -> dict:  # type: ignore[no-untyped-def]
+        del messages, response_format
+        return {
+            "sentence_list": [
+                {"orig": "a", "new": [" first ", ""]},
+                {"orig": "b", "new": ["   ", "second"]},
+            ]
+        }
+
+
+def test_parse_report_to_phrases_removes_empty_and_preserves_order(monkeypatch):  # type: ignore[no-untyped-def]
+    monkeypatch.setattr("radfact_lite.phrase_parser.load_system_prompt", lambda *args, **kwargs: "sys")
+    monkeypatch.setattr("radfact_lite.phrase_parser.load_few_shot_json", lambda *args, **kwargs: [])
+
+    result = parse_report_to_phrases(StubClient(), "report text", ReportType.TOOTHFAIRY)  # type: ignore[arg-type]
+    assert result == ["first", "second"]
+
+
+def test_parse_report_to_phrases_uses_report_to_phrases_task(monkeypatch):  # type: ignore[no-untyped-def]
+    captured = {}
+
+    def fake_load_system(task, report_type, filename):  # type: ignore[no-untyped-def]
+        captured["task"] = task
+        captured["modality"] = report_type.value
+        captured["filename"] = filename
+        return "sys"
+
+    monkeypatch.setattr("radfact_lite.phrase_parser.load_system_prompt", fake_load_system)
+    monkeypatch.setattr("radfact_lite.phrase_parser.load_few_shot_json", lambda *args, **kwargs: [])
+
+    parse_report_to_phrases(StubClient(), "report text", ReportType.TOOTHFAIRY)  # type: ignore[arg-type]
+    assert captured == {
+        "task": "report_to_phrases",
+        "modality": "toothfairy",
+        "filename": "system_message.txt",
+    }

radfact_lite-0.1.0/tests/test_pipeline_flow.py

@@ -0,0 +1,44 @@
+from unittest.mock import patch
+
+from radfact_lite.pipeline import RadFactLitePipeline
+from radfact_lite.rf_types import EntailmentDecision, ModelConfig, PipelineModels, ReportType
+
+
+class FakeClient:
+    def complete_json(self, messages: list, response_format: dict) -> dict:  # type: ignore[no-untyped-def]
+        del messages, response_format
+        return {}
+
+
+def _models() -> PipelineModels:
+    cfg = ModelConfig(model="dummy")
+    return PipelineModels(parse_model=cfg, entailment_model=cfg, filtering_model=cfg)
+
+
+def test_pipeline_narrative_with_filtering_end_to_end_mocked() -> None:
+    with patch("radfact_lite.pipeline.build_json_client", return_value=FakeClient()):
+        pipeline = RadFactLitePipeline(_models(), report_type=ReportType.TOOTHFAIRY)
+
+    with patch("radfact_lite.pipeline.parse_report_to_phrases", side_effect=[['a', 'b'], ['a', 'c']]):
+        with patch("radfact_lite.pipeline.filter_positive_findings", side_effect=[['a', 'b'], ['a', 'c']]):
+
+            def fake_assess(client, reference_phrases, hypothesis_phrase, report_type):  # type: ignore[no-untyped-def]
+                del client, report_type
+                return EntailmentDecision(
+                    phrase=hypothesis_phrase,
+                    status="entailment" if hypothesis_phrase in reference_phrases else "not_entailment",
+                    evidence=[hypothesis_phrase] if hypothesis_phrase in reference_phrases else [],
+                )
+
+            with patch("radfact_lite.pipeline.assess_entailment", side_effect=fake_assess):
+                aggregate, per_sample = pipeline.compute_radfact(
+                    candidates_by_id={"s1": "candidate"},
+                    references_by_id={"s1": "reference"},
+                    is_narrative_text=True,
+                    filter_negatives=True,
+                )
+
+    assert aggregate.logical_precision == 0.5
+    assert aggregate.logical_recall == 0.5
+    assert aggregate.logical_f1 == 0.5
+    assert len(per_sample) == 1