radmatch 0.2.1__py3-none-any.whl

radmatch/__init__.py

@@ -0,0 +1,29 @@
+"""RadMatch — LLM-based evaluation framework for radiology report generation.
+
+`run_all` chains the three stages over a directory of reports; the per-stage functions
+run them individually, each caching to disk so a costly earlier stage is reused.
+See README.md.
+"""
+
+from __future__ import annotations
+
+from radmatch.finding_extraction.inference import ExtractionStats, extract_findings
+from radmatch.indication_extraction.inference import extract_indications
+from radmatch.llm_utils.llm_clients import assert_credentials_for
+from radmatch.matching.inference import MatchingContext, match_dataset, match_findings
+from radmatch.pipeline_runner import run_all
+from radmatch.scoring.pipeline import ScoringContext, score_dataset, score_pair
+
+__all__ = [
+    "ExtractionStats",
+    "MatchingContext",
+    "ScoringContext",
+    "assert_credentials_for",
+    "extract_findings",
+    "extract_indications",
+    "match_dataset",
+    "match_findings",
+    "run_all",
+    "score_dataset",
+    "score_pair",
+]

radmatch/assets/fewshot/README.md

@@ -0,0 +1,37 @@
+# Few-shot bundles
+
+Pass `--fewshot <bundle>` to any subcommand to adapt the LLM to a dataset's report
+style. Five bundles ship with RadMatch:
+
+| Bundle | Modality |
+|---|---|
+| `abdomen-ct` | Abdominal CT |
+| `brain-mr` | Brain MR |
+| `chest-ct` | Chest CT |
+| `chest-xr` | Chest X-ray |
+| `head-ct` | Head CT |
+
+The report text in these bundles is de-identified: dates, names, and institutions are
+replaced with `[DATE]`, `[NAME]`, `[HOSPITAL]` placeholders. Where the source de-identifier
+mis-tagged a series/image number as a date, the tag is left in place (e.g. `S301- [DATE:440]`
+is series 301, image 440) — those are intra-study coordinates, not identifiers.
+
+## Adding a bundle
+
+Create `<bundle-name>/` here with this layout. A missing or incomplete directory logs a
+warning and falls back to zero-shot prompting rather than failing.
+
+```
+<bundle-name>/
+├── reports_gt/         # Ground-truth report text:  example_1.txt, example_2.txt, ...
+├── reports_pred/       # Predicted report text:     example_1.txt, ...
+├── findings_gt/        # Stage 1 output for each GT report:   example_1.json, ...
+├── findings_pred/      # Stage 1 output for each pred report: example_1.json, ...
+├── matching/           # Stage 2 examples: pred↔GT alignment
+└── attribute_errors/   # Stage 3b examples: per-pair attribute errors
+```
+
+`reports_*` are context for a human reading the bundle; the prompts consume
+`findings_*`, `matching/` and `attribute_errors/`. Bundles installed from the wheel live
+under `radmatch/assets/fewshot/` in site-packages, so a bundle of your own is easiest to
+add in a source checkout.

radmatch/assets/fewshot/abdomen-ct/attribute_errors/README.md

@@ -0,0 +1,26 @@
+# Attribute-errors few-shot examples (abdomen-ct)
+
+Three Stage 3b examples, each derived from the matches of the corresponding `abdomen-ct/matching/example_N.json`. INC pairs (status conflicts) are excluded — Stage 3b never sees them in production.
+
+| File | Drawn from | Highlights |
+|------|------------|------------|
+| `example_1.json` | matching/example_1 | No free-text attribute errors — comparison flip on mesenteric LN and ratio diff on portal vein are owned by Stage 3a deterministic |
+| `example_2.json` | matching/example_2 | No free-text attribute errors on the consolidated hematoma + active-bleeding match |
+| `example_3.json` | matching/example_3 | Minor certainty error on hepatic-cyst pair (gt 'cyst' definite vs pred 'small hypodense focus' hedged); CBD prior-measurement diff is Stage 3a |
+
+## Format
+
+Each file is one example:
+
+```json
+{
+  "scenario": "...",
+  "description": "...",
+  "user":      { "series_uuid": "...", "pairs": [{"pred_finding": {...}, "gt_finding": {...}}, ...] },
+  "assistant": { "errors_per_match": [[...], [...], ...] }
+}
+```
+
+`user` and `assistant` are stored as raw JSON objects (parsed dicts at load time). `radmatch.llm_utils.prompts.load_attribute_errors_fewshot` reads these files when `--fewshot abdomen-ct` is passed, and the Stage 3b message builder serialises each side at construction time.
+
+INC pairs (clinical_status mismatches) are excluded upstream in `evaluate_attribute_errors_batched`, so all pairs in these examples have matching `clinical_status` on both sides.

radmatch/assets/fewshot/abdomen-ct/attribute_errors/example_1.json

@@ -0,0 +1,38 @@
+{
+  "scenario": "No free-text attribute errors — deterministic comparators own these differences",
+  "description": "Stage 3b call derived from the matches in abdomen-ct/matching/example_1.json. The mesenteric-LN comparison flip (stable vs worsening) and the portal-vein percentage spread (60% vs 50–60%) are both handled by Stage 3a deterministic comparators, so Stage 3b emits no free-text attribute errors. The renal-cysts descriptor pair ('subcentimeter' vs 'small') is treated as near-synonymous — no error.",
+  "user": {
+    "series_uuid": "abdomen-ct_example_1",
+    "pairs": [
+      {
+        "pred_finding": {"finding_id": "p_1", "text": "Hepatic veins are patent.", "clinical_status": "normal", "clinical_significance": "routine", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_1", "text": "Hepatic veins patent.", "clinical_status": "normal", "clinical_significance": "routine", "comparison": null, "measurements": []}
+      },
+      {
+        "pred_finding": {"finding_id": "p_2", "text": "Multiple small hypodense lesions in both kidneys, likely representing cysts.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_2", "text": "Multiple subcentimeter hypodense lesions in the bilateral kidneys, likely small cysts.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": null, "measurements": []}
+      },
+      {
+        "pred_finding": {"finding_id": "p_3", "text": "No hydronephrosis.", "clinical_status": "normal", "clinical_significance": "routine", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_3", "text": "No hydronephrosis.", "clinical_status": "normal", "clinical_significance": "routine", "comparison": null, "measurements": []}
+      },
+      {
+        "pred_finding": {"finding_id": "p_4", "text": "Three mesenteric lymph nodes measuring up to 12 mm, worsening compared to prior CT.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "worsening", "measurements": [{"value": 3, "unit": null, "category": "count"}, {"value": 12, "unit": "mm", "category": "size"}]},
+        "gt_finding":   {"finding_id": "gt_4", "text": "Three mesenteric lymph nodes measuring up to 12 mm, unchanged compared to prior CT.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": "stable", "measurements": [{"value": 3, "unit": null, "category": "count"}, {"value": 12, "unit": "mm", "category": "size"}]}
+      },
+      {
+        "pred_finding": {"finding_id": "p_5", "text": "Portal vein shows approximately 50-60% narrowing at the porta hepatis.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": null, "measurements": [{"value": 50, "unit": "pct", "category": "ratio"}, {"value": 60, "unit": "pct", "category": "ratio"}]},
+        "gt_finding":   {"finding_id": "gt_5", "text": "Main portal vein demonstrates approximately 60% luminal narrowing at the porta hepatis.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": null, "measurements": [{"value": 60, "unit": "pct", "category": "ratio"}]}
+      }
+    ]
+  },
+  "assistant": {
+    "errors_per_match": [
+      [],
+      [],
+      [],
+      [],
+      []
+    ]
+  }
+}

radmatch/assets/fewshot/abdomen-ct/attribute_errors/example_2.json

@@ -0,0 +1,28 @@
+{
+  "scenario": "No free-text attribute errors on consolidated trauma findings",
+  "description": "Stage 3b call derived from the matches in abdomen-ct/matching/example_2.json. The adductor hematoma + active-bleeding entries are consolidated descriptions of the same lesion and align cleanly. The pred 'Findings-section text initially described the lesion as 9 x 6 cm' clause documents a pred report inconsistency rather than a Stage 3b-scoreable attribute error. None of the 4 LLM attribute dimensions (location, severity, morphology, certainty) apply to the matched pairs.",
+  "user": {
+    "series_uuid": "abdomen-ct_example_2",
+    "pairs": [
+      {
+        "pred_finding": {"finding_id": "p_1", "text": "Gallbladder surgically absent.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_1", "text": "Gallbladder surgically absent.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": null, "measurements": []}
+      },
+      {
+        "pred_finding": {"finding_id": "p_2", "text": "Left adductor intramuscular hematoma measuring 7 x 4.7 cm (decreased from 9 x 5 cm on prior exam) with a small 6 mm central focus of hyperdensity on early-phase imaging that did not persist on delayed imaging. Findings-section text initially described the lesion as 9 x 6 cm.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "improving", "measurements": [{"value": 7, "unit": "cm", "category": "size"}, {"value": 4.7, "unit": "cm", "category": "size"}, {"value": 9, "unit": "cm", "category": "size"}, {"value": 5, "unit": "cm", "category": "size"}, {"value": 6, "unit": "mm", "category": "size"}]},
+        "gt_finding":   {"finding_id": "gt_2", "text": "Left adductor intramuscular hematoma measuring 7 x 4.7 cm (decreased from 9 x 5 cm on prior exam) with a small 6 mm central focus of hyperdensity / contrast blush on early-phase imaging that did not persist on delayed imaging.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "improving", "measurements": [{"value": 7, "unit": "cm", "category": "size"}, {"value": 4.7, "unit": "cm", "category": "size"}, {"value": 9, "unit": "cm", "category": "size"}, {"value": 5, "unit": "cm", "category": "size"}, {"value": 6, "unit": "mm", "category": "size"}]}
+      },
+      {
+        "pred_finding": {"finding_id": "p_3", "text": "Small focus of slow, active bleeding which did not continue on delays, versus pseudoaneurysm.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_3", "text": "Small focus of slow, active bleeding which did not continue on delays, versus pseudoaneurysm.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": null, "measurements": []}
+      }
+    ]
+  },
+  "assistant": {
+    "errors_per_match": [
+      [],
+      [],
+      []
+    ]
+  }
+}

radmatch/assets/fewshot/abdomen-ct/attribute_errors/example_3.json

@@ -0,0 +1,35 @@
+{
+  "scenario": "Certainty minor + clean pairs (size/comparison handled deterministically)",
+  "description": "Stage 3b call derived from the matches in abdomen-ct/matching/example_3.json. The hepatic-cyst pair's size + comparison losses (gt 9 mm + stable; pred drops both) are deterministic Stage 3a concerns. There is also a certainty diff on the hepatic-cyst pair: gt asserts a definite 'cyst', pred hedges to 'small hypodense focus' — a one-step certainty difference (definite vs hedged-descriptive) without crossing into 'possible / cannot exclude', tagged minor. CBD prior-measurement difference (6 mm vs 5 mm) is Stage 3a.",
+  "user": {
+    "series_uuid": "abdomen-ct_example_3",
+    "pairs": [
+      {
+        "pred_finding": {"finding_id": "p_1", "text": "Small hypodense focus within the right hepatic lobe.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_1", "text": "Cyst in the right hepatic lobe measuring 9 mm unchanged.", "clinical_status": "abnormal", "clinical_significance": "notable", "comparison": "stable", "measurements": [{"value": 9, "unit": "mm", "category": "size"}]}
+      },
+      {
+        "pred_finding": {"finding_id": "p_2", "text": "Common bile duct measures up to 8 mm, increased from 5 mm previously, and tapers distally.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "worsening", "measurements": [{"value": 8, "unit": "mm", "category": "size"}, {"value": 5, "unit": "mm", "category": "size"}]},
+        "gt_finding":   {"finding_id": "gt_2", "text": "Common bile duct measures up to 8 mm, increased from 6 mm previously, and tapers distally.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "worsening", "measurements": [{"value": 8, "unit": "mm", "category": "size"}, {"value": 6, "unit": "mm", "category": "size"}]}
+      },
+      {
+        "pred_finding": {"finding_id": "p_3", "text": "Focus of internal density within the common bile duct.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": null, "measurements": []},
+        "gt_finding":   {"finding_id": "gt_3", "text": "Focus of internal density within the common bile duct.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": null, "measurements": []}
+      },
+      {
+        "pred_finding": {"finding_id": "p_4", "text": "2 cm structure with fecalized internal contents associated with the proximal jejunum, not identified on the prior study, causing approximately 50% luminal narrowing.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "new", "measurements": [{"value": 2, "unit": "cm", "category": "size"}, {"value": 50, "unit": "pct", "category": "ratio"}]},
+        "gt_finding":   {"finding_id": "gt_4", "text": "Approximately 2 cm structure with fecalized internal contents associated with the proximal jejunum, not identified on the prior study, causing approximately 50% luminal narrowing.", "clinical_status": "abnormal", "clinical_significance": "urgent", "comparison": "new", "measurements": [{"value": 2, "unit": "cm", "category": "size"}, {"value": 50, "unit": "pct", "category": "ratio"}]}
+      }
+    ]
+  },
+  "assistant": {
+    "errors_per_match": [
+      [
+        {"dimension": "certainty", "severity": "minor", "reasoning": "Gt asserts a definite cyst; pred hedges to 'small hypodense focus' — one step down on the certainty scale without crossing into 'possible / cannot exclude'."}
+      ],
+      [],
+      [],
+      []
+    ]
+  }
+}

radmatch/assets/fewshot/abdomen-ct/findings_gt/example_1.json

@@ -0,0 +1,256 @@
+[
+  {
+    "text": "Bibasilar dependent atelectasis.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "Liver normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Biliary tree normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Hepatic veins patent.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Portal veins patent.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "No intrahepatic biliary dilatation.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Gallbladder normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Common bile duct measures up to 7 mm in diameter, but tapers distally.",
+    "measurements": [
+      {
+        "value": 7,
+        "unit": "mm",
+        "category": "size"
+      }
+    ],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Spleen normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Pancreas normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Adrenal glands normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Multiple subcentimeter hypodense lesions in the bilateral kidneys, likely small cysts.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "No hydronephrosis.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "No obstructing renal calculi.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Gastrointestinal tract normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Appendix normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "No significant diverticulosis.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "No evidence of diverticulitis.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "No evidence of obstruction.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "No free fluid.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Bladder normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Uterus normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Ovaries normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Atherosclerosis present.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "No aneurysm.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Prominent inguinal lymph nodes, but not enlarged by size criteria.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "Prominent left paraaortic lymph nodes, but not enlarged by size criteria.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "Three mesenteric lymph nodes measuring up to 12 mm, unchanged compared to prior CT.",
+    "measurements": [
+      {
+        "value": 3,
+        "unit": null,
+        "category": "count"
+      },
+      {
+        "value": 12,
+        "unit": "mm",
+        "category": "size"
+      }
+    ],
+    "comparison": "stable",
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "Abdominal wall normal.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  },
+  {
+    "text": "Multilevel degenerative changes of the lower thoracic spine, unchanged compared to prior imaging.",
+    "measurements": [],
+    "comparison": "stable",
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "Multilevel degenerative changes of the lumbar spine, unchanged compared to prior imaging.",
+    "measurements": [],
+    "comparison": "stable",
+    "clinical_status": "abnormal",
+    "clinical_significance": "notable"
+  },
+  {
+    "text": "Main portal vein demonstrates approximately 60% luminal narrowing at the porta hepatis.",
+    "measurements": [
+      {
+        "value": 60,
+        "unit": "pct",
+        "category": "ratio"
+      }
+    ],
+    "comparison": null,
+    "clinical_status": "abnormal",
+    "clinical_significance": "urgent"
+  },
+  {
+    "text": "No evidence of obstruction, diverticulitis, appendicitis, nephrolithiasis, or other acute findings to explain the patient's clinical symptoms.",
+    "measurements": [],
+    "comparison": null,
+    "clinical_status": "normal",
+    "clinical_significance": "routine"
+  }
+]