dicleak 0.1.0__py3-none-any.whl

dicleak/__init__.py

@@ -0,0 +1,3 @@
+"""dicleak: dataset integrity auditor for medical-imaging AI."""
+
+__version__ = "0.1.0"

dicleak/checks.py

@@ -0,0 +1,208 @@
+"""The seven integrity checks. Each returns a list of Finding."""
+
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Finding:
+    check_id: str
+    severity: str  # CRITICAL / HIGH / MEDIUM / LOW
+    title: str
+    detail: str
+    deduction: int
+
+
+SEVERITY_ORDER = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}
+SEVERITY_ICON = {"CRITICAL": "[!!]", "HIGH": "[!]", "MEDIUM": "[*]", "LOW": "[.]"}
+
+_NAME_LIKE = re.compile(r"[A-Za-z]+[ _\-][A-Za-z]+")
+
+
+def _splits_of(rows):
+    return sorted({r["split"] for r in rows})
+
+
+def check_leakage(rows):
+    """CRITICAL: the same patient appears in more than one split."""
+    by_patient = defaultdict(set)
+    for r in rows:
+        by_patient[r["patient_id"]].add(r["split"])
+    leaked = {p: sorted(s) for p, s in by_patient.items() if len(s) > 1}
+    findings = []
+    for i, (patient, splits) in enumerate(sorted(leaked.items())):
+        if i >= 10:
+            findings.append(Finding(
+                "leakage", "CRITICAL",
+                "Patient-level data leakage (+%d more patients)" % (len(leaked) - 10),
+                "Additional leaked patients not listed individually.",
+                0,
+            ))
+            break
+        findings.append(Finding(
+            "leakage", "CRITICAL",
+            "Patient '%s' appears in multiple splits: %s" % (patient, ", ".join(splits)),
+            "The same patient in train and test lets the model memorize "
+            "patient-specific features instead of learning the disease. "
+            "This inflates reported accuracy and is a known cause of "
+            "non-reproducible results and retractions in medical AI.",
+            10 if i == 0 else 5,
+        ))
+    return findings
+
+
+def check_single_class_split(rows):
+    """CRITICAL: a split that contains only one label value."""
+    findings = []
+    by_split = defaultdict(set)
+    for r in rows:
+        by_split[r["split"]].add(r["label"])
+    for split in sorted(by_split):
+        labels = by_split[split]
+        if len(labels) == 1:
+            findings.append(Finding(
+                "single_class_split", "CRITICAL",
+                "Split '%s' contains only one class: '%s'" % (split, next(iter(labels))),
+                "A split with a single class cannot train or fairly evaluate "
+                "a classifier. Check your stratification or labeling pipeline.",
+                15,
+            ))
+    return findings
+
+
+def check_stratification(rows, threshold=0.20):
+    """HIGH: label distribution differs strongly between splits."""
+    findings = []
+    by_split = defaultdict(Counter)
+    for r in rows:
+        by_split[r["split"]][r["label"]] += 1
+    labels = {lbl for c in by_split.values() for lbl in c}
+    for lbl in sorted(labels):
+        shares = {}
+        for split, counter in by_split.items():
+            total = sum(counter.values())
+            shares[split] = counter[lbl] / total if total else 0.0
+        spread = max(shares.values()) - min(shares.values())
+        if spread > threshold:
+            detail = ", ".join(
+                "%s=%.0f%%" % (s, shares[s] * 100) for s in sorted(shares))
+            findings.append(Finding(
+                "stratification", "HIGH",
+                "Class '%s' is unevenly distributed across splits (%s)" % (lbl, detail),
+                "A spread of %.0fpp between splits means the model sees a "
+                "different class balance at train vs. test time, which skews "
+                "metrics. Stratify by label when splitting." % (spread * 100),
+                8,
+            ))
+    return findings
+
+
+def check_split_ratios(rows):
+    """MEDIUM: test split too small, or a split nearly empty."""
+    findings = []
+    counter = Counter(r["split"] for r in rows)
+    total = sum(counter.values())
+    for split in sorted(counter):
+        share = counter[split] / total
+        if split == "test" and share < 0.10:
+            findings.append(Finding(
+                "split_ratios", "MEDIUM",
+                "Test split is only %.1f%% of the data (%d rows)" % (share * 100, counter[split]),
+                "A tiny test set gives noisy performance estimates with wide "
+                "confidence intervals. Aim for at least 10-20%.",
+                6,
+            ))
+        elif share < 0.05:
+            findings.append(Finding(
+                "split_ratios", "MEDIUM",
+                "Split '%s' is only %.1f%% of the data (%d rows)" % (split, share * 100, counter[split]),
+                "A near-empty split contributes little and may indicate a "
+                "splitting bug.",
+                6,
+            ))
+    return findings
+
+
+def check_patient_count(rows):
+    """LOW/MEDIUM: too few unique patients to generalize."""
+    n = len({r["patient_id"] for r in rows})
+    if n < 20:
+        return [Finding(
+            "patient_count", "MEDIUM",
+            "Only %d unique patients in the whole dataset" % n,
+            "With so few patients, results are unlikely to generalize and "
+            "reviewers will question external validity.",
+            8,
+        )]
+    if n < 50:
+        return [Finding(
+            "patient_count", "LOW",
+            "Only %d unique patients in the whole dataset" % n,
+            "Small patient counts limit generalizability; consider "
+            "multi-center data or external validation.",
+            4,
+        )]
+    return []
+
+
+def check_duplicates(rows):
+    """MEDIUM: the same file listed twice (only when filepaths are given).
+
+    Without filepaths, repeated (patient, split, label) rows are expected --
+    one patient normally contributes multiple images -- so the check is skipped.
+    """
+    keyed = [r for r in rows if r.get("filepath")]
+    if not keyed:
+        return []
+    seen = Counter(r["filepath"] for r in keyed)
+    dup_files = sum(1 for c in seen.values() if c > 1)
+    if dup_files:
+        return [Finding(
+            "duplicates", "MEDIUM",
+            "%d file(s) listed more than once in the manifest" % dup_files,
+            "Duplicate files overweight those samples and can hint at a "
+            "broken export or merge step upstream.",
+            5,
+        )]
+    return []
+
+
+def check_phi_risk(rows):
+    """HIGH: patient ids that look like real names (de-identification failure)."""
+    flagged = sorted({r["patient_id"] for r in rows if _NAME_LIKE.search(r["patient_id"])})
+    if not flagged:
+        return []
+    shown = ", ".join("'%s'" % p for p in flagged[:5])
+    more = " (+%d more)" % (len(flagged) - 5) if len(flagged) > 5 else ""
+    return [Finding(
+        "phi_risk", "HIGH",
+        "%d patient id(s) look like personal names: %s%s" % (len(flagged), shown, more),
+        "IDs resembling names suggest the data was not de-identified. "
+        "Sharing or publishing this dataset may violate privacy rules "
+        "(HIPAA/GDPR) and journal data policies.",
+        8,
+    )]
+
+
+ALL_CHECKS = (
+    check_leakage,
+    check_single_class_split,
+    check_stratification,
+    check_split_ratios,
+    check_patient_count,
+    check_duplicates,
+    check_phi_risk,
+)
+
+
+def run_all(rows):
+    findings = []
+    for check in ALL_CHECKS:
+        findings.extend(check(rows))
+    findings.sort(key=lambda f: (SEVERITY_ORDER[f.severity], f.check_id))
+    return findings
+
+
+def score(findings):
+    return max(0, 100 - sum(f.deduction for f in findings))

dicleak/cli.py

@@ -0,0 +1,44 @@
+"""Command-line interface."""
+
+import argparse
+import sys
+
+from . import __version__
+from .checks import run_all, score
+from .readers import load_manifest
+from .report import render_markdown, render_terminal
+
+
+def build_parser():
+    p = argparse.ArgumentParser(
+        prog="dicleak",
+        description="Audit a medical-imaging AI dataset manifest for "
+                    "patient-level leakage and split flaws.",
+    )
+    p.add_argument("manifest", help="CSV with patient_id, split and label columns")
+    p.add_argument("--report", metavar="OUT.md",
+                   help="also write a Markdown report to OUT.md")
+    p.add_argument("--cta", metavar="URL",
+                   help="append a service link footer to the Markdown report")
+    p.add_argument("--version", action="version", version="dicleak " + __version__)
+    return p
+
+
+def main(argv=None):
+    args = build_parser().parse_args(argv)
+    rows, warnings = load_manifest(args.manifest)
+    for w in warnings:
+        print("warning: %s" % w, file=sys.stderr)
+    findings = run_all(rows)
+    value = score(findings)
+    print(render_terminal(findings, value, rows))
+    if args.report:
+        md = render_markdown(findings, value, rows, cta_url=args.cta)
+        with open(args.report, "w", encoding="utf-8") as fh:
+            fh.write(md)
+        print("Markdown report written to %s" % args.report)
+    return 0 if value >= 60 else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dicleak/readers.py

@@ -0,0 +1,93 @@
+"""CSV manifest loading and column normalization."""
+
+import csv
+import sys
+
+# accepted header spellings -> canonical field name
+COLUMN_ALIASES = {
+    "patient_id": "patient_id",
+    "patientid": "patient_id",
+    "patient": "patient_id",
+    "subject_id": "patient_id",
+    "subjectid": "patient_id",
+    "subject": "patient_id",
+    "pid": "patient_id",
+    "split": "split",
+    "subset": "split",
+    "fold": "split",
+    "partition": "split",
+    "set": "split",
+    "phase": "split",
+    "label": "label",
+    "class": "label",
+    "target": "label",
+    "diagnosis": "label",
+    "annotation": "label",
+    "filepath": "filepath",
+    "file_path": "filepath",
+    "path": "filepath",
+    "filename": "filepath",
+    "file": "filepath",
+}
+
+SPLIT_ALIASES = {
+    "train": "train",
+    "training": "train",
+    "trn": "train",
+    "val": "val",
+    "valid": "val",
+    "validation": "val",
+    "dev": "val",
+    "test": "test",
+    "testing": "test",
+    "tst": "test",
+}
+
+REQUIRED = ("patient_id", "split", "label")
+
+
+def _normalize_header(name):
+    return name.strip().lower().replace("-", "_").replace(" ", "_")
+
+
+def load_manifest(path):
+    """Load a CSV manifest, return (rows, warnings).
+
+    rows: list of dicts with keys patient_id / split / label / filepath.
+    Raises SystemExit with a message on fatal problems.
+    """
+    try:
+        fh = open(path, newline="", encoding="utf-8-sig")
+    except OSError as exc:
+        sys.exit("dicleak: cannot open %s: %s" % (path, exc))
+    with fh:
+        reader = csv.DictReader(fh)
+        if not reader.fieldnames:
+            sys.exit("dicleak: %s looks empty (no header row)" % path)
+        mapping = {}
+        for raw in reader.fieldnames:
+            key = _normalize_header(raw)
+            if key in COLUMN_ALIASES:
+                mapping[raw] = COLUMN_ALIASES[key]
+        missing = [c for c in REQUIRED if c not in mapping.values()]
+        if missing:
+            sys.exit(
+                "dicleak: %s is missing required column(s): %s "
+                "(need patient id, split and label columns)" % (path, ", ".join(missing))
+            )
+        rows = []
+        warnings = []
+        for lineno, raw_row in enumerate(reader, start=2):
+            row = {}
+            for raw_key, value in raw_row.items():
+                if raw_key in mapping:
+                    row[mapping[raw_key]] = (value or "").strip()
+            if not row.get("patient_id"):
+                warnings.append("line %d: empty patient id, skipped" % lineno)
+                continue
+            split_raw = row.get("split", "").lower()
+            row["split"] = SPLIT_ALIASES.get(split_raw, split_raw)
+            rows.append(row)
+    if not rows:
+        sys.exit("dicleak: no usable data rows in %s" % path)
+    return rows, warnings

dicleak/report.py

@@ -0,0 +1,89 @@
+"""Terminal and Markdown report rendering."""
+
+from collections import Counter
+
+from .checks import SEVERITY_ICON
+
+
+def _stats(rows):
+    patients = {r["patient_id"] for r in rows}
+    splits = Counter(r["split"] for r in rows)
+    labels = Counter(r["label"] for r in rows)
+    return {
+        "rows": len(rows),
+        "patients": len(patients),
+        "splits": dict(sorted(splits.items())),
+        "labels": dict(sorted(labels.items())),
+    }
+
+
+def _grade(score):
+    if score >= 90:
+        return "A - publishable shape"
+    if score >= 75:
+        return "B - minor issues"
+    if score >= 60:
+        return "C - fix before training"
+    return "D - do not train on this"
+
+
+def render_terminal(findings, score_value, rows):
+    st = _stats(rows)
+    lines = []
+    lines.append("=" * 60)
+    lines.append("dicleak dataset audit")
+    lines.append("=" * 60)
+    lines.append("rows: %d   patients: %d" % (st["rows"], st["patients"]))
+    lines.append("splits: %s" % ", ".join(
+        "%s=%d" % kv for kv in st["splits"].items()))
+    lines.append("labels: %s" % ", ".join(
+        "%s=%d" % kv for kv in st["labels"].items()))
+    lines.append("-" * 60)
+    lines.append("score: %d/100  [%s]" % (score_value, _grade(score_value)))
+    lines.append("-" * 60)
+    if not findings:
+        lines.append("No issues found. Clean split.")
+    for f in findings:
+        icon = SEVERITY_ICON.get(f.severity, "[?]")
+        lines.append("%s %s (-%d)" % (icon, f.title, f.deduction))
+        lines.append("    %s" % f.detail)
+    lines.append("=" * 60)
+    return "\n".join(lines)
+
+
+def render_markdown(findings, score_value, rows, cta_url=None):
+    st = _stats(rows)
+    lines = []
+    lines.append("# dicleak dataset audit report")
+    lines.append("")
+    lines.append("**Score: %d/100** - %s" % (score_value, _grade(score_value)))
+    lines.append("")
+    lines.append("## Dataset overview")
+    lines.append("")
+    lines.append("| metric | value |")
+    lines.append("|---|---|")
+    lines.append("| rows | %d |" % st["rows"])
+    lines.append("| unique patients | %d |" % st["patients"])
+    for split, n in st["splits"].items():
+        lines.append("| split `%s` | %d |" % (split, n))
+    for label, n in st["labels"].items():
+        lines.append("| label `%s` | %d |" % (label, n))
+    lines.append("")
+    lines.append("## Findings (%d)" % len(findings))
+    lines.append("")
+    if not findings:
+        lines.append("No issues found. Clean split.")
+    for f in findings:
+        lines.append("### %s %s (-%d)" % (
+            SEVERITY_ICON.get(f.severity, "[?]"), f.title, f.deduction))
+        lines.append("")
+        lines.append("Severity: **%s** | Check: `%s`" % (f.severity, f.check_id))
+        lines.append("")
+        lines.append(f.detail)
+        lines.append("")
+    if cta_url:
+        lines.append("---")
+        lines.append(
+            "Need expert eyes on your medical-imaging dataset? "
+            "[Clinical data audit service](%s)" % cta_url)
+    return "\n".join(lines)

dicleak-0.1.0.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.4
+Name: dicleak
+Version: 0.1.0
+Summary: Dataset integrity auditor for medical-imaging AI: catch patient-level data leakage and split flaws before they sink your model or paper
+Author: dicleak contributors
+License: MIT
+Keywords: medical-imaging,dicom,dataset,data-leakage,machine-learning,validation
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Dynamic: license-file
+
+# dicleak
+
+Dataset integrity auditor for medical-imaging AI. Point it at a dataset manifest
+CSV and it catches the flaws that sink models and papers **before** you train:
+patient-level data leakage across splits, single-class splits, broken
+stratification, tiny test sets, duplicate files, un-deidentified IDs.
+
+医学影像 AI 数据集完整性审计工具。对 manifest CSV 跑一遍，在训练**之前**
+揪出致命缺陷：病人级数据泄露、单类别数据划分、分布失衡、测试集过小、
+文件重复、未脱敏的 ID。
+
+## Install
+
+```bash
+pip install dicleak
+```
+
+## Usage
+
+```bash
+dicleak splits.csv
+dicleak splits.csv --report audit.md
+dicleak splits.csv --report audit.md --cta "https://your-service.example/audit"
+```
+
+The manifest CSV needs three columns (headers are flexible):
+
+| concept | accepted headers |
+|---|---|
+| patient id | `patient_id`, `patientid`, `patient`, `subject_id`, `subject`, `pid` |
+| split | `split`, `subset`, `fold`, `partition`, `set` (`train`/`val`/`test` and common aliases) |
+| label | `label`, `class`, `target`, `diagnosis`, `annotation` |
+| file path (optional) | `filepath`, `file_path`, `path`, `filename` |
+
+Exit code is `0` when the score is 60+, `1` otherwise — handy for CI gates.
+
+## The 7 checks
+
+| # | check | severity |
+|---|---|---|
+| 1 | patient-level leakage across splits | CRITICAL |
+| 2 | split containing a single class | CRITICAL |
+| 3 | label distribution drift between splits | HIGH |
+| 4 | test split < 10% (or any split < 5%) | MEDIUM |
+| 5 | duplicate files in the manifest | MEDIUM |
+| 6 | patient IDs that look like real names | HIGH |
+| 7 | too few unique patients | LOW/MEDIUM |
+
+## Why this exists
+
+A disturbing share of medical-AI papers report inflated accuracy because the
+same patient ended up in both the training and test sets. Reviewers now check
+for this. `dicleak` makes the check a one-liner.
+
+## Sample
+
+`samples/flawed_splits.csv` is a deliberately broken manifest (leaked patients,
+single-class val split, tiny test set, name-like IDs...). It scores 26/100:
+
+```bash
+dicleak samples/flawed_splits.csv
+```
+
+## License
+
+MIT

dicleak-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+dicleak/__init__.py,sha256=Z58sFiJBCSs5i2Hlkf3Kmk1OLDnkwM6Kjr6NNdCwzps,88
+dicleak/checks.py,sha256=jzBPbCZVXgFCq6NjU-CdkccOuHC1pSH-ncoQKb1UZew,7398
+dicleak/cli.py,sha256=maRKzU9krDaJgH7VOFza-THLEaMFeK1UHFJIEEM6FZU,1462
+dicleak/readers.py,sha256=iyM6jYJZGHPXGolOjjc6WQUrbKEpDBypQPBuU9S6Oiw,2794
+dicleak/report.py,sha256=twRJgZMgJfdCrnZLXqOqiUd4inrdqHXgeZasqw4M58A,3012
+dicleak-0.1.0.dist-info/licenses/LICENSE,sha256=0c9Ek3JRHnBVjyvmVPPLUif7Et7T8OqimjItkx0nDxE,1078
+dicleak-0.1.0.dist-info/METADATA,sha256=zgGuPG0kLb-0ZI-QyFCBGhCgyO2QBYSzITIgrheU1pk,2623
+dicleak-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+dicleak-0.1.0.dist-info/entry_points.txt,sha256=ZkDhmsZ3nAYyyOIvPAo8TlP04XK4QAEJBM4VodYNd3I,45
+dicleak-0.1.0.dist-info/top_level.txt,sha256=ptyYTc2g-qU83RaxQgnNvsHceE9fkhINTY1kMiOb0jM,8
+dicleak-0.1.0.dist-info/RECORD,,

dicleak-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dicleak-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dicleak = dicleak.cli:main

dicleak-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 manulint contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dicleak-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+dicleak