cvic 1.2.2__tar.gz → 1.2.4__tar.gz

{cvic-1.2.2 → cvic-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: cvic
-Version: 1.2.2
+Version: 1.2.4
 Summary: Local hyperparameter optimization of image classifiers using cross-validation + Ray Tune + timm
 Project-URL: Homepage, https://github.com/ljbuturovic/cvic
 Project-URL: Repository, https://github.com/ljbuturovic/cvic

{cvic-1.2.2 → cvic-1.2.4}/cvic/tunic.py

@@ -156,67 +156,92 @@ def _split_by_label(by_label, training_fraction: float, val_fraction: float,
     With item_group=None this is a plain per-item split (frozen by
     tests/test_split_golden.py). When item_group is given (a mapping item ->
     group id), whole groups are assigned atomically so no group is ever split
-    across train/val — the leakage protection for grouped datasets like BHAM,
-    where one lesion contributes several images. Group-aware splitting requires
-    class-pure groups (all of a group's items share one label); a group that
-    spans classes raises ValueError rather than leaking silently.
+    across train/val — the leakage protection for grouped datasets like BHAM
+    (one lesion -> several images) and bret (one patient -> two eyes).
+
+    Groups MAY span classes: a bret patient can have one referable eye and one
+    non-referable eye, so a patient group carries both labels. Such a group is
+    assigned by its majority item label (ties broken toward the larger label
+    index, which keeps the rarer positive class populated for stratification),
+    and all of its items travel together to the same side. Class-pure groups
+    reduce exactly to the previous per-class behavior, so the BHAM golden splits
+    in tests/test_split_golden.py stay byte-identical.
     """
     import random
-    from collections import defaultdict, OrderedDict
+    from collections import OrderedDict, Counter
     rng = random.Random(seed)
 
-    if item_group is not None:
-        group_labels: dict = defaultdict(set)
-        for label in by_label:
-            for it in by_label[label]:
-                group_labels[item_group[it]].add(label)
-        straddling = [g for g, labs in group_labels.items() if len(labs) > 1]
-        if straddling:
-            raise ValueError(
-                f"{len(straddling)} group(s) span multiple classes; stratified "
-                "group-aware holdout requires class-pure groups (e.g. one lesion "
-                "-> one label). Dedupe or use a non-stratified split.")
-
-    train_items: list = []
-    val_items: list = []
-    dist: dict = {}
+    if item_group is None:
+        # Plain per-item stratified split (frozen by tests/test_split_golden.py).
+        train_items: list = []
+        val_items: list = []
+        dist: dict = {}
+        for label in sorted(by_label):
+            items = list(by_label[label])
+            n_total = len(items)
+            n_tr = max(1, int(round(n_total * training_fraction)))
+            n_va = max(1, int(round(n_total * val_fraction)))
+            if n_tr + n_va > n_total:
+                n_va = max(1, n_total - n_tr)
+            rng.shuffle(items)
+            tr, va = items[:n_tr], items[n_tr:n_tr + n_va]
+            train_items.extend(tr)
+            val_items.extend(va)
+            dist[label] = {"total": n_total, "train": len(tr), "val": len(va)}
+        return train_items, val_items, dist
+
+    # Group-aware: assign whole groups atomically, stratified by a single
+    # per-group label so mixed groups never split across train/val.
+    grouped: dict = OrderedDict()      # gid -> [items across all classes], first-seen order
+    item_label: dict = {}
     for label in sorted(by_label):
-        items = list(by_label[label])
-        n_total = len(items)
+        for it in by_label[label]:
+            grouped.setdefault(item_group[it], []).append(it)
+            item_label[it] = label
+    group_strat: dict = {}
+    for gid, members in grouped.items():
+        cnt = Counter(item_label[it] for it in members)
+        # majority label; tie -> larger label index (keeps rare positives populated)
+        group_strat[gid] = max(cnt.items(), key=lambda kv: (kv[1], kv[0]))[0]
+    strata: dict = OrderedDict()       # strat label -> [gid, ...], first-seen order
+    for gid in grouped:
+        strata.setdefault(group_strat[gid], []).append(gid)
+
+    train_items = []
+    val_items = []
+    for label in sorted(strata):
+        gids = list(strata[label])
+        rng.shuffle(gids)
+        n_total = sum(len(grouped[g]) for g in gids)
         n_tr = max(1, int(round(n_total * training_fraction)))
         n_va = max(1, int(round(n_total * val_fraction)))
         if n_tr + n_va > n_total:
             n_va = max(1, n_total - n_tr)
-
-        if item_group is None:
-            rng.shuffle(items)
-            tr, va = items[:n_tr], items[n_tr:n_tr + n_va]
-        else:
-            # Group items within this class, shuffle whole groups, then fill
-            # val to its target first (so val is never empty) and train next;
-            # atomic groups mean realized counts only approximate the targets.
-            grouped: dict = OrderedDict()
-            for it in items:
-                grouped.setdefault(item_group[it], []).append(it)
-            group_ids = list(grouped.keys())
-            rng.shuffle(group_ids)
-            tr, va = [], []
-            for gid in group_ids:
-                members = grouped[gid]
-                if len(va) < n_va:
-                    va.extend(members)
-                elif len(tr) < n_tr:
-                    tr.extend(members)
-                # else: held-out remainder (matches the fractional design)
-            if not tr and len(group_ids) >= 2:
-                # degenerate: val absorbed everything — hand the last group to train
-                moved = grouped[group_ids[-1]]
-                va = [x for x in va if x not in set(moved)]
-                tr = list(moved)
-
+        tr, va = [], []
+        for gid in gids:
+            members = grouped[gid]
+            if len(va) < n_va:
+                va.extend(members)
+            elif len(tr) < n_tr:
+                tr.extend(members)
+            # else: held-out remainder (matches the fractional design)
+        if not tr and len(gids) >= 2:
+            # degenerate: val absorbed everything — hand the last group to train
+            moved = grouped[gids[-1]]
+            mv = set(moved)
+            va = [x for x in va if x not in mv]
+            tr = list(moved)
         train_items.extend(tr)
         val_items.extend(va)
-        dist[label] = {"total": n_total, "train": len(tr), "val": len(va)}
+
+    # Report distribution by TRUE per-item label (mixed groups contribute to both).
+    tr_set, va_set = set(train_items), set(val_items)
+    dist = {}
+    for label in sorted(by_label):
+        items = by_label[label]
+        dist[label] = {"total": len(items),
+                       "train": sum(1 for it in items if it in tr_set),
+                       "val": sum(1 for it in items if it in va_set)}
     return train_items, val_items, dist
 
 

{cvic-1.2.2 → cvic-1.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "cvic"
-version = "1.2.2"
+version = "1.2.4"
 description = "Local hyperparameter optimization of image classifiers using cross-validation + Ray Tune + timm"
 readme = "README.md"
 requires-python = ">=3.12"

{cvic-1.2.2 → cvic-1.2.4}/tests/test_holdout_grouping.py

@@ -65,13 +65,26 @@ def test_grouped_holdout_reproducible_and_seed_sensitive():
     assert (a[0], a[1]) != (c[0], c[1]), "different seed must give a different split"
 
 
-def test_grouped_holdout_rejects_class_straddling_group():
-    """A group whose items span two classes must raise, not silently leak."""
+def test_grouped_holdout_keeps_class_straddling_group_whole():
+    """A group whose items span two classes is assigned atomically, not rejected.
+
+    (Previously this raised ValueError; bret patients can have one referable and
+    one non-referable eye, so a group legitimately carries both labels. The
+    contract is now: keep the whole group on one side — never leak it, never
+    drop it.)
+    """
     by_label = {0: ["x0", "x1"], 1: ["x2", "x3"]}
     # 'x1' and 'x2' share a group id but sit in different classes.
     item_group = {"x0": "g0", "x1": "gX", "x2": "gX", "x3": "g1"}
-    with pytest.raises(ValueError, match="span multiple classes"):
-        _split_by_label(by_label, 0.8, 0.2, seed=0, item_group=item_group)
+    for seed in range(6):
+        train, val, _ = _split_by_label(by_label, 0.8, 0.2, seed=seed, item_group=item_group)
+        # no group straddles train/val
+        assert {item_group[it] for it in train}.isdisjoint({item_group[it] for it in val})
+        # the mixed group's two items land together
+        assert ("x1" in train) == ("x2" in train)
+        assert ("x1" in val) == ("x2" in val)
+        # nothing lost or duplicated
+        assert set(train).isdisjoint(val)
 
 
 def test_ungrouped_still_matches_when_item_group_none():

cvic-1.2.4/tests/test_holdout_mixed_groups.py

@@ -0,0 +1,132 @@
+"""tunic group-aware hold-out must tolerate label-mixed groups (bret patients).
+
+bret patients can have one referable eye and one non-referable eye, so a patient
+group legitimately spans both classes. tunic's group-aware hold-out splitter
+(_split_by_label with item_group) used to REJECT such groups with a ValueError
+("group(s) span multiple classes"); it now assigns the whole group atomically by
+its majority label. These tests pin the new contract:
+
+  * a mixed group no longer raises, and is never split across train/val;
+  * NO group (mixed or pure) is ever split across train/val (leakage guard);
+  * class-pure grouped splits are unchanged (delegated to test_split_golden.py,
+    re-asserted here as a no-group-split invariant);
+  * the split is deterministic for a fixed seed.
+"""
+
+from collections import defaultdict
+
+import pytest
+
+from cvic.tunic import _split_by_label
+
+
+def _group_of(item_group, item):
+    return item_group[item]
+
+
+def _assert_no_group_split(train, val, item_group):
+    """No group id may appear on both sides."""
+    tg = {item_group[i] for i in train}
+    vg = {item_group[i] for i in val}
+    overlap = tg & vg
+    assert not overlap, f"group(s) split across train/val: {sorted(overlap)}"
+
+
+# --------------------------------------------------------------------------- #
+# mixed groups
+# --------------------------------------------------------------------------- #
+
+def test_mixed_group_does_not_raise_and_is_not_split():
+    # patient p_mix has one negative (0) and one positive (1) eye
+    by_label = {
+        0: ["p1_l", "p1_r", "p2_l", "p2_r", "p3_l", "p3_r", "pmix_neg"],
+        1: ["p4_l", "p4_r", "p5_l", "p5_r", "pmix_pos"],
+    }
+    item_group = {
+        "p1_l": "p1", "p1_r": "p1", "p2_l": "p2", "p2_r": "p2",
+        "p3_l": "p3", "p3_r": "p3", "p4_l": "p4", "p4_r": "p4",
+        "p5_l": "p5", "p5_r": "p5",
+        "pmix_neg": "pmix", "pmix_pos": "pmix",   # one group, both labels
+    }
+    # must not raise (the old behavior raised ValueError here)
+    train, val, dist = _split_by_label(by_label, 0.8, 0.2, seed=0, item_group=item_group)
+    _assert_no_group_split(train, val, item_group)
+    # both eyes of the mixed patient land together
+    assert ("pmix_neg" in train) == ("pmix_pos" in train)
+    assert ("pmix_neg" in val) == ("pmix_pos" in val)
+    # dist reports by true per-item label and totals are preserved
+    assert dist[0]["total"] == 7 and dist[1]["total"] == 5
+    assert dist[0]["train"] + dist[0]["val"] <= 7
+    # every placed item is accounted for once
+    assert len(set(train) & set(val)) == 0
+
+
+def test_mixed_group_assigned_by_majority_then_larger_label():
+    # pmaj: two negatives + one positive -> majority label 0
+    # ptie: one negative + one positive -> tie -> larger label index (1)
+    by_label = {
+        0: ["a", "b", "pmaj_n1", "pmaj_n2", "ptie_n"],
+        1: ["c", "d", "pmaj_p1", "ptie_p"],
+    }
+    item_group = {
+        "a": "a", "b": "b", "c": "c", "d": "d",
+        "pmaj_n1": "pmaj", "pmaj_n2": "pmaj", "pmaj_p1": "pmaj",
+        "ptie_n": "ptie", "ptie_p": "ptie",
+    }
+    # Just assert it runs, keeps groups whole, and is leakage-free across a few seeds.
+    for seed in range(5):
+        train, val, _ = _split_by_label(by_label, 0.7, 0.3, seed=seed, item_group=item_group)
+        _assert_no_group_split(train, val, item_group)
+        assert ("pmaj_n1" in train) == ("pmaj_p1" in train)   # group whole
+        assert ("ptie_n" in train) == ("ptie_p" in train)
+
+
+def test_many_mixed_groups_never_leak():
+    # 30 patients, every 3rd one label-mixed; both eyes present
+    by_label = defaultdict(list)
+    item_group = {}
+    for p in range(30):
+        l, r = f"p{p}_l", f"p{p}_r"
+        item_group[l] = item_group[r] = f"p{p}"
+        if p % 3 == 0:                 # mixed
+            by_label[0].append(l)
+            by_label[1].append(r)
+        elif p % 3 == 1:               # pure negative
+            by_label[0].extend([l, r])
+        else:                          # pure positive
+            by_label[1].extend([l, r])
+    by_label = dict(by_label)
+    train, val, _ = _split_by_label(by_label, 0.8, 0.2, seed=42, item_group=item_group)
+    _assert_no_group_split(train, val, item_group)
+    # nothing dropped-then-duplicated
+    assert len(set(train)) == len(train) and len(set(val)) == len(val)
+
+
+# --------------------------------------------------------------------------- #
+# pure groups: unchanged invariants
+# --------------------------------------------------------------------------- #
+
+def test_pure_groups_no_split_and_deterministic():
+    by_label = {
+        0: [f"n{i}_{e}" for i in range(8) for e in ("l", "r")],
+        1: [f"p{i}_{e}" for i in range(4) for e in ("l", "r")],
+    }
+    item_group = {}
+    for i in range(8):
+        for e in ("l", "r"):
+            item_group[f"n{i}_{e}"] = f"n{i}"
+    for i in range(4):
+        for e in ("l", "r"):
+            item_group[f"p{i}_{e}"] = f"p{i}"
+    a = _split_by_label(by_label, 0.8, 0.2, seed=7, item_group=item_group)
+    b = _split_by_label(by_label, 0.8, 0.2, seed=7, item_group=item_group)
+    assert a == b                                   # deterministic
+    _assert_no_group_split(a[0], a[1], item_group)
+
+
+def test_ungrouped_path_unaffected():
+    """item_group=None still does a plain per-item split (golden path)."""
+    by_label = {0: list(range(20)), 1: list(range(100, 110))}
+    train, val, dist = _split_by_label(by_label, 0.8, 0.2, seed=1, item_group=None)
+    assert set(train) & set(val) == set()
+    assert dist[0]["total"] == 20 and dist[1]["total"] == 10

{cvic-1.2.2 → cvic-1.2.4}/uv.lock

@@ -322,7 +322,7 @@ nvtx = [
 
 [[package]]
 name = "cvic"
-version = "1.2.2"
+version = "1.2.4"
 source = { editable = "." }
 dependencies = [
     { name = "matplotlib" },