cnn-classifier-toolkit 0.1.0__py3-none-any.whl

cnn_classifier/core/seeding.py

@@ -0,0 +1,37 @@
+"""Deterministic seeding helpers."""
+
+from __future__ import annotations
+
+import contextlib
+import random
+
+import numpy as np
+import torch
+
+
+def seed_everything(seed: int, deterministic: bool = True) -> None:
+    """Seed Python, NumPy, and PyTorch, and request deterministic kernels.
+
+    MPS does not guarantee bit-identical results across versions or devices even
+    with a fixed seed; this function makes runs repeatable within one environment
+    as far as the backend allows.
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+    if deterministic:
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+        with contextlib.suppress(RuntimeError):
+            # Some backends do not expose a deterministic implementation for
+            # every op. The run metadata records the requested policy.
+            torch.use_deterministic_algorithms(True, warn_only=True)
+
+
+def make_generator(seed: int) -> torch.Generator:
+    """Return a seeded CPU ``torch.Generator``."""
+    generator = torch.Generator()
+    generator.manual_seed(seed)
+    return generator

cnn_classifier/core/types.py

@@ -0,0 +1,97 @@
+"""Shared data contracts exchanged between adapters, models, and runs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from torch.utils.data import Dataset
+
+from cnn_classifier.core.errors import DataError
+
+DatasetLike = Dataset[Any]
+
+
+@dataclass(frozen=True)
+class SampleRecord:
+    path: Path
+    label: str
+    split: Literal["train", "val", "test"]
+    group: str | None = None
+    severity: float | None = None
+
+
+@dataclass(frozen=True)
+class SampleMetadata:
+    paths: tuple[Path, ...]
+    labels: tuple[int, ...]
+    groups: tuple[str | None, ...]
+    source_indices: tuple[int, ...]
+
+
+@dataclass(frozen=True)
+class NormalizationStats:
+    mean: tuple[float, ...]
+    std: tuple[float, ...]
+    source: Literal["computed", "supplied", "none"]
+
+
+@dataclass(frozen=True)
+class ModelInputContract:
+    input_shape: tuple[int, int, int]
+    dtype: Literal["float32"]
+    value_range: tuple[float, float]
+    normalization: NormalizationStats | None
+    class_names: tuple[str, ...]
+    label_map: dict[str, int]
+    channel_policy: Literal["rgb", "gray", "as_is"]
+    stem_adaptation: str | None
+
+
+@dataclass(frozen=True)
+class SplitInfo:
+    mode: Literal["random", "random_stratified", "group", "explicit", "train_only"]
+    train_indices: tuple[int, ...]
+    val_indices: tuple[int, ...]
+    test_indices: tuple[int, ...] | None
+    seed: int
+    group_column: str | None
+    fingerprint: str
+
+    def __post_init__(self) -> None:
+        all_indices = [*self.train_indices, *self.val_indices]
+        if self.test_indices is not None:
+            all_indices.extend(self.test_indices)
+        if len(all_indices) != len(set(all_indices)):
+            raise DataError("train, val, and test indices overlap")
+        if any(index < 0 for index in all_indices):
+            raise DataError("split indices must be non-negative")
+        if not self.fingerprint:
+            raise DataError("split fingerprint must not be empty")
+
+
+@dataclass(frozen=True)
+class DatasetBundle:
+    train: DatasetLike
+    val: DatasetLike
+    test: DatasetLike | None
+    class_names: tuple[str, ...]
+    in_channels: int
+    image_size: tuple[int, int]
+    normalization: NormalizationStats | None
+    split_info: SplitInfo
+    fingerprint: str
+    sample_metadata: SampleMetadata
+    model_contract: ModelInputContract
+    test_fingerprint: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.class_names:
+            raise DataError("class_names must not be empty")
+        if len(set(self.class_names)) != len(self.class_names):
+            raise DataError("class_names must be unique")
+        if self.in_channels <= 0:
+            raise DataError("in_channels must be positive")
+        if self.image_size[0] <= 0 or self.image_size[1] <= 0:
+            raise DataError("image_size must be positive")

cnn_classifier/data/__init__.py

@@ -0,0 +1,22 @@
+"""Reusable data layer for dataset adapters, splits, and image contracts."""
+
+from cnn_classifier.data.base import DatasetAdapter
+from cnn_classifier.data.manifest import Manifest, fingerprint_records
+from cnn_classifier.data.splits import (
+    SplitIndices,
+    assert_no_group_overlap,
+    group_split_indices,
+    random_split_indices,
+    stratified_split_indices,
+)
+
+__all__ = [
+    "DatasetAdapter",
+    "Manifest",
+    "SplitIndices",
+    "assert_no_group_overlap",
+    "fingerprint_records",
+    "group_split_indices",
+    "random_split_indices",
+    "stratified_split_indices",
+]

cnn_classifier/data/adapters/__init__.py

@@ -0,0 +1,19 @@
+"""Dataset adapter registry and public builders."""
+
+from cnn_classifier.data.adapters.image_folder import ImageFolderAdapter
+from cnn_classifier.data.adapters.manifest import ManifestAdapter
+from cnn_classifier.data.adapters.numpy import NumpyAdapter
+from cnn_classifier.data.adapters.registry import DATASETS, build_dataset, register_dataset
+from cnn_classifier.data.adapters.synthetic import SyntheticAdapter
+from cnn_classifier.data.adapters.torchvision import TorchvisionAdapter
+
+__all__ = [
+    "DATASETS",
+    "ImageFolderAdapter",
+    "ManifestAdapter",
+    "NumpyAdapter",
+    "SyntheticAdapter",
+    "TorchvisionAdapter",
+    "build_dataset",
+    "register_dataset",
+]

cnn_classifier/data/adapters/_common.py

@@ -0,0 +1,241 @@
+"""Shared dataset construction helpers used by the concrete adapters."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+import torch
+from torch.utils.data import Dataset, Subset
+
+from cnn_classifier.config.schema import DataConfig
+from cnn_classifier.core.errors import ConfigError, DataError
+from cnn_classifier.core.hashing import stable_hash
+from cnn_classifier.core.types import (
+    DatasetBundle,
+    ModelInputContract,
+    NormalizationStats,
+    SampleMetadata,
+    SplitInfo,
+)
+from cnn_classifier.data.image_io import ChannelPolicy, read_image
+from cnn_classifier.data.splits import (
+    SplitIndices,
+    assert_no_group_overlap,
+    group_split_indices,
+    random_split_indices,
+    stratified_split_indices,
+)
+from cnn_classifier.data.stats import compute_normalization
+from cnn_classifier.data.transforms import EnsureTensor, build_transforms
+
+SplitName = Literal["train", "val", "test"]
+
+
+@dataclass(frozen=True)
+class RawSample:
+    """A source image (path or tensor) and its logical label/split."""
+
+    source: Any
+    label: str
+    split: SplitName
+    group: str | None = None
+    severity: float | None = None
+
+
+class RawDataset(Dataset[tuple[torch.Tensor, int, int]]):
+    """Apply label mapping, image decoding, and an optional transform."""
+
+    def __init__(
+        self,
+        samples: Sequence[RawSample],
+        label_to_index: dict[str, int],
+        channels: ChannelPolicy,
+        value_range: Any,
+        transform: Any = None,
+    ) -> None:
+        self.samples = tuple(samples)
+        self.label_to_index = label_to_index
+        self.channels = channels
+        self.value_range = value_range
+        self.transform = transform
+
+    def __len__(self) -> int:
+        return len(self.samples)
+
+    def __getitem__(self, index: int) -> tuple[torch.Tensor, int, int]:
+        sample = self.samples[index]
+        if isinstance(sample.source, Path):
+            image = read_image(sample.source, channels=self.channels, value_range=self.value_range)
+        elif torch.is_tensor(sample.source):
+            image = sample.source.to(torch.float32)
+        else:
+            image = EnsureTensor()(sample.source)
+        if self.transform is not None:
+            image = self.transform(image)
+        return image, self.label_to_index[sample.label], index
+
+
+def resolve_class_names(
+    labels: Sequence[str],
+    config: DataConfig,
+) -> tuple[str, ...]:
+    """Resolve explicit class names or derive a stable sorted order."""
+    if isinstance(config.class_names, tuple):
+        names = config.class_names
+        if config.num_classes is not None and len(names) != config.num_classes:
+            raise ConfigError("data.class_names length must match data.num_classes")
+        missing = set(labels) - set(names)
+        if missing:
+            raise ConfigError(f"data.class_names is missing labels: {sorted(missing)}")
+        return names
+    return tuple(sorted(set(labels)))
+
+
+def assign_split(samples: Sequence[RawSample], config: DataConfig) -> tuple[SplitIndices, str]:
+    """Resolve the configured split policy into source indices."""
+    mode = config.split.mode
+    if mode == "explicit":
+        train = tuple(i for i, sample in enumerate(samples) if sample.split == "train")
+        val = tuple(i for i, sample in enumerate(samples) if sample.split == "val")
+        test = tuple(i for i, sample in enumerate(samples) if sample.split == "test")
+        return SplitIndices(train, val, test or None), "explicit"
+    labels = [sample.label for sample in samples]
+    if mode == "random":
+        return random_split_indices(labels, config.split.val_fraction, config.split.seed), "random"
+    if mode == "random_stratified":
+        return (
+            stratified_split_indices(labels, config.split.val_fraction, config.split.seed),
+            "random_stratified",
+        )
+    if mode == "group":
+        groups = [sample.group for sample in samples]
+        if any(group is None for group in groups):
+            raise DataError("group split requires a group for every sample")
+        return (
+            group_split_indices(
+                [str(group) for group in groups],
+                config.split.val_fraction,
+                config.split.seed,
+            ),
+            "group",
+        )
+    if mode == "train_only":
+        return SplitIndices(tuple(range(len(samples))), ()), "train_only"
+    raise ConfigError(f"unsupported split mode {mode!r}")
+
+
+def resolve_normalization(
+    config: DataConfig,
+    base: Dataset[Any],
+    train_indices: tuple[int, ...],
+) -> NormalizationStats | None:
+    """Resolve computed, supplied, or disabled normalization statistics."""
+    if config.normalization.mode == "none":
+        return None
+    if config.normalization.mode == "supplied":
+        if config.normalization.mean is None or config.normalization.std is None:
+            raise ConfigError("supplied normalization requires mean and std")
+        return NormalizationStats(
+            mean=config.normalization.mean,
+            std=config.normalization.std,
+            source="supplied",
+        )
+    if not train_indices:
+        raise DataError("cannot compute normalization without training samples")
+    return compute_normalization(Subset(base, train_indices))
+
+
+def build_bundle(
+    samples: Sequence[RawSample],
+    config: DataConfig,
+    *,
+    class_names: Sequence[str],
+    split: SplitIndices,
+    mode: str,
+    fingerprint: str,
+    group_column: str | None = None,
+) -> DatasetBundle:
+    """Build the final bundle, computing normalization before augmentation."""
+    names = tuple(class_names)
+    if any(sample.group is not None for sample in samples):
+        group_values = [
+            str(sample.group) if sample.group is not None else f"__ungrouped_{index}"
+            for index, sample in enumerate(samples)
+        ]
+        assert_no_group_overlap(split, group_values)
+    label_to_index = {name: index for index, name in enumerate(names)}
+    base = RawDataset(samples, label_to_index, config.channels, config.value_range)
+    normalization = resolve_normalization(config, base, split.train_indices)
+    train_base = RawDataset(
+        samples,
+        label_to_index,
+        config.channels,
+        config.value_range,
+        transform=build_transforms(config, normalization, train=True),
+    )
+    eval_base = RawDataset(
+        samples,
+        label_to_index,
+        config.channels,
+        config.value_range,
+        transform=build_transforms(config, normalization, train=False),
+    )
+    test_indices = split.test_indices
+    metadata = SampleMetadata(
+        paths=tuple(
+            sample.source if isinstance(sample.source, Path) else Path(f"<sample:{index}>")
+            for index, sample in enumerate(samples)
+        ),
+        labels=tuple(label_to_index[sample.label] for sample in samples),
+        groups=tuple(sample.group for sample in samples),
+        source_indices=tuple(range(len(samples))),
+    )
+    contract = ModelInputContract(
+        input_shape=(3 if config.channels == "rgb" else 1, *config.image_size),
+        dtype="float32",
+        value_range=(0.0, 1.0),
+        normalization=normalization,
+        class_names=names,
+        label_map=label_to_index,
+        channel_policy=config.channels,
+        stem_adaptation=None,
+    )
+    return DatasetBundle(
+        train=Subset(train_base, split.train_indices),
+        val=Subset(eval_base, split.val_indices),
+        test=Subset(eval_base, test_indices) if test_indices else None,
+        class_names=names,
+        in_channels=int(contract.input_shape[0]),
+        image_size=config.image_size,
+        normalization=normalization,
+        split_info=SplitInfo(
+            mode=mode,  # type: ignore[arg-type]
+            train_indices=split.train_indices,
+            val_indices=split.val_indices,
+            test_indices=test_indices,
+            seed=config.split.seed,
+            group_column=group_column,
+            fingerprint=fingerprint,
+        ),
+        fingerprint=fingerprint,
+        sample_metadata=metadata,
+        model_contract=contract,
+    )
+
+
+def stable_sample_fingerprint(samples: Sequence[RawSample], seed: int) -> str:
+    """Fingerprint a sample list without requiring source files to be read."""
+    payload = [
+        {
+            "source": str(sample.source) if isinstance(sample.source, Path) else "<tensor>",
+            "label": sample.label,
+            "split": sample.split,
+            "group": sample.group,
+            "severity": sample.severity,
+        }
+        for sample in samples
+    ]
+    return stable_hash({"samples": payload, "seed": seed})

cnn_classifier/data/adapters/image_folder.py

@@ -0,0 +1,65 @@
+"""ImageFolder adapter with explicit train/val/test directories."""
+
+from __future__ import annotations
+
+from cnn_classifier.config.schema import DataConfig
+from cnn_classifier.core.errors import ConfigError, DataError
+from cnn_classifier.core.types import DatasetBundle
+from cnn_classifier.data.adapters._common import (
+    RawSample,
+    assign_split,
+    build_bundle,
+    resolve_class_names,
+    stable_sample_fingerprint,
+)
+from cnn_classifier.data.adapters.registry import register_dataset
+
+_IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp", ".webp"}
+
+
+@register_dataset("image_folder")
+class ImageFolderAdapter:
+    """Read class directories from train/val/test split folders."""
+
+    @staticmethod
+    def build(config: DataConfig) -> DatasetBundle:
+        if config.root is None:
+            raise ConfigError("image_folder adapter requires data.root")
+        split_dirs = config.adapter_params.get(
+            "split_dirs", {"train": "train", "val": "val", "test": "test"}
+        )
+        samples: list[RawSample] = []
+        observed_classes: set[str] = set()
+        for split_name in ("train", "val", "test"):
+            directory = config.root / split_dirs[split_name]
+            if not directory.is_dir():
+                if split_name == "test":
+                    continue
+                raise DataError(f"image_folder split directory does not exist: {directory}")
+            for class_dir in sorted(path for path in directory.iterdir() if path.is_dir()):
+                observed_classes.add(class_dir.name)
+                for image_path in sorted(class_dir.iterdir()):
+                    if image_path.is_file() and image_path.suffix.lower() in _IMAGE_SUFFIXES:
+                        samples.append(
+                            RawSample(
+                                source=image_path,
+                                label=class_dir.name,
+                                split=split_name,
+                            )
+                        )
+        if not samples:
+            raise DataError(f"image_folder found no images under {config.root}")
+        class_names = resolve_class_names(sorted(observed_classes), config)
+        explicit = config.model_copy(
+            update={"split": config.split.model_copy(update={"mode": "explicit"})}
+        )
+        split, mode = assign_split(samples, explicit)
+        fingerprint = stable_sample_fingerprint(samples, config.split.seed)
+        return build_bundle(
+            samples,
+            explicit,
+            class_names=class_names,
+            split=split,
+            mode=mode,
+            fingerprint=fingerprint,
+        )

cnn_classifier/data/adapters/manifest.py

@@ -0,0 +1,62 @@
+"""CSV manifest adapter."""
+
+from __future__ import annotations
+
+from cnn_classifier.config.schema import DataConfig
+from cnn_classifier.core.errors import ConfigError
+from cnn_classifier.core.types import DatasetBundle
+from cnn_classifier.data.adapters._common import (
+    RawSample,
+    assign_split,
+    build_bundle,
+    resolve_class_names,
+)
+from cnn_classifier.data.adapters.registry import register_dataset
+from cnn_classifier.data.manifest import Manifest, fingerprint_records
+
+
+@register_dataset("manifest")
+class ManifestAdapter:
+    """Read images and labels from a CSV manifest."""
+
+    @staticmethod
+    def build(config: DataConfig) -> DatasetBundle:
+        if config.manifest is None:
+            raise ConfigError("manifest adapter requires data.manifest")
+        image_root = config.image_root or config.manifest.parent
+        columns = config.adapter_params.get(
+            "columns",
+            {
+                "path": "path",
+                "label": "label",
+                "split": "split",
+                "group": "group",
+                "severity": "severity",
+            },
+        )
+        manifest = Manifest.from_csv(config.manifest, image_root, columns)
+        samples = [
+            RawSample(
+                source=record.path,
+                label=record.label,
+                split=record.split,
+                group=record.group,
+                severity=record.severity,
+            )
+            for record in manifest.records
+        ]
+        class_names = resolve_class_names([sample.label for sample in samples], config)
+        explicit = config.model_copy(
+            update={"split": config.split.model_copy(update={"mode": "explicit"})}
+        )
+        split, mode = assign_split(samples, explicit)
+        fingerprint = fingerprint_records(manifest.records)
+        return build_bundle(
+            samples,
+            explicit,
+            class_names=class_names,
+            split=split,
+            mode=mode,
+            fingerprint=fingerprint,
+            group_column=config.split.group_column,
+        )

cnn_classifier/data/adapters/numpy.py

@@ -0,0 +1,96 @@
+"""NumPy array adapter for scientific image tensors."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from cnn_classifier.config.schema import DataConfig
+from cnn_classifier.core.errors import ConfigError, DataError
+from cnn_classifier.core.hashing import stable_hash
+from cnn_classifier.core.types import DatasetBundle
+from cnn_classifier.data.adapters._common import (
+    RawSample,
+    assign_split,
+    build_bundle,
+    resolve_class_names,
+)
+from cnn_classifier.data.adapters.registry import register_dataset
+
+
+def _load_arrays(array_path: Path, labels_path: Path | None) -> tuple[np.ndarray, np.ndarray]:
+    try:
+        if array_path.suffix == ".npz":
+            payload = np.load(array_path)
+            images = np.asarray(payload["images"])
+            labels = np.asarray(payload["labels"] if labels_path is None else np.load(labels_path))
+        else:
+            images = np.asarray(np.load(array_path))
+            if labels_path is None:
+                raise ConfigError("numpy adapter requires labels_path unless the array is .npz")
+            labels = np.asarray(np.load(labels_path))
+    except OSError as exc:
+        raise DataError(f"cannot load NumPy dataset: {exc}") from exc
+    if images.ndim == 3:
+        images = images[:, None, :, :]
+    if images.ndim != 4:
+        raise DataError(f"numpy images must be [N,C,H,W] or [N,H,W]; got {images.shape}")
+    if len(images) != len(labels):
+        raise DataError("numpy images and labels have different lengths")
+    return images, labels
+
+
+def _to_float(images: np.ndarray) -> np.ndarray:
+    data = images.astype(np.float64)
+    if np.issubdtype(images.dtype, np.integer):
+        data = data / float(np.iinfo(images.dtype).max)
+    return np.clip(data, 0.0, 1.0).astype(np.float32)
+
+
+@register_dataset("numpy")
+class NumpyAdapter:
+    """Read images and labels from .npy or .npz arrays."""
+
+    @staticmethod
+    def build(config: DataConfig) -> DatasetBundle:
+        raw_path = config.adapter_params.get("array_path")
+        if raw_path is None:
+            raise ConfigError("numpy adapter requires adapter_params.array_path")
+        array_path = Path(raw_path)
+        if not array_path.is_absolute() and config.root is not None:
+            array_path = config.root / array_path
+        labels_raw = config.adapter_params.get("labels_path")
+        labels_path = Path(labels_raw) if labels_raw is not None else None
+        if labels_path is not None and not labels_path.is_absolute() and config.root is not None:
+            labels_path = config.root / labels_path
+        images, labels = _load_arrays(array_path, labels_path)
+        images = _to_float(images)
+        class_values = sorted({int(value) for value in labels.tolist()})
+        class_names = resolve_class_names([f"class_{value}" for value in class_values], config)
+        samples = [
+            RawSample(
+                source=torch.from_numpy(images[index]),
+                label=f"class_{int(labels[index])}",
+                split="train",
+            )
+            for index in range(len(images))
+        ]
+        split, mode = assign_split(samples, config)
+        fingerprint = stable_hash(
+            {
+                "adapter": "numpy",
+                "shape": list(images.shape),
+                "labels": labels.tolist(),
+                "seed": config.split.seed,
+            }
+        )
+        return build_bundle(
+            samples,
+            config,
+            class_names=class_names,
+            split=split,
+            mode=mode,
+            fingerprint=fingerprint,
+        )

cnn_classifier/data/adapters/registry.py

@@ -0,0 +1,29 @@
+"""Dataset adapter registry."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TypeAlias
+
+from cnn_classifier.config.schema import DataConfig
+from cnn_classifier.core.errors import ConfigError, RegistryError
+from cnn_classifier.core.registry import Registry
+from cnn_classifier.core.types import DatasetBundle
+from cnn_classifier.data.base import DatasetAdapter
+
+AdapterClass: TypeAlias = type[DatasetAdapter]
+DATASETS: Registry[AdapterClass] = Registry("dataset")
+
+
+def register_dataset(name: str) -> Callable[[AdapterClass], AdapterClass]:
+    """Register a dataset adapter class under ``name``."""
+    return DATASETS.register(name)
+
+
+def build_dataset(config: DataConfig) -> DatasetBundle:
+    """Resolve and build the configured dataset adapter."""
+    try:
+        adapter = DATASETS.get(config.adapter)
+    except RegistryError as exc:
+        raise ConfigError(f"unknown dataset adapter {config.adapter!r}") from exc
+    return adapter.build(config)