cnn-classifier-toolkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. cnn_classifier/__init__.py +5 -0
  2. cnn_classifier/_version.py +1 -0
  3. cnn_classifier/cli.py +1085 -0
  4. cnn_classifier/config/__init__.py +17 -0
  5. cnn_classifier/config/loader.py +102 -0
  6. cnn_classifier/config/presets/default.yaml +47 -0
  7. cnn_classifier/config/resolve.py +108 -0
  8. cnn_classifier/config/schema.py +251 -0
  9. cnn_classifier/core/__init__.py +19 -0
  10. cnn_classifier/core/device.py +50 -0
  11. cnn_classifier/core/errors.py +21 -0
  12. cnn_classifier/core/hashing.py +18 -0
  13. cnn_classifier/core/registry.py +46 -0
  14. cnn_classifier/core/seeding.py +37 -0
  15. cnn_classifier/core/types.py +97 -0
  16. cnn_classifier/data/__init__.py +22 -0
  17. cnn_classifier/data/adapters/__init__.py +19 -0
  18. cnn_classifier/data/adapters/_common.py +241 -0
  19. cnn_classifier/data/adapters/image_folder.py +65 -0
  20. cnn_classifier/data/adapters/manifest.py +62 -0
  21. cnn_classifier/data/adapters/numpy.py +96 -0
  22. cnn_classifier/data/adapters/registry.py +29 -0
  23. cnn_classifier/data/adapters/synthetic.py +114 -0
  24. cnn_classifier/data/adapters/torchvision.py +192 -0
  25. cnn_classifier/data/base.py +17 -0
  26. cnn_classifier/data/contract.py +28 -0
  27. cnn_classifier/data/image_io.py +116 -0
  28. cnn_classifier/data/inspection.py +111 -0
  29. cnn_classifier/data/loaders.py +79 -0
  30. cnn_classifier/data/manifest.py +98 -0
  31. cnn_classifier/data/splits.py +136 -0
  32. cnn_classifier/data/stats.py +106 -0
  33. cnn_classifier/data/transforms.py +192 -0
  34. cnn_classifier/diagnostics/__init__.py +22 -0
  35. cnn_classifier/diagnostics/activations.py +56 -0
  36. cnn_classifier/diagnostics/architecture.py +172 -0
  37. cnn_classifier/diagnostics/determinism.py +25 -0
  38. cnn_classifier/diagnostics/gradients.py +32 -0
  39. cnn_classifier/diagnostics/sanity.py +49 -0
  40. cnn_classifier/diagnostics/tracing.py +56 -0
  41. cnn_classifier/evaluation/__init__.py +53 -0
  42. cnn_classifier/evaluation/bootstrap.py +96 -0
  43. cnn_classifier/evaluation/calibration.py +74 -0
  44. cnn_classifier/evaluation/compare.py +111 -0
  45. cnn_classifier/evaluation/error_analysis.py +61 -0
  46. cnn_classifier/evaluation/evaluator.py +143 -0
  47. cnn_classifier/evaluation/latency.py +58 -0
  48. cnn_classifier/evaluation/metrics.py +94 -0
  49. cnn_classifier/evaluation/predict.py +64 -0
  50. cnn_classifier/models/__init__.py +50 -0
  51. cnn_classifier/models/base.py +62 -0
  52. cnn_classifier/models/residual_cnn.py +138 -0
  53. cnn_classifier/models/simple_cnn.py +55 -0
  54. cnn_classifier/models/summary.py +39 -0
  55. cnn_classifier/models/torchvision.py +46 -0
  56. cnn_classifier/models/transfer.py +193 -0
  57. cnn_classifier/runs/__init__.py +6 -0
  58. cnn_classifier/runs/artifacts.py +50 -0
  59. cnn_classifier/runs/context.py +130 -0
  60. cnn_classifier/tracking/__init__.py +34 -0
  61. cnn_classifier/tracking/base.py +16 -0
  62. cnn_classifier/tracking/composite.py +20 -0
  63. cnn_classifier/tracking/console.py +14 -0
  64. cnn_classifier/tracking/csv.py +19 -0
  65. cnn_classifier/tracking/tensorboard.py +22 -0
  66. cnn_classifier/training/__init__.py +37 -0
  67. cnn_classifier/training/amp.py +51 -0
  68. cnn_classifier/training/callbacks.py +229 -0
  69. cnn_classifier/training/checkpoint.py +151 -0
  70. cnn_classifier/training/losses.py +67 -0
  71. cnn_classifier/training/optim.py +40 -0
  72. cnn_classifier/training/schedulers.py +53 -0
  73. cnn_classifier/training/state.py +35 -0
  74. cnn_classifier/training/trainer.py +357 -0
  75. cnn_classifier/viz/__init__.py +42 -0
  76. cnn_classifier/viz/architecture.py +276 -0
  77. cnn_classifier/viz/compare.py +36 -0
  78. cnn_classifier/viz/data.py +56 -0
  79. cnn_classifier/viz/evaluation.py +134 -0
  80. cnn_classifier/viz/features.py +122 -0
  81. cnn_classifier/viz/report.py +821 -0
  82. cnn_classifier/viz/style.py +69 -0
  83. cnn_classifier/viz/templates/report.html.j2 +355 -0
  84. cnn_classifier/viz/training.py +83 -0
  85. cnn_classifier_toolkit-0.1.0.dist-info/METADATA +249 -0
  86. cnn_classifier_toolkit-0.1.0.dist-info/RECORD +89 -0
  87. cnn_classifier_toolkit-0.1.0.dist-info/WHEEL +4 -0
  88. cnn_classifier_toolkit-0.1.0.dist-info/entry_points.txt +2 -0
  89. cnn_classifier_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
@@ -0,0 +1,114 @@
1
+ """Deterministic synthetic dataset adapter for tests and smoke runs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import defaultdict
6
+ from collections.abc import Sequence
7
+
8
+ import numpy as np
9
+ import torch
10
+
11
+ from cnn_classifier.config.schema import DataConfig
12
+ from cnn_classifier.core.errors import ConfigError
13
+ from cnn_classifier.core.hashing import stable_hash
14
+ from cnn_classifier.core.types import DatasetBundle
15
+ from cnn_classifier.data.adapters._common import (
16
+ RawSample,
17
+ assign_split,
18
+ build_bundle,
19
+ )
20
+ from cnn_classifier.data.adapters.registry import register_dataset
21
+ from cnn_classifier.data.splits import SplitIndices
22
+
23
+
24
+ def _three_way_split(
25
+ labels: Sequence[str],
26
+ val_fraction: float,
27
+ test_fraction: float,
28
+ seed: int,
29
+ ) -> SplitIndices:
30
+ if not 0.0 <= test_fraction < 1.0:
31
+ raise ConfigError("synthetic test_fraction must be in [0, 1)")
32
+ if test_fraction + val_fraction >= 1.0:
33
+ raise ConfigError("synthetic test_fraction + val_fraction must be smaller than 1")
34
+ by_class: dict[str, list[int]] = defaultdict(list)
35
+ for index, label in enumerate(labels):
36
+ by_class[label].append(index)
37
+ rng = np.random.default_rng(seed)
38
+ train: list[int] = []
39
+ val: list[int] = []
40
+ test: list[int] = []
41
+ for label in sorted(by_class):
42
+ indices = np.array(by_class[label])
43
+ shuffled = indices[rng.permutation(len(indices))].tolist()
44
+ test_count = round(len(shuffled) * test_fraction)
45
+ val_count = round(len(shuffled) * val_fraction)
46
+ if len(shuffled) >= 3:
47
+ test_count = max(1, test_count)
48
+ val_count = max(1, val_count)
49
+ test.extend(shuffled[:test_count])
50
+ val.extend(shuffled[test_count : test_count + val_count])
51
+ train.extend(shuffled[test_count + val_count :])
52
+ return SplitIndices(tuple(sorted(train)), tuple(sorted(val)), tuple(sorted(test)))
53
+
54
+
55
+ @register_dataset("synthetic")
56
+ class SyntheticAdapter:
57
+ """Generate a tiny deterministic tensor dataset."""
58
+
59
+ @staticmethod
60
+ def build(config: DataConfig) -> DatasetBundle:
61
+ params = config.adapter_params
62
+ num_samples = int(params.get("num_samples", 64))
63
+ num_classes = int(params.get("num_classes", config.num_classes or 2))
64
+ if num_samples <= 0 or num_classes <= 0:
65
+ raise ConfigError("synthetic adapter requires positive sample and class counts")
66
+ image_size = tuple(params.get("image_size", config.image_size))
67
+ channels = 3 if config.channels == "rgb" else 1
68
+ generator = torch.Generator().manual_seed(config.split.seed)
69
+ images = torch.rand(
70
+ num_samples, channels, image_size[0], image_size[1], generator=generator
71
+ )
72
+ if isinstance(config.class_names, tuple):
73
+ class_names = config.class_names
74
+ if len(class_names) != num_classes:
75
+ raise ConfigError("data.class_names length must match synthetic num_classes")
76
+ else:
77
+ class_names = tuple(f"class_{index}" for index in range(num_classes))
78
+ labels = [class_names[index % num_classes] for index in range(num_samples)]
79
+ if config.num_classes is not None and config.num_classes != num_classes:
80
+ raise ConfigError("synthetic num_classes must match data.num_classes")
81
+ samples = [
82
+ RawSample(source=images[index], label=labels[index], split="train")
83
+ for index in range(num_samples)
84
+ ]
85
+ test_fraction = float(params.get("test_fraction", 0.0))
86
+ if test_fraction > 0:
87
+ if config.split.mode not in {"random", "random_stratified"}:
88
+ raise ConfigError(
89
+ "synthetic test_fraction requires a random or random_stratified split"
90
+ )
91
+ split = _three_way_split(
92
+ labels, config.split.val_fraction, test_fraction, config.split.seed
93
+ )
94
+ mode = "random_stratified"
95
+ else:
96
+ split, mode = assign_split(samples, config)
97
+ fingerprint = stable_hash(
98
+ {
99
+ "adapter": "synthetic",
100
+ "num_samples": num_samples,
101
+ "num_classes": num_classes,
102
+ "image_size": list(image_size),
103
+ "seed": config.split.seed,
104
+ "test_fraction": test_fraction,
105
+ }
106
+ )
107
+ return build_bundle(
108
+ samples,
109
+ config,
110
+ class_names=class_names,
111
+ split=split,
112
+ mode=mode,
113
+ fingerprint=fingerprint,
114
+ )
@@ -0,0 +1,192 @@
1
+ """Torchvision dataset adapter for CIFAR-10 and similar built-ins."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Any
7
+
8
+ import torch
9
+ import torchvision
10
+ from torch.utils.data import Dataset, Subset
11
+
12
+ from cnn_classifier.config.schema import DataConfig
13
+ from cnn_classifier.core.errors import ConfigError, DataError
14
+ from cnn_classifier.core.hashing import stable_hash
15
+ from cnn_classifier.core.types import (
16
+ DatasetBundle,
17
+ ModelInputContract,
18
+ SampleMetadata,
19
+ SplitInfo,
20
+ )
21
+ from cnn_classifier.data.adapters._common import resolve_normalization
22
+ from cnn_classifier.data.adapters.registry import register_dataset
23
+ from cnn_classifier.data.splits import (
24
+ SplitIndices,
25
+ random_split_indices,
26
+ stratified_split_indices,
27
+ )
28
+ from cnn_classifier.data.transforms import EnsureTensor, build_transforms
29
+
30
+
31
+ class _TripleDataset(Dataset[tuple[Any, int, int]]):
32
+ def __init__(self, base: Dataset[Any]) -> None:
33
+ self.base = base
34
+
35
+ def __len__(self) -> int:
36
+ return len(self.base) # type: ignore[arg-type]
37
+
38
+ def __getitem__(self, index: int) -> tuple[Any, int, int]:
39
+ image, label = self.base[index]
40
+ return image, int(label), index
41
+
42
+
43
+ class _TensorTripleDataset(Dataset[tuple[torch.Tensor, int, int]]):
44
+ def __init__(self, base: Dataset[Any]) -> None:
45
+ self.base = base
46
+
47
+ def __len__(self) -> int:
48
+ return len(self.base) # type: ignore[arg-type]
49
+
50
+ def __getitem__(self, index: int) -> tuple[torch.Tensor, int, int]:
51
+ image, label, source_index = self.base[index]
52
+ return EnsureTensor()(image), int(label), int(source_index)
53
+
54
+
55
+ class _TransformedTripleDataset(Dataset[tuple[torch.Tensor, int, int]]):
56
+ def __init__(self, base: Dataset[Any], transform: Any) -> None:
57
+ self.base = base
58
+ self.transform = transform
59
+
60
+ def __len__(self) -> int:
61
+ return len(self.base) # type: ignore[arg-type]
62
+
63
+ def __getitem__(self, index: int) -> tuple[torch.Tensor, int, int]:
64
+ image, label, source_index = self.base[index]
65
+ return self.transform(image), int(label), int(source_index)
66
+
67
+
68
+ def _dataset_targets(dataset: Dataset[Any]) -> list[int]:
69
+ targets = getattr(dataset, "targets", None)
70
+ if targets is None:
71
+ targets = getattr(dataset, "labels", None)
72
+ if targets is None:
73
+ raise DataError("torchvision dataset exposes neither targets nor labels")
74
+ return [int(value) for value in targets]
75
+
76
+
77
+ def _sample_limit(config: DataConfig, key: str) -> int | None:
78
+ value = config.adapter_params.get(key)
79
+ if value is None:
80
+ return None
81
+ if not isinstance(value, int) or isinstance(value, bool) or value <= 0:
82
+ raise ConfigError(f"adapter_params.{key} must be a positive integer")
83
+ return value
84
+
85
+
86
+ @register_dataset("torchvision")
87
+ class TorchvisionAdapter:
88
+ """Wrap a torchvision classification dataset with the package contracts."""
89
+
90
+ @staticmethod
91
+ def build(config: DataConfig) -> DatasetBundle:
92
+ if config.root is None:
93
+ raise ConfigError("torchvision adapter requires data.root")
94
+ dataset_name = str(config.adapter_params.get("dataset", "CIFAR10"))
95
+ dataset_class = getattr(torchvision.datasets, dataset_name, None)
96
+ if dataset_class is None:
97
+ raise ConfigError(f"unknown torchvision dataset {dataset_name!r}")
98
+ download = bool(config.adapter_params.get("download", True))
99
+ try:
100
+ train_base = dataset_class(root=str(config.root), train=True, download=download)
101
+ test_base = dataset_class(root=str(config.root), train=False, download=download)
102
+ except (OSError, RuntimeError) as exc:
103
+ raise DataError(f"cannot load torchvision dataset {dataset_name}: {exc}") from exc
104
+ class_names = tuple(str(name) for name in train_base.classes)
105
+ if isinstance(config.class_names, tuple) and tuple(config.class_names) != class_names:
106
+ raise ConfigError("data.class_names must match the torchvision class order")
107
+ label_map = {name: index for index, name in enumerate(class_names)}
108
+ targets = _dataset_targets(train_base)
109
+ test_targets = _dataset_targets(test_base)[: len(test_base)]
110
+ test_fingerprint = stable_hash(
111
+ {
112
+ "adapter": "torchvision",
113
+ "dataset": dataset_name,
114
+ "test_size": len(test_targets),
115
+ "test_targets": test_targets,
116
+ "class_names": class_names,
117
+ }
118
+ )
119
+ if config.split.mode == "random":
120
+ split = random_split_indices(targets, config.split.val_fraction, config.split.seed)
121
+ mode = "random"
122
+ elif config.split.mode == "random_stratified":
123
+ split = stratified_split_indices(targets, config.split.val_fraction, config.split.seed)
124
+ mode = "random_stratified"
125
+ else:
126
+ raise ConfigError("torchvision adapter supports random or random_stratified splits")
127
+ max_train = _sample_limit(config, "max_train_samples")
128
+ max_val = _sample_limit(config, "max_val_samples")
129
+ if max_train is not None or max_val is not None:
130
+ split = SplitIndices(
131
+ split.train_indices[:max_train] if max_train is not None else split.train_indices,
132
+ split.val_indices[:max_val] if max_val is not None else split.val_indices,
133
+ )
134
+ train_triple = _TripleDataset(train_base)
135
+ stats_source = _TensorTripleDataset(train_triple)
136
+ normalization = resolve_normalization(config, stats_source, split.train_indices)
137
+ train_transform = build_transforms(config, normalization, train=True)
138
+ eval_transform = build_transforms(config, normalization, train=False)
139
+ train = Subset(
140
+ _TransformedTripleDataset(train_triple, train_transform), split.train_indices
141
+ )
142
+ val = Subset(_TransformedTripleDataset(train_triple, eval_transform), split.val_indices)
143
+ test = _TransformedTripleDataset(_TripleDataset(test_base), eval_transform)
144
+ fingerprint = stable_hash(
145
+ {
146
+ "adapter": "torchvision",
147
+ "dataset": dataset_name,
148
+ "train_size": len(train_base),
149
+ "test_size": len(test_base),
150
+ "train_samples": len(split.train_indices),
151
+ "val_samples": len(split.val_indices),
152
+ "seed": config.split.seed,
153
+ }
154
+ )
155
+ contract = ModelInputContract(
156
+ input_shape=(3, *config.image_size),
157
+ dtype="float32",
158
+ value_range=(0.0, 1.0),
159
+ normalization=normalization,
160
+ class_names=class_names,
161
+ label_map=label_map,
162
+ channel_policy="rgb",
163
+ stem_adaptation=None,
164
+ )
165
+ metadata = SampleMetadata(
166
+ paths=tuple(Path(f"<torchvision:{index}>") for index in range(len(train_base))),
167
+ labels=tuple(targets),
168
+ groups=tuple(None for _ in targets),
169
+ source_indices=tuple(range(len(train_base))),
170
+ )
171
+ return DatasetBundle(
172
+ train=train,
173
+ val=val,
174
+ test=test,
175
+ class_names=class_names,
176
+ in_channels=3,
177
+ image_size=config.image_size,
178
+ normalization=normalization,
179
+ split_info=SplitInfo(
180
+ mode=mode, # type: ignore[arg-type]
181
+ train_indices=split.train_indices,
182
+ val_indices=split.val_indices,
183
+ test_indices=None,
184
+ seed=config.split.seed,
185
+ group_column=None,
186
+ fingerprint=fingerprint,
187
+ ),
188
+ fingerprint=fingerprint,
189
+ sample_metadata=metadata,
190
+ model_contract=contract,
191
+ test_fingerprint=test_fingerprint,
192
+ )
@@ -0,0 +1,17 @@
1
+ """Dataset adapter protocol and shared bundle types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Protocol
6
+
7
+ from cnn_classifier.config.schema import DataConfig
8
+ from cnn_classifier.core.types import DatasetBundle
9
+
10
+
11
+ class DatasetAdapter(Protocol):
12
+ """Builds a train/val/test bundle from a data configuration."""
13
+
14
+ @staticmethod
15
+ def build(config: DataConfig) -> DatasetBundle:
16
+ """Return the resolved datasets and their contracts."""
17
+ ...
@@ -0,0 +1,28 @@
1
+ """Model input contract derivation and serialization."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from dataclasses import asdict, replace
7
+ from pathlib import Path
8
+
9
+ from cnn_classifier.core.types import DatasetBundle, ModelInputContract
10
+
11
+
12
+ def build_model_contract(
13
+ bundle: DatasetBundle,
14
+ *,
15
+ model_name: str,
16
+ stem_adaptation: str | None = None,
17
+ ) -> ModelInputContract:
18
+ """Return the exact model-input contract for a selected model."""
19
+ _ = model_name
20
+ return replace(bundle.model_contract, stem_adaptation=stem_adaptation)
21
+
22
+
23
+ def write_model_contract(contract: ModelInputContract, path: Path) -> Path:
24
+ """Write the contract as deterministic JSON."""
25
+ path.parent.mkdir(parents=True, exist_ok=True)
26
+ payload = asdict(contract)
27
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
28
+ return path
@@ -0,0 +1,116 @@
1
+ """Explicit 8-bit and 16-bit image readers with a documented value range."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+ from typing import Literal
7
+
8
+ import numpy as np
9
+ import tifffile
10
+ import torch
11
+ from PIL import Image
12
+
13
+ from cnn_classifier.config.schema import ValueRangeConfig
14
+ from cnn_classifier.core.errors import DataError
15
+
16
+ ChannelPolicy = Literal["rgb", "gray", "as_is"]
17
+ _PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
18
+ _INTEGER_MAX = {
19
+ np.dtype("uint8"): 255.0,
20
+ np.dtype("uint16"): 65535.0,
21
+ np.dtype("uint32"): float(2**32 - 1),
22
+ np.dtype("int16"): 32767.0,
23
+ np.dtype("int32"): float(2**31 - 1),
24
+ }
25
+
26
+
27
+ def _reject_unsupported_png(path: Path) -> None:
28
+ """Reject 16-bit multichannel PNGs that Pillow would silently downcast."""
29
+ with path.open("rb") as handle:
30
+ header = handle.read(26)
31
+ if len(header) < 26 or header[:8] != _PNG_SIGNATURE:
32
+ return
33
+ if header[12:16] != b"IHDR" or int.from_bytes(header[8:12], "big") != 13:
34
+ return
35
+ bit_depth = header[24]
36
+ color_type = header[25]
37
+ if bit_depth == 16 and color_type in {2, 4, 6}:
38
+ raise DataError(
39
+ f"cannot read {path}: 16-bit RGB PNG is not supported without "
40
+ "truncation; use a 16-bit TIFF or an 8-bit PNG"
41
+ )
42
+
43
+
44
+ def _read_array(path: Path) -> np.ndarray:
45
+ suffix = path.suffix.lower()
46
+ try:
47
+ if suffix in {".tif", ".tiff"}:
48
+ return np.asarray(tifffile.imread(path))
49
+ if suffix == ".png":
50
+ _reject_unsupported_png(path)
51
+ with Image.open(path) as image:
52
+ return np.asarray(image)
53
+ except DataError:
54
+ raise
55
+ except (OSError, ValueError) as exc:
56
+ raise DataError(f"cannot read image {path}: {exc}") from exc
57
+
58
+
59
+ def _to_channels(array: np.ndarray, channels: ChannelPolicy) -> np.ndarray:
60
+ if array.ndim == 2:
61
+ array = array[:, :, None]
62
+ if array.ndim != 3:
63
+ raise DataError(f"image must be 2-D or 3-D; got shape {array.shape}")
64
+ if channels == "as_is":
65
+ return array
66
+ if channels == "gray":
67
+ if array.shape[2] == 1:
68
+ return array
69
+ if array.dtype == np.uint8:
70
+ return np.asarray(Image.fromarray(array).convert("L"), dtype=np.uint8)[:, :, None]
71
+ result: np.ndarray = array.astype(np.float64).mean(axis=2, keepdims=True)
72
+ return result
73
+ if array.shape[2] == 1:
74
+ return np.repeat(array, 3, axis=2)
75
+ if array.shape[2] < 3:
76
+ raise DataError(f"RGB image needs at least 3 channels; got {array.shape[2]}")
77
+ return array[:, :, :3]
78
+
79
+
80
+ def _scale(array: np.ndarray, value_range: ValueRangeConfig | None) -> np.ndarray:
81
+ original_dtype = array.dtype
82
+ data = array.astype(np.float64)
83
+ if value_range is not None and value_range.mode == "fixed":
84
+ if value_range.min is None or value_range.max is None:
85
+ raise DataError("fixed value_range requires min and max")
86
+ low, high = float(value_range.min), float(value_range.max)
87
+ if low >= high:
88
+ raise DataError("fixed value_range min must be smaller than max")
89
+ return np.clip((data - low) / (high - low), 0.0, 1.0)
90
+ if np.issubdtype(original_dtype, np.integer):
91
+ maximum = _INTEGER_MAX.get(original_dtype)
92
+ if maximum is None:
93
+ maximum = float(np.iinfo(original_dtype).max)
94
+ return data / maximum
95
+ return np.clip(data, 0.0, 1.0)
96
+
97
+
98
+ def read_image(
99
+ path: Path,
100
+ channels: ChannelPolicy = "rgb",
101
+ value_range: ValueRangeConfig | None = None,
102
+ ) -> torch.Tensor:
103
+ """Read an image as ``float32 [C,H,W]`` in ``[0,1]``.
104
+
105
+ The reader never silently converts a 16-bit source to 8 bits. TIFF files use
106
+ ``tifffile``; other formats use Pillow, including 16-bit PNG mode ``I;16``.
107
+ """
108
+ if not path.is_file():
109
+ raise DataError(f"image does not exist: {path}")
110
+ array = _read_array(path)
111
+ array = _to_channels(array, channels)
112
+ scaled = _scale(array, value_range)
113
+ tensor = torch.from_numpy(np.ascontiguousarray(scaled.transpose(2, 0, 1))).to(torch.float32)
114
+ if tensor.ndim != 3:
115
+ raise DataError(f"unexpected image shape after conversion: {tuple(tensor.shape)}")
116
+ return tensor
@@ -0,0 +1,111 @@
1
+ """Dataset inspection for class balance, duplicates, and leakage."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ from dataclasses import dataclass, field
7
+
8
+ import torch
9
+
10
+ from cnn_classifier.core.errors import DataError
11
+ from cnn_classifier.core.types import DatasetBundle
12
+
13
+
14
+ @dataclass(frozen=True)
15
+ class DataInspectionReport:
16
+ class_counts: dict[str, dict[str, int]] = field(default_factory=dict)
17
+ duplicates: dict[str, int] = field(default_factory=dict)
18
+ corrupt: list[str] = field(default_factory=list)
19
+ sizes: dict[str, int] = field(default_factory=dict)
20
+ split_overlap: list[str] = field(default_factory=list)
21
+ group_leakage: list[str] = field(default_factory=list)
22
+ warnings: list[str] = field(default_factory=list)
23
+
24
+
25
+ def _content_hash(image: torch.Tensor) -> str:
26
+ data = image.detach().cpu().contiguous().numpy().tobytes()
27
+ return hashlib.sha256(data).hexdigest()
28
+
29
+
30
+ def inspect_dataset(bundle: DatasetBundle, max_samples: int | None = None) -> DataInspectionReport:
31
+ """Inspect split class balance, duplicate content, and group leakage."""
32
+ class_counts: dict[str, dict[str, int]] = {}
33
+ content_hashes: dict[str, int] = {}
34
+ corrupt: list[str] = []
35
+ sizes: dict[str, int] = {}
36
+ path_sets: dict[str, set[str]] = {}
37
+ group_sets: dict[str, set[str]] = {}
38
+
39
+ splits = {"train": bundle.train, "val": bundle.val}
40
+ if bundle.test is not None:
41
+ splits["test"] = bundle.test
42
+ for split_name, dataset in splits.items():
43
+ counts: dict[str, int] = dict.fromkeys(bundle.class_names, 0)
44
+ path_sets[split_name] = set()
45
+ group_sets[split_name] = set()
46
+ limit = len(dataset) if max_samples is None else min(len(dataset), max_samples) # type: ignore[arg-type]
47
+ for index in range(limit):
48
+ source_index = -1
49
+ try:
50
+ image, label, source_index = dataset[index]
51
+ except (DataError, OSError, ValueError) as exc:
52
+ path = (
53
+ bundle.sample_metadata.paths[source_index]
54
+ if 0 <= source_index < len(bundle.sample_metadata.paths)
55
+ else f"<sample:{index}>"
56
+ )
57
+ corrupt.append(f"{path}: {exc}")
58
+ continue
59
+ class_name = bundle.class_names[int(label)]
60
+ counts[class_name] += 1
61
+ digest = _content_hash(image)
62
+ content_hashes[digest] = content_hashes.get(digest, 0) + 1
63
+ _, height, width = image.shape
64
+ sizes[f"{height}x{width}x{image.shape[0]}"] = (
65
+ sizes.get(f"{height}x{width}x{image.shape[0]}", 0) + 1
66
+ )
67
+ source_path = (
68
+ bundle.sample_metadata.paths[source_index]
69
+ if 0 <= source_index < len(bundle.sample_metadata.paths)
70
+ else None
71
+ )
72
+ if source_path is not None:
73
+ path_sets[split_name].add(str(source_path))
74
+ group = (
75
+ bundle.sample_metadata.groups[source_index]
76
+ if 0 <= source_index < len(bundle.sample_metadata.groups)
77
+ else None
78
+ )
79
+ if group is not None:
80
+ group_sets[split_name].add(group)
81
+ class_counts[split_name] = counts
82
+
83
+ duplicates = {digest: count for digest, count in content_hashes.items() if count > 1}
84
+ split_overlap: list[str] = []
85
+ if "train" in path_sets and "val" in path_sets:
86
+ overlap = path_sets["train"] & path_sets["val"]
87
+ split_overlap = sorted(overlap)
88
+ group_leakage = sorted(group_sets.get("train", set()) & group_sets.get("val", set()))
89
+ warnings: list[str] = []
90
+ if split_overlap:
91
+ warnings.append(f"{len(split_overlap)} path(s) appear in both train and val")
92
+ if group_leakage:
93
+ warnings.append(f"group leakage across train and val: {', '.join(group_leakage)}")
94
+ if duplicates:
95
+ warnings.append(f"{len(duplicates)} duplicate content hash(es) found")
96
+ if corrupt:
97
+ warnings.append(f"{len(corrupt)} unreadable sample(s) found")
98
+ for split_name, counts in class_counts.items():
99
+ nonzero = [count for count in counts.values() if count > 0]
100
+ if nonzero and max(nonzero) > 2 * max(1, min(nonzero)):
101
+ warnings.append(f"{split_name} classes are imbalanced: {counts}")
102
+
103
+ return DataInspectionReport(
104
+ class_counts=class_counts,
105
+ duplicates=duplicates,
106
+ corrupt=corrupt,
107
+ sizes=sizes,
108
+ split_overlap=split_overlap,
109
+ group_leakage=group_leakage,
110
+ warnings=warnings,
111
+ )
@@ -0,0 +1,79 @@
1
+ """DataLoader construction with device-aware worker defaults."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import platform
6
+ from dataclasses import dataclass
7
+ from typing import Any
8
+
9
+ import torch
10
+ from torch.utils.data import DataLoader
11
+
12
+ from cnn_classifier.config.schema import RuntimeConfig, TrainingConfig
13
+ from cnn_classifier.core.seeding import make_generator
14
+ from cnn_classifier.core.types import DatasetBundle
15
+
16
+
17
+ @dataclass(frozen=True)
18
+ class DataLoaderBundle:
19
+ train: DataLoader[Any]
20
+ val: DataLoader[Any]
21
+ test: DataLoader[Any] | None
22
+
23
+
24
+ def _worker_count(runtime: RuntimeConfig, device: torch.device) -> int:
25
+ if isinstance(runtime.num_workers, int):
26
+ return runtime.num_workers
27
+ if device.type == "mps":
28
+ return 0
29
+ if device.type == "cuda" and platform.system() == "Linux":
30
+ return 4
31
+ return 2
32
+
33
+
34
+ def _pin_memory(runtime: RuntimeConfig, device: torch.device) -> bool:
35
+ if isinstance(runtime.pin_memory, bool):
36
+ return runtime.pin_memory
37
+ return device.type == "cuda"
38
+
39
+
40
+ def build_loaders(
41
+ bundle: DatasetBundle,
42
+ training: TrainingConfig,
43
+ runtime: RuntimeConfig,
44
+ device: torch.device,
45
+ ) -> DataLoaderBundle:
46
+ """Build train/val/test loaders with seeded shuffling and worker policy."""
47
+ workers = _worker_count(runtime, device)
48
+ pin_memory = _pin_memory(runtime, device)
49
+ persistent = runtime.persistent_workers and workers > 0
50
+ train = DataLoader(
51
+ bundle.train,
52
+ batch_size=training.batch_size,
53
+ shuffle=True,
54
+ num_workers=workers,
55
+ pin_memory=pin_memory,
56
+ persistent_workers=persistent,
57
+ generator=make_generator(bundle.split_info.seed),
58
+ )
59
+ val = DataLoader(
60
+ bundle.val,
61
+ batch_size=training.batch_size,
62
+ shuffle=False,
63
+ num_workers=workers,
64
+ pin_memory=pin_memory,
65
+ persistent_workers=persistent,
66
+ )
67
+ test = (
68
+ DataLoader(
69
+ bundle.test,
70
+ batch_size=training.batch_size,
71
+ shuffle=False,
72
+ num_workers=workers,
73
+ pin_memory=pin_memory,
74
+ persistent_workers=persistent,
75
+ )
76
+ if bundle.test is not None
77
+ else None
78
+ )
79
+ return DataLoaderBundle(train=train, val=val, test=test)