cnn-classifier-toolkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cnn_classifier/__init__.py +5 -0
- cnn_classifier/_version.py +1 -0
- cnn_classifier/cli.py +1085 -0
- cnn_classifier/config/__init__.py +17 -0
- cnn_classifier/config/loader.py +102 -0
- cnn_classifier/config/presets/default.yaml +47 -0
- cnn_classifier/config/resolve.py +108 -0
- cnn_classifier/config/schema.py +251 -0
- cnn_classifier/core/__init__.py +19 -0
- cnn_classifier/core/device.py +50 -0
- cnn_classifier/core/errors.py +21 -0
- cnn_classifier/core/hashing.py +18 -0
- cnn_classifier/core/registry.py +46 -0
- cnn_classifier/core/seeding.py +37 -0
- cnn_classifier/core/types.py +97 -0
- cnn_classifier/data/__init__.py +22 -0
- cnn_classifier/data/adapters/__init__.py +19 -0
- cnn_classifier/data/adapters/_common.py +241 -0
- cnn_classifier/data/adapters/image_folder.py +65 -0
- cnn_classifier/data/adapters/manifest.py +62 -0
- cnn_classifier/data/adapters/numpy.py +96 -0
- cnn_classifier/data/adapters/registry.py +29 -0
- cnn_classifier/data/adapters/synthetic.py +114 -0
- cnn_classifier/data/adapters/torchvision.py +192 -0
- cnn_classifier/data/base.py +17 -0
- cnn_classifier/data/contract.py +28 -0
- cnn_classifier/data/image_io.py +116 -0
- cnn_classifier/data/inspection.py +111 -0
- cnn_classifier/data/loaders.py +79 -0
- cnn_classifier/data/manifest.py +98 -0
- cnn_classifier/data/splits.py +136 -0
- cnn_classifier/data/stats.py +106 -0
- cnn_classifier/data/transforms.py +192 -0
- cnn_classifier/diagnostics/__init__.py +22 -0
- cnn_classifier/diagnostics/activations.py +56 -0
- cnn_classifier/diagnostics/architecture.py +172 -0
- cnn_classifier/diagnostics/determinism.py +25 -0
- cnn_classifier/diagnostics/gradients.py +32 -0
- cnn_classifier/diagnostics/sanity.py +49 -0
- cnn_classifier/diagnostics/tracing.py +56 -0
- cnn_classifier/evaluation/__init__.py +53 -0
- cnn_classifier/evaluation/bootstrap.py +96 -0
- cnn_classifier/evaluation/calibration.py +74 -0
- cnn_classifier/evaluation/compare.py +111 -0
- cnn_classifier/evaluation/error_analysis.py +61 -0
- cnn_classifier/evaluation/evaluator.py +143 -0
- cnn_classifier/evaluation/latency.py +58 -0
- cnn_classifier/evaluation/metrics.py +94 -0
- cnn_classifier/evaluation/predict.py +64 -0
- cnn_classifier/models/__init__.py +50 -0
- cnn_classifier/models/base.py +62 -0
- cnn_classifier/models/residual_cnn.py +138 -0
- cnn_classifier/models/simple_cnn.py +55 -0
- cnn_classifier/models/summary.py +39 -0
- cnn_classifier/models/torchvision.py +46 -0
- cnn_classifier/models/transfer.py +193 -0
- cnn_classifier/runs/__init__.py +6 -0
- cnn_classifier/runs/artifacts.py +50 -0
- cnn_classifier/runs/context.py +130 -0
- cnn_classifier/tracking/__init__.py +34 -0
- cnn_classifier/tracking/base.py +16 -0
- cnn_classifier/tracking/composite.py +20 -0
- cnn_classifier/tracking/console.py +14 -0
- cnn_classifier/tracking/csv.py +19 -0
- cnn_classifier/tracking/tensorboard.py +22 -0
- cnn_classifier/training/__init__.py +37 -0
- cnn_classifier/training/amp.py +51 -0
- cnn_classifier/training/callbacks.py +229 -0
- cnn_classifier/training/checkpoint.py +151 -0
- cnn_classifier/training/losses.py +67 -0
- cnn_classifier/training/optim.py +40 -0
- cnn_classifier/training/schedulers.py +53 -0
- cnn_classifier/training/state.py +35 -0
- cnn_classifier/training/trainer.py +357 -0
- cnn_classifier/viz/__init__.py +42 -0
- cnn_classifier/viz/architecture.py +276 -0
- cnn_classifier/viz/compare.py +36 -0
- cnn_classifier/viz/data.py +56 -0
- cnn_classifier/viz/evaluation.py +134 -0
- cnn_classifier/viz/features.py +122 -0
- cnn_classifier/viz/report.py +821 -0
- cnn_classifier/viz/style.py +69 -0
- cnn_classifier/viz/templates/report.html.j2 +355 -0
- cnn_classifier/viz/training.py +83 -0
- cnn_classifier_toolkit-0.1.0.dist-info/METADATA +249 -0
- cnn_classifier_toolkit-0.1.0.dist-info/RECORD +89 -0
- cnn_classifier_toolkit-0.1.0.dist-info/WHEEL +4 -0
- cnn_classifier_toolkit-0.1.0.dist-info/entry_points.txt +2 -0
- cnn_classifier_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Training components: optimization, scheduling, losses, AMP, and callbacks."""
|
|
2
|
+
|
|
3
|
+
from cnn_classifier.training.amp import AmpPolicy, autocast_context, resolve_amp
|
|
4
|
+
from cnn_classifier.training.callbacks import (
|
|
5
|
+
Callback,
|
|
6
|
+
EarlyStopping,
|
|
7
|
+
EMACallback,
|
|
8
|
+
GradientMonitor,
|
|
9
|
+
ModelCheckpoint,
|
|
10
|
+
NaNDetector,
|
|
11
|
+
TrackerCallback,
|
|
12
|
+
UnfreezeCallback,
|
|
13
|
+
)
|
|
14
|
+
from cnn_classifier.training.losses import MixedBatch, build_loss, mixup_cutmix_batch
|
|
15
|
+
from cnn_classifier.training.optim import build_optimizer
|
|
16
|
+
from cnn_classifier.training.schedulers import build_scheduler
|
|
17
|
+
from cnn_classifier.training.state import TrainingState
|
|
18
|
+
|
|
19
|
+
__all__ = [
|
|
20
|
+
"AmpPolicy",
|
|
21
|
+
"Callback",
|
|
22
|
+
"EMACallback",
|
|
23
|
+
"EarlyStopping",
|
|
24
|
+
"GradientMonitor",
|
|
25
|
+
"MixedBatch",
|
|
26
|
+
"ModelCheckpoint",
|
|
27
|
+
"NaNDetector",
|
|
28
|
+
"TrackerCallback",
|
|
29
|
+
"TrainingState",
|
|
30
|
+
"UnfreezeCallback",
|
|
31
|
+
"autocast_context",
|
|
32
|
+
"build_loss",
|
|
33
|
+
"build_optimizer",
|
|
34
|
+
"build_scheduler",
|
|
35
|
+
"mixup_cutmix_batch",
|
|
36
|
+
"resolve_amp",
|
|
37
|
+
]
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Device-aware mixed-precision policy."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import Literal
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
|
|
12
|
+
from cnn_classifier.core.errors import ConfigError
|
|
13
|
+
|
|
14
|
+
AmpRequest = Literal["auto", "off", "fp16", "bf16"]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class AmpPolicy:
|
|
19
|
+
enabled: bool
|
|
20
|
+
dtype: torch.dtype | None
|
|
21
|
+
scaler: torch.amp.GradScaler | None
|
|
22
|
+
device_type: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def resolve_amp(device: torch.device, requested: AmpRequest) -> AmpPolicy:
|
|
26
|
+
"""Resolve the requested AMP mode against the actual device."""
|
|
27
|
+
if requested == "off":
|
|
28
|
+
return AmpPolicy(False, None, None, device.type)
|
|
29
|
+
if requested == "auto":
|
|
30
|
+
if device.type == "cuda":
|
|
31
|
+
return AmpPolicy(True, torch.float16, torch.amp.GradScaler("cuda"), "cuda")
|
|
32
|
+
return AmpPolicy(False, None, None, device.type)
|
|
33
|
+
if requested == "fp16":
|
|
34
|
+
if device.type != "cuda":
|
|
35
|
+
raise ConfigError("fp16 AMP is only supported on CUDA devices")
|
|
36
|
+
return AmpPolicy(True, torch.float16, torch.amp.GradScaler("cuda"), "cuda")
|
|
37
|
+
if requested == "bf16":
|
|
38
|
+
if device.type == "mps":
|
|
39
|
+
raise ConfigError("bf16 AMP is not enabled for MPS in this release")
|
|
40
|
+
return AmpPolicy(True, torch.bfloat16, None, device.type)
|
|
41
|
+
raise ConfigError(f"unknown AMP request {requested!r}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@contextlib.contextmanager
|
|
45
|
+
def autocast_context(policy: AmpPolicy) -> Iterator[None]:
|
|
46
|
+
"""Enter autocast when the policy is enabled; otherwise do nothing."""
|
|
47
|
+
if not policy.enabled or policy.dtype is None:
|
|
48
|
+
yield
|
|
49
|
+
return
|
|
50
|
+
with torch.autocast(device_type=policy.device_type, dtype=policy.dtype):
|
|
51
|
+
yield
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""Training callbacks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import math
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, Protocol
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
from torch import nn
|
|
13
|
+
|
|
14
|
+
from cnn_classifier.config.schema import UnfreezeEvent
|
|
15
|
+
from cnn_classifier.core.errors import ConfigError
|
|
16
|
+
from cnn_classifier.models.transfer import unfreeze_modules
|
|
17
|
+
from cnn_classifier.training.checkpoint import save_checkpoint
|
|
18
|
+
from cnn_classifier.training.state import TrainingState
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Callback(Protocol):
|
|
22
|
+
def on_train_start(self, state: TrainingState) -> None: ...
|
|
23
|
+
|
|
24
|
+
def on_epoch_end(self, state: TrainingState) -> None: ...
|
|
25
|
+
|
|
26
|
+
def on_train_end(self, state: TrainingState) -> None: ...
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _NoOpCallback:
|
|
30
|
+
def on_train_start(self, state: TrainingState) -> None:
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
def on_train_end(self, state: TrainingState) -> None:
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _is_better(value: float, best: float, mode: str, min_delta: float) -> bool:
|
|
41
|
+
if mode == "max":
|
|
42
|
+
return value > best + min_delta
|
|
43
|
+
return value < best - min_delta
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _initial_best(mode: str) -> float:
|
|
47
|
+
return -math.inf if mode == "max" else math.inf
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class EarlyStopping(_NoOpCallback):
|
|
51
|
+
def __init__(self, monitor: str, mode: str = "max", patience: int = 10, min_delta: float = 0.0):
|
|
52
|
+
self.monitor = monitor
|
|
53
|
+
self.mode = mode
|
|
54
|
+
self.patience = patience
|
|
55
|
+
self.min_delta = min_delta
|
|
56
|
+
self.best = _initial_best(mode)
|
|
57
|
+
self.wait = 0
|
|
58
|
+
|
|
59
|
+
def on_train_start(self, state: TrainingState) -> None:
|
|
60
|
+
self.best = _initial_best(self.mode)
|
|
61
|
+
self.wait = 0
|
|
62
|
+
|
|
63
|
+
def restore_from_history(
|
|
64
|
+
self,
|
|
65
|
+
history: Sequence[dict[str, float]],
|
|
66
|
+
fallback: float,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Restore best value and patience progress after a resume."""
|
|
69
|
+
best = _initial_best(self.mode)
|
|
70
|
+
seen = 0
|
|
71
|
+
last_improvement = -1
|
|
72
|
+
for row in history:
|
|
73
|
+
if self.monitor not in row:
|
|
74
|
+
continue
|
|
75
|
+
value = float(row[self.monitor])
|
|
76
|
+
seen += 1
|
|
77
|
+
if _is_better(value, best, self.mode, self.min_delta):
|
|
78
|
+
best = value
|
|
79
|
+
last_improvement = seen - 1
|
|
80
|
+
if last_improvement >= 0:
|
|
81
|
+
self.best = best
|
|
82
|
+
self.wait = seen - 1 - last_improvement
|
|
83
|
+
else:
|
|
84
|
+
self.best = fallback
|
|
85
|
+
self.wait = 0
|
|
86
|
+
|
|
87
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
88
|
+
if self.monitor not in state.metrics:
|
|
89
|
+
raise ConfigError(f"early_stopping monitor {self.monitor!r} is not a logged metric")
|
|
90
|
+
value = float(state.metrics[self.monitor])
|
|
91
|
+
if _is_better(value, self.best, self.mode, self.min_delta):
|
|
92
|
+
self.best = value
|
|
93
|
+
self.wait = 0
|
|
94
|
+
else:
|
|
95
|
+
self.wait += 1
|
|
96
|
+
if self.wait >= self.patience:
|
|
97
|
+
state.stop_requested = True
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class ModelCheckpoint(_NoOpCallback):
|
|
101
|
+
def __init__(self, monitor: str, mode: str = "max", save_last: bool = True):
|
|
102
|
+
self.monitor = monitor
|
|
103
|
+
self.mode = mode
|
|
104
|
+
self.save_last = save_last
|
|
105
|
+
self.best = _initial_best(mode)
|
|
106
|
+
|
|
107
|
+
def on_train_start(self, state: TrainingState) -> None:
|
|
108
|
+
self.best = _initial_best(self.mode)
|
|
109
|
+
|
|
110
|
+
def restore_best(self, best: float) -> None:
|
|
111
|
+
"""Restore the best monitored value after a resume."""
|
|
112
|
+
self.best = best
|
|
113
|
+
|
|
114
|
+
def _save(self, state: TrainingState, name: str) -> None:
|
|
115
|
+
save_checkpoint(
|
|
116
|
+
Path(state.run.directory) / "checkpoints" / name,
|
|
117
|
+
model=state.eval_model or state.model,
|
|
118
|
+
optimizer=state.optimizer,
|
|
119
|
+
scheduler=state.scheduler,
|
|
120
|
+
scaler=state.scaler,
|
|
121
|
+
epoch=state.epoch,
|
|
122
|
+
global_step=state.global_step,
|
|
123
|
+
best_metric=state.best_metric,
|
|
124
|
+
best_epoch=state.best_epoch,
|
|
125
|
+
config_hash=state.run.config_hash,
|
|
126
|
+
class_names=state.class_names,
|
|
127
|
+
normalization=state.normalization,
|
|
128
|
+
model_contract=state.model_contract,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
132
|
+
if self.monitor not in state.metrics:
|
|
133
|
+
raise ConfigError(f"checkpoint monitor {self.monitor!r} is not a logged metric")
|
|
134
|
+
value = float(state.metrics[self.monitor])
|
|
135
|
+
improved = _is_better(value, self.best, self.mode, 0.0)
|
|
136
|
+
if improved:
|
|
137
|
+
self.best = value
|
|
138
|
+
state.best_metric = value
|
|
139
|
+
state.best_epoch = state.epoch
|
|
140
|
+
if self.save_last:
|
|
141
|
+
self._save(state, "last.pt")
|
|
142
|
+
if improved:
|
|
143
|
+
self._save(state, "best.pt")
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class GradientMonitor(_NoOpCallback):
|
|
147
|
+
def __init__(self) -> None:
|
|
148
|
+
self.last_report: dict[str, float] = {}
|
|
149
|
+
|
|
150
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
151
|
+
report = {}
|
|
152
|
+
for name, parameter in state.model.named_parameters():
|
|
153
|
+
if parameter.grad is not None:
|
|
154
|
+
report[name] = float(parameter.grad.detach().norm().item())
|
|
155
|
+
self.last_report = report
|
|
156
|
+
if report:
|
|
157
|
+
state.metrics["grad_norm"] = float(
|
|
158
|
+
sum(value * value for value in report.values()) ** 0.5
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class NaNDetector(_NoOpCallback):
|
|
163
|
+
def __init__(self) -> None:
|
|
164
|
+
self.failed = False
|
|
165
|
+
|
|
166
|
+
def on_train_start(self, state: TrainingState) -> None:
|
|
167
|
+
self.failed = False
|
|
168
|
+
|
|
169
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
170
|
+
values = list(state.metrics.values())
|
|
171
|
+
gradients = [
|
|
172
|
+
parameter.grad for parameter in state.model.parameters() if parameter.grad is not None
|
|
173
|
+
]
|
|
174
|
+
nonfinite = any(not math.isfinite(float(value)) for value in values) or any(
|
|
175
|
+
not bool(torch.isfinite(gradient).all()) for gradient in gradients
|
|
176
|
+
)
|
|
177
|
+
if nonfinite:
|
|
178
|
+
self.failed = True
|
|
179
|
+
state.stop_requested = True
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class EMACallback(_NoOpCallback):
|
|
183
|
+
def __init__(self, decay: float = 0.999) -> None:
|
|
184
|
+
if not 0.0 < decay < 1.0:
|
|
185
|
+
raise ConfigError("EMA decay must be in (0, 1)")
|
|
186
|
+
self.decay = decay
|
|
187
|
+
self.ema: nn.Module | None = None
|
|
188
|
+
|
|
189
|
+
def on_train_start(self, state: TrainingState) -> None:
|
|
190
|
+
self.ema = copy.deepcopy(state.model)
|
|
191
|
+
for parameter in self.ema.parameters():
|
|
192
|
+
parameter.requires_grad = False
|
|
193
|
+
|
|
194
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
195
|
+
if self.ema is None:
|
|
196
|
+
self.on_train_start(state)
|
|
197
|
+
assert self.ema is not None
|
|
198
|
+
with torch.no_grad():
|
|
199
|
+
for ema_parameter, parameter in zip(
|
|
200
|
+
self.ema.parameters(), state.model.parameters(), strict=True
|
|
201
|
+
):
|
|
202
|
+
ema_parameter.mul_(self.decay).add_(parameter, alpha=1.0 - self.decay)
|
|
203
|
+
for ema_buffer, buffer in zip(self.ema.buffers(), state.model.buffers(), strict=True):
|
|
204
|
+
if torch.is_floating_point(ema_buffer):
|
|
205
|
+
ema_buffer.mul_(self.decay).add_(buffer, alpha=1.0 - self.decay)
|
|
206
|
+
else:
|
|
207
|
+
ema_buffer.copy_(buffer)
|
|
208
|
+
state.eval_model = self.ema
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class UnfreezeCallback(_NoOpCallback):
|
|
212
|
+
def __init__(self, schedule: Sequence[UnfreezeEvent]) -> None:
|
|
213
|
+
self.schedule = tuple(schedule)
|
|
214
|
+
|
|
215
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
216
|
+
for event in self.schedule:
|
|
217
|
+
if event.epoch == state.epoch:
|
|
218
|
+
unfreeze_modules(state.model, event.modules)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class TrackerCallback(_NoOpCallback):
|
|
222
|
+
def __init__(self, tracker: Any) -> None:
|
|
223
|
+
self.tracker = tracker
|
|
224
|
+
|
|
225
|
+
def on_epoch_end(self, state: TrainingState) -> None:
|
|
226
|
+
self.tracker.log_metrics(state.metrics, state.epoch)
|
|
227
|
+
|
|
228
|
+
def on_train_end(self, state: TrainingState) -> None:
|
|
229
|
+
self.tracker.close()
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""Checkpoint serialization with config identity and RNG state."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import random
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import asdict, is_dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any, cast
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import torch
|
|
13
|
+
from torch import nn
|
|
14
|
+
|
|
15
|
+
from cnn_classifier.core.errors import RunError
|
|
16
|
+
|
|
17
|
+
CheckpointPayload = dict[str, Any]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _jsonable(value: Any) -> Any:
|
|
21
|
+
if value is None:
|
|
22
|
+
return None
|
|
23
|
+
if is_dataclass(value):
|
|
24
|
+
return cast(Any, asdict)(value)
|
|
25
|
+
return value
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _normalization_dict(value: Any) -> dict[str, Any] | None:
|
|
29
|
+
data = _jsonable(value)
|
|
30
|
+
if data is None:
|
|
31
|
+
return None
|
|
32
|
+
if not isinstance(data, dict):
|
|
33
|
+
return {"value": data}
|
|
34
|
+
return {
|
|
35
|
+
"mean": tuple(round(float(item), 8) for item in data.get("mean", ())),
|
|
36
|
+
"std": tuple(round(float(item), 8) for item in data.get("std", ())),
|
|
37
|
+
"source": data.get("source"),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _contract_dict(value: Any) -> dict[str, Any] | None:
|
|
42
|
+
data = _jsonable(value)
|
|
43
|
+
if data is None or not isinstance(data, dict):
|
|
44
|
+
return None
|
|
45
|
+
return {
|
|
46
|
+
"input_shape": list(data.get("input_shape", ())),
|
|
47
|
+
"dtype": data.get("dtype"),
|
|
48
|
+
"value_range": list(data.get("value_range", ())),
|
|
49
|
+
"class_names": list(data.get("class_names", ())),
|
|
50
|
+
"label_map": dict(data.get("label_map", {})),
|
|
51
|
+
"channel_policy": data.get("channel_policy"),
|
|
52
|
+
"stem_adaptation": data.get("stem_adaptation"),
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def validate_checkpoint_identity(
|
|
57
|
+
payload: CheckpointPayload,
|
|
58
|
+
*,
|
|
59
|
+
class_names: Sequence[str] | None = None,
|
|
60
|
+
normalization: Any = None,
|
|
61
|
+
model_contract: Any = None,
|
|
62
|
+
) -> None:
|
|
63
|
+
"""Reject a checkpoint whose class map, normalization, or contract changed."""
|
|
64
|
+
stored_names = payload.get("class_names")
|
|
65
|
+
if (
|
|
66
|
+
class_names is not None
|
|
67
|
+
and stored_names is not None
|
|
68
|
+
and tuple(stored_names) != tuple(class_names)
|
|
69
|
+
):
|
|
70
|
+
raise RunError(
|
|
71
|
+
f"checkpoint class names {tuple(stored_names)!r} do not match "
|
|
72
|
+
f"current class names {tuple(class_names)!r}"
|
|
73
|
+
)
|
|
74
|
+
if (
|
|
75
|
+
payload.get("normalization") is not None or normalization is not None
|
|
76
|
+
) and _normalization_dict(payload.get("normalization")) != _normalization_dict(normalization):
|
|
77
|
+
raise RunError("checkpoint normalization does not match the current run")
|
|
78
|
+
stored_contract = payload.get("model_contract")
|
|
79
|
+
if (
|
|
80
|
+
stored_contract is not None
|
|
81
|
+
and model_contract is not None
|
|
82
|
+
and _contract_dict(stored_contract) != _contract_dict(model_contract)
|
|
83
|
+
):
|
|
84
|
+
raise RunError("checkpoint model contract does not match the current run")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def save_checkpoint(
|
|
88
|
+
path: Path,
|
|
89
|
+
*,
|
|
90
|
+
model: nn.Module,
|
|
91
|
+
optimizer: torch.optim.Optimizer | None,
|
|
92
|
+
scheduler: Any,
|
|
93
|
+
scaler: Any,
|
|
94
|
+
epoch: int,
|
|
95
|
+
global_step: int,
|
|
96
|
+
best_metric: float,
|
|
97
|
+
best_epoch: int,
|
|
98
|
+
config_hash: str,
|
|
99
|
+
class_names: tuple[str, ...],
|
|
100
|
+
normalization: Any,
|
|
101
|
+
model_contract: Any,
|
|
102
|
+
) -> None:
|
|
103
|
+
"""Save a complete, resumable checkpoint."""
|
|
104
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
105
|
+
payload: CheckpointPayload = {
|
|
106
|
+
"model": model.state_dict(),
|
|
107
|
+
"optimizer": optimizer.state_dict() if optimizer is not None else None,
|
|
108
|
+
"scheduler": scheduler.state_dict() if scheduler is not None else None,
|
|
109
|
+
"scaler": scaler.state_dict() if scaler is not None else None,
|
|
110
|
+
"epoch": int(epoch),
|
|
111
|
+
"global_step": int(global_step),
|
|
112
|
+
"best_metric": float(best_metric),
|
|
113
|
+
"best_epoch": int(best_epoch),
|
|
114
|
+
"config_hash": config_hash,
|
|
115
|
+
"class_names": tuple(class_names),
|
|
116
|
+
"normalization": _jsonable(normalization),
|
|
117
|
+
"model_contract": _jsonable(model_contract),
|
|
118
|
+
"rng": {
|
|
119
|
+
"torch": torch.get_rng_state(),
|
|
120
|
+
"numpy": np.random.get_state(),
|
|
121
|
+
"python": random.getstate(),
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
if torch.cuda.is_available():
|
|
125
|
+
payload["rng"]["cuda"] = torch.cuda.get_rng_state_all()
|
|
126
|
+
torch.save(payload, path)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def load_checkpoint(
|
|
130
|
+
path: Path,
|
|
131
|
+
map_location: str | torch.device = "cpu",
|
|
132
|
+
expected_config_hash: str | None = None,
|
|
133
|
+
allow_config_change: bool = False,
|
|
134
|
+
) -> CheckpointPayload:
|
|
135
|
+
"""Load a trusted checkpoint and enforce its config identity."""
|
|
136
|
+
try:
|
|
137
|
+
payload = torch.load(path, map_location=map_location, weights_only=False)
|
|
138
|
+
except (OSError, RuntimeError, ValueError) as exc:
|
|
139
|
+
raise RunError(f"cannot load checkpoint {path}: {exc}") from exc
|
|
140
|
+
if not isinstance(payload, dict):
|
|
141
|
+
raise RunError(f"checkpoint {path} does not contain a mapping")
|
|
142
|
+
if (
|
|
143
|
+
expected_config_hash is not None
|
|
144
|
+
and payload.get("config_hash") != expected_config_hash
|
|
145
|
+
and not allow_config_change
|
|
146
|
+
):
|
|
147
|
+
raise RunError(
|
|
148
|
+
f"checkpoint config hash {payload.get('config_hash')!r} does not match "
|
|
149
|
+
f"expected {expected_config_hash!r}"
|
|
150
|
+
)
|
|
151
|
+
return payload
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Loss functions and batch-level MixUp/CutMix."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import nn
|
|
10
|
+
|
|
11
|
+
from cnn_classifier.config.schema import LossConfig
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class MixedBatch:
|
|
16
|
+
images: torch.Tensor
|
|
17
|
+
labels_a: torch.Tensor
|
|
18
|
+
labels_b: torch.Tensor
|
|
19
|
+
lam: float
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def build_loss(config: LossConfig) -> nn.Module:
|
|
23
|
+
"""Build the configured classification loss."""
|
|
24
|
+
if config.name == "cross_entropy":
|
|
25
|
+
return nn.CrossEntropyLoss(label_smoothing=config.label_smoothing)
|
|
26
|
+
raise ValueError(f"unknown loss {config.name!r}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def mixup_cutmix_batch(
|
|
30
|
+
images: torch.Tensor,
|
|
31
|
+
labels: torch.Tensor,
|
|
32
|
+
mixup_alpha: float,
|
|
33
|
+
cutmix_alpha: float,
|
|
34
|
+
generator: torch.Generator,
|
|
35
|
+
) -> MixedBatch:
|
|
36
|
+
"""Apply MixUp, CutMix, or neither, returning labels for both mixtures."""
|
|
37
|
+
if images.ndim != 4:
|
|
38
|
+
raise ValueError("MixUp/CutMix expects [B,C,H,W] images")
|
|
39
|
+
if mixup_alpha <= 0 and cutmix_alpha <= 0:
|
|
40
|
+
return MixedBatch(images, labels, labels, 1.0)
|
|
41
|
+
batch_size = images.shape[0]
|
|
42
|
+
permutation = torch.randperm(batch_size, generator=generator).to(images.device)
|
|
43
|
+
use_cutmix = cutmix_alpha > 0 and (
|
|
44
|
+
mixup_alpha <= 0 or float(torch.rand(1, generator=generator).item()) < 0.5
|
|
45
|
+
)
|
|
46
|
+
if use_cutmix:
|
|
47
|
+
lam = float(torch.rand(1, generator=generator).item())
|
|
48
|
+
cut_ratio = math.sqrt(1.0 - lam)
|
|
49
|
+
cut_h = int(images.shape[2] * cut_ratio)
|
|
50
|
+
cut_w = int(images.shape[3] * cut_ratio)
|
|
51
|
+
center_y = int(torch.randint(images.shape[2], (1,), generator=generator).item())
|
|
52
|
+
center_x = int(torch.randint(images.shape[3], (1,), generator=generator).item())
|
|
53
|
+
y1 = max(center_y - cut_h // 2, 0)
|
|
54
|
+
y2 = min(center_y + cut_h // 2, images.shape[2])
|
|
55
|
+
x1 = max(center_x - cut_w // 2, 0)
|
|
56
|
+
x2 = min(center_x + cut_w // 2, images.shape[3])
|
|
57
|
+
mixed = images.clone()
|
|
58
|
+
mixed[:, :, y1:y2, x1:x2] = images[permutation, :, y1:y2, x1:x2]
|
|
59
|
+
lam = 1.0 - ((y2 - y1) * (x2 - x1) / (images.shape[2] * images.shape[3]))
|
|
60
|
+
return MixedBatch(mixed, labels, labels[permutation], lam)
|
|
61
|
+
lam = float(
|
|
62
|
+
torch.distributions.Beta(torch.tensor(mixup_alpha), torch.tensor(mixup_alpha))
|
|
63
|
+
.sample()
|
|
64
|
+
.item()
|
|
65
|
+
)
|
|
66
|
+
mixed = lam * images + (1.0 - lam) * images[permutation]
|
|
67
|
+
return MixedBatch(mixed, labels, labels[permutation], lam)
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""Optimizer construction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
from torch import nn
|
|
9
|
+
|
|
10
|
+
from cnn_classifier.config.schema import TrainingConfig
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def build_optimizer(
|
|
14
|
+
model: nn.Module,
|
|
15
|
+
config: TrainingConfig,
|
|
16
|
+
param_groups: list[dict[str, Any]] | None = None,
|
|
17
|
+
) -> torch.optim.Optimizer:
|
|
18
|
+
"""Build Adam, AdamW, or SGD with the configured hyperparameters."""
|
|
19
|
+
parameters: Any = model.parameters() if param_groups is None else param_groups
|
|
20
|
+
optimizer = config.optimizer
|
|
21
|
+
if optimizer.name == "sgd":
|
|
22
|
+
return torch.optim.SGD(
|
|
23
|
+
parameters,
|
|
24
|
+
lr=optimizer.lr,
|
|
25
|
+
momentum=optimizer.momentum,
|
|
26
|
+
weight_decay=optimizer.weight_decay,
|
|
27
|
+
)
|
|
28
|
+
if optimizer.name == "adam":
|
|
29
|
+
return torch.optim.Adam(
|
|
30
|
+
parameters,
|
|
31
|
+
lr=optimizer.lr,
|
|
32
|
+
weight_decay=optimizer.weight_decay,
|
|
33
|
+
)
|
|
34
|
+
if optimizer.name == "adamw":
|
|
35
|
+
return torch.optim.AdamW(
|
|
36
|
+
parameters,
|
|
37
|
+
lr=optimizer.lr,
|
|
38
|
+
weight_decay=optimizer.weight_decay,
|
|
39
|
+
)
|
|
40
|
+
raise ValueError(f"unknown optimizer {optimizer.name!r}")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Learning-rate scheduler construction."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
|
|
7
|
+
import torch
|
|
8
|
+
|
|
9
|
+
from cnn_classifier.config.schema import SchedulerConfig
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _cosine(epoch: int, epochs: int, min_ratio: float) -> float:
|
|
13
|
+
progress = min(epoch / max(1, epochs - 1), 1.0)
|
|
14
|
+
return min_ratio + (1.0 - min_ratio) * 0.5 * (1.0 + math.cos(math.pi * progress))
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_scheduler(
|
|
18
|
+
optimizer: torch.optim.Optimizer,
|
|
19
|
+
config: SchedulerConfig,
|
|
20
|
+
epochs: int,
|
|
21
|
+
) -> torch.optim.lr_scheduler.LRScheduler:
|
|
22
|
+
"""Build the configured scheduler; warmup is explicit."""
|
|
23
|
+
if config.name == "constant":
|
|
24
|
+
return torch.optim.lr_scheduler.LambdaLR(optimizer, lambda _epoch: 1.0)
|
|
25
|
+
if config.name == "cosine":
|
|
26
|
+
return torch.optim.lr_scheduler.LambdaLR(
|
|
27
|
+
optimizer,
|
|
28
|
+
lambda epoch: _cosine(epoch, epochs, config.min_lr_ratio),
|
|
29
|
+
)
|
|
30
|
+
if config.name == "cosine_warmup":
|
|
31
|
+
if config.warmup_epochs <= 0:
|
|
32
|
+
return torch.optim.lr_scheduler.LambdaLR(
|
|
33
|
+
optimizer,
|
|
34
|
+
lambda epoch: _cosine(epoch, epochs, config.min_lr_ratio),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
def warmup_lambda(epoch: int) -> float:
|
|
38
|
+
if epoch < config.warmup_epochs:
|
|
39
|
+
return (epoch + 1) / config.warmup_epochs
|
|
40
|
+
progress = (epoch - config.warmup_epochs) / max(1, epochs - config.warmup_epochs - 1)
|
|
41
|
+
progress = min(progress, 1.0)
|
|
42
|
+
return config.min_lr_ratio + (1.0 - config.min_lr_ratio) * 0.5 * (
|
|
43
|
+
1.0 + math.cos(math.pi * progress)
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return torch.optim.lr_scheduler.LambdaLR(optimizer, warmup_lambda)
|
|
47
|
+
if config.name == "step":
|
|
48
|
+
return torch.optim.lr_scheduler.StepLR(
|
|
49
|
+
optimizer, step_size=config.step_size, gamma=config.gamma
|
|
50
|
+
)
|
|
51
|
+
if config.name == "plateau":
|
|
52
|
+
return torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", patience=5)
|
|
53
|
+
raise ValueError(f"unknown scheduler {config.name!r}")
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Mutable state shared by the trainer and callbacks."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import nn
|
|
10
|
+
|
|
11
|
+
from cnn_classifier.config.schema import RunConfig
|
|
12
|
+
from cnn_classifier.runs.context import RunContext
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass
|
|
16
|
+
class TrainingState:
|
|
17
|
+
model: nn.Module
|
|
18
|
+
eval_model: nn.Module | None
|
|
19
|
+
optimizer: torch.optim.Optimizer | None
|
|
20
|
+
scheduler: Any
|
|
21
|
+
scaler: Any
|
|
22
|
+
epoch: int
|
|
23
|
+
global_step: int
|
|
24
|
+
history: list[dict[str, float]]
|
|
25
|
+
metrics: dict[str, float]
|
|
26
|
+
best_metric: float
|
|
27
|
+
best_epoch: int
|
|
28
|
+
stop_requested: bool
|
|
29
|
+
run: RunContext
|
|
30
|
+
config: RunConfig
|
|
31
|
+
device: torch.device
|
|
32
|
+
class_names: tuple[str, ...] = ()
|
|
33
|
+
normalization: Any = None
|
|
34
|
+
model_contract: Any = None
|
|
35
|
+
extra: dict[str, Any] = field(default_factory=dict)
|