cnn-classifier-toolkit 0.1.0__py3-none-any.whl

cnn_classifier/evaluation/error_analysis.py

@@ -0,0 +1,61 @@
+"""Error analysis helpers for evaluation reports."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class ErrorExample:
+    index: int
+    true: str
+    predicted: str
+    confidence: float
+
+
+def most_confident_errors(
+    probabilities: np.ndarray,
+    y_true: np.ndarray,
+    class_names: Sequence[str],
+    max_examples: int = 32,
+) -> list[ErrorExample]:
+    """Return wrong predictions ranked by model confidence."""
+    probs = np.asarray(probabilities, dtype=np.float64)
+    truth = np.asarray(y_true, dtype=np.int64)
+    predictions = probs.argmax(axis=1)
+    confidence = probs.max(axis=1)
+    wrong = np.flatnonzero(predictions != truth)
+    order = wrong[np.argsort(confidence[wrong])[::-1]][:max_examples]
+    return [
+        ErrorExample(
+            index=int(index),
+            true=class_names[int(truth[index])],
+            predicted=class_names[int(predictions[index])],
+            confidence=float(confidence[index]),
+        )
+        for index in order
+    ]
+
+
+def confusion_pairs(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    class_names: Sequence[str],
+) -> list[tuple[str, str, int]]:
+    """Return off-diagonal confusion pairs sorted by count."""
+    truth = np.asarray(y_true, dtype=np.int64)
+    predictions = np.asarray(y_pred, dtype=np.int64)
+    counts: dict[tuple[int, int], int] = {}
+    for actual, predicted in zip(truth, predictions, strict=True):
+        if actual == predicted:
+            continue
+        key = (int(actual), int(predicted))
+        counts[key] = counts.get(key, 0) + 1
+    ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+    return [
+        (class_names[actual], class_names[predicted], count)
+        for (actual, predicted), count in ordered
+    ]

cnn_classifier/evaluation/evaluator.py

@@ -0,0 +1,143 @@
+"""Inference, TTA, metrics, and prediction artifact helpers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import torch
+from torch import nn
+from torch.utils.data import DataLoader
+
+from cnn_classifier.config.schema import EvaluationConfig
+from cnn_classifier.core.errors import DataError
+from cnn_classifier.evaluation.bootstrap import ConfidenceInterval, bootstrap_ci
+from cnn_classifier.evaluation.latency import LatencyReport, measure_latency
+from cnn_classifier.evaluation.metrics import MetricsReport, accuracy, classification_metrics
+
+
+@dataclass(frozen=True)
+class EvaluationResult:
+    metrics: MetricsReport
+    probabilities: np.ndarray
+    y_true: np.ndarray
+    y_pred: np.ndarray
+    indices: np.ndarray
+    paths: tuple[str, ...]
+    latency: LatencyReport | None
+    base_metrics: MetricsReport | None = None
+    tta_metrics: MetricsReport | None = None
+    confidence_intervals: dict[str, ConfidenceInterval] = field(default_factory=dict)
+
+
+@torch.no_grad()
+def _predict(
+    model: nn.Module,
+    loader: DataLoader[Any],
+    device: torch.device,
+    config: EvaluationConfig,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
+    model.eval()
+    base_parts: list[np.ndarray] = []
+    tta_parts: list[np.ndarray] = []
+    labels_parts: list[np.ndarray] = []
+    index_parts: list[np.ndarray] = []
+    use_hflip = config.tta.enabled and "hflip" in config.tta.transforms
+    for images, labels, indices in loader:
+        images = images.to(device)
+        base = torch.softmax(model(images), dim=1)
+        base_parts.append(base.cpu().numpy())
+        if use_hflip:
+            flipped = torch.flip(images, dims=[3])
+            tta = (base + torch.softmax(model(flipped), dim=1)) / 2.0
+        else:
+            tta = base
+        tta_parts.append(tta.cpu().numpy())
+        labels_parts.append(labels.numpy())
+        index_parts.append(indices.numpy())
+    if not base_parts:
+        raise DataError("evaluation loader produced no samples")
+    return (
+        np.concatenate(base_parts),
+        np.concatenate(tta_parts),
+        np.concatenate(labels_parts),
+        np.concatenate(index_parts),
+        np.concatenate(index_parts),
+    )
+
+
+def evaluate_model(
+    model: nn.Module,
+    loader: DataLoader[Any],
+    device: torch.device | str,
+    class_names: tuple[str, ...] | list[str],
+    config: EvaluationConfig,
+) -> EvaluationResult:
+    """Evaluate on a loader without touching training or test state."""
+    runtime_device = torch.device(device) if isinstance(device, str) else device
+    was_training = model.training
+    base_probs, tta_probs, y_true, indices, _ = _predict(model, loader, runtime_device, config)
+    y_pred = base_probs.argmax(axis=1)
+    names = tuple(class_names)
+    base_metrics = classification_metrics(y_true, y_pred, base_probs, names, top_k=config.top_k)
+    tta_metrics = None
+    if config.tta.enabled:
+        tta_pred = tta_probs.argmax(axis=1)
+        tta_metrics = classification_metrics(y_true, tta_pred, tta_probs, names, top_k=config.top_k)
+    metrics = tta_metrics or base_metrics
+    intervals: dict[str, ConfidenceInterval] = {}
+    if config.bootstrap.enabled:
+        used_probs = tta_probs if config.tta.enabled else base_probs
+        used_pred = used_probs.argmax(axis=1)
+        intervals["accuracy"] = bootstrap_ci(
+            y_true,
+            used_pred,
+            accuracy,
+            n_resamples=config.bootstrap.n_resamples,
+            confidence=config.bootstrap.confidence,
+            seed=0,
+            stratify=config.bootstrap.stratify,
+        )
+    latency = None
+    if config.latency.enabled:
+        sample_shape = loader.dataset[0][0].shape
+        input_shape = (
+            int(sample_shape[0]),
+            int(sample_shape[1]),
+            int(sample_shape[2]),
+        )
+        latency = measure_latency(
+            model,
+            input_shape,
+            runtime_device,
+            config.latency.warmup,
+            config.latency.iterations,
+        )
+    model.train(was_training)
+    return EvaluationResult(
+        metrics=metrics,
+        probabilities=tta_probs if config.tta.enabled else base_probs,
+        y_true=y_true,
+        y_pred=tta_probs.argmax(axis=1) if config.tta.enabled else y_pred,
+        indices=indices,
+        paths=tuple(str(int(index)) for index in indices),
+        latency=latency,
+        base_metrics=base_metrics,
+        tta_metrics=tta_metrics,
+        confidence_intervals=intervals,
+    )
+
+
+def save_predictions(result: EvaluationResult, path: Path) -> Path:
+    """Save prediction arrays in a portable ``.npz`` archive."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    np.savez_compressed(
+        path,
+        probabilities=result.probabilities,
+        y_true=result.y_true,
+        y_pred=result.y_pred,
+        indices=result.indices,
+    )
+    return path

cnn_classifier/evaluation/latency.py

@@ -0,0 +1,58 @@
+"""Inference latency and throughput measurement."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+from torch import nn
+
+
+@dataclass(frozen=True)
+class LatencyReport:
+    mean_ms: float
+    p50_ms: float
+    p95_ms: float
+    throughput: float
+
+
+def _synchronize(device: torch.device) -> None:
+    if device.type == "cuda":
+        torch.cuda.synchronize(device)
+    elif device.type == "mps":
+        torch.mps.synchronize()
+
+
+@torch.no_grad()
+def measure_latency(
+    model: nn.Module,
+    input_shape: tuple[int, int, int],
+    device: torch.device | str,
+    warmup: int = 20,
+    iterations: int = 100,
+) -> LatencyReport:
+    """Measure single-image forward-pass latency with warmup and sync."""
+    runtime_device = torch.device(device) if isinstance(device, str) else device
+    was_training = model.training
+    model.eval()
+    sample = torch.zeros(1, *input_shape, device=runtime_device)
+    for _ in range(max(0, warmup)):
+        model(sample)
+    _synchronize(runtime_device)
+    timings = []
+    for _ in range(max(1, iterations)):
+        start = time.perf_counter()
+        model(sample)
+        _synchronize(runtime_device)
+        timings.append((time.perf_counter() - start) * 1000.0)
+    model.train(was_training)
+    values = np.array(timings, dtype=np.float64)
+    mean_ms = float(values.mean())
+    return LatencyReport(
+        mean_ms=mean_ms,
+        p50_ms=float(np.percentile(values, 50)),
+        p95_ms=float(np.percentile(values, 95)),
+        throughput=float(1000.0 / mean_ms) if mean_ms > 0 else 0.0,
+    )

cnn_classifier/evaluation/metrics.py

@@ -0,0 +1,94 @@
+"""Classification metrics with JSON-friendly reports."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+
+from cnn_classifier.evaluation.calibration import CalibrationReport, expected_calibration_error
+
+
+@dataclass(frozen=True)
+class ClassMetrics:
+    precision: float
+    recall: float
+    f1: float
+    support: int
+
+
+@dataclass(frozen=True)
+class MetricsReport:
+    accuracy: float
+    top_k_accuracy: float
+    macro_f1: float
+    per_class: dict[str, ClassMetrics]
+    confusion_matrix: list[list[int]]
+    nll: float | None
+    calibration: CalibrationReport | None
+    support: int
+
+
+def accuracy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+    """Return top-1 accuracy for equal-length arrays."""
+    truth = np.asarray(y_true)
+    predictions = np.asarray(y_pred)
+    if len(truth) != len(predictions):
+        raise ValueError("y_true and y_pred must have the same length")
+    if len(truth) == 0:
+        return 0.0
+    return float(np.mean(truth == predictions))
+
+
+def classification_metrics(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    probabilities: np.ndarray,
+    class_names: list[str] | tuple[str, ...],
+    top_k: int = 3,
+) -> MetricsReport:
+    """Compute top-k, per-class, confusion, NLL, and calibration metrics."""
+    truth = np.asarray(y_true, dtype=np.int64)
+    predictions = np.asarray(y_pred, dtype=np.int64)
+    probs = np.asarray(probabilities, dtype=np.float64)
+    names = tuple(class_names)
+    if len(truth) != len(predictions) or len(truth) != len(probs):
+        raise ValueError("y_true, y_pred, and probabilities must have equal length")
+    if probs.ndim != 2 or probs.shape[1] != len(names):
+        raise ValueError("probabilities must be [N, len(class_names)]")
+    count = len(names)
+    confusion = [[0 for _ in range(count)] for _ in range(count)]
+    for true_label, predicted in zip(truth, predictions, strict=True):
+        confusion[int(true_label)][int(predicted)] += 1
+    per_class: dict[str, ClassMetrics] = {}
+    f1_scores: list[float] = []
+    for index, name in enumerate(names):
+        true_positive = confusion[index][index]
+        false_positive = sum(confusion[row][index] for row in range(count) if row != index)
+        false_negative = sum(confusion[index][column] for column in range(count) if column != index)
+        predicted_positive = true_positive + false_positive
+        actual_positive = true_positive + false_negative
+        precision = true_positive / predicted_positive if predicted_positive else 0.0
+        recall = true_positive / actual_positive if actual_positive else 0.0
+        f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
+        support = sum(confusion[index])
+        per_class[name] = ClassMetrics(
+            precision=float(precision), recall=float(recall), f1=float(f1), support=int(support)
+        )
+        f1_scores.append(f1)
+    top_k = max(1, min(top_k, count))
+    top_indices = np.argsort(probs, axis=1)[:, -top_k:]
+    top_k_accuracy = float(
+        np.mean([truth[index] in top_indices[index] for index in range(len(truth))])
+    )
+    nll = float(-np.mean(np.log(np.clip(probs[np.arange(len(truth)), truth], 1e-12, 1.0))))
+    return MetricsReport(
+        accuracy=accuracy(truth, predictions),
+        top_k_accuracy=top_k_accuracy,
+        macro_f1=float(np.mean(f1_scores)),
+        per_class=per_class,
+        confusion_matrix=confusion,
+        nll=nll,
+        calibration=expected_calibration_error(probs, truth),
+        support=len(truth),
+    )

cnn_classifier/evaluation/predict.py

@@ -0,0 +1,64 @@
+"""Single-image and folder prediction."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import torch
+from PIL import Image
+from torch import nn
+
+from cnn_classifier.core.errors import DataError
+
+
+@dataclass(frozen=True)
+class Predictions:
+    paths: tuple[Path, ...]
+    probabilities: np.ndarray
+    predicted_labels: np.ndarray
+    class_names: tuple[str, ...]
+
+    def top_k(self, k: int = 3) -> list[list[tuple[str, float]]]:
+        order = np.argsort(self.probabilities, axis=1)[:, -k:][:, ::-1]
+        return [
+            [(self.class_names[int(index)], float(row[index])) for index in row] for row in order
+        ]
+
+
+@torch.no_grad()
+def predict_paths(
+    model: nn.Module,
+    paths: Sequence[Path],
+    transform: Any,
+    device: torch.device | str,
+    class_names: Sequence[str],
+) -> Predictions:
+    """Predict a sequence of images without requiring a DataLoader."""
+    runtime_device = torch.device(device) if isinstance(device, str) else device
+    was_training = model.training
+    model.eval()
+    batch = []
+    for path in paths:
+        try:
+            with Image.open(path) as image:
+                tensor = transform(image.copy())
+        except (OSError, ValueError) as exc:
+            raise DataError(f"cannot read prediction image {path}: {exc}") from exc
+        if tensor.ndim == 3:
+            tensor = tensor.unsqueeze(0)
+        batch.append(tensor)
+    if not batch:
+        raise DataError("predict_paths received no paths")
+    probabilities = torch.softmax(model(torch.cat(batch).to(runtime_device)), dim=1)
+    model.train(was_training)
+    labels = probabilities.argmax(dim=1).cpu().numpy()
+    return Predictions(
+        paths=tuple(Path(path) for path in paths),
+        probabilities=probabilities.cpu().numpy(),
+        predicted_labels=np.asarray(labels),
+        class_names=tuple(class_names),
+    )

cnn_classifier/models/__init__.py

@@ -0,0 +1,50 @@
+"""Model builders and registry."""
+
+from cnn_classifier.models.base import (
+    MODELS,
+    ModelBuildContext,
+    ModelBuilder,
+    build_model,
+    register_model,
+)
+from cnn_classifier.models.residual_cnn import (
+    ResidualCNN,
+    ResidualCNNBuilder,
+    ResidualCNNParams,
+    StageConfig,
+    StemConfig,
+)
+from cnn_classifier.models.simple_cnn import SimpleCNN, SimpleCNNBuilder, SimpleCNNParams
+from cnn_classifier.models.summary import model_summary
+from cnn_classifier.models.torchvision import TorchvisionBuilder, TorchvisionParams
+from cnn_classifier.models.transfer import (
+    apply_stem_adaptation,
+    apply_transfer,
+    build_param_groups,
+    replace_classifier,
+    unfreeze_modules,
+)
+
+__all__ = [
+    "MODELS",
+    "ModelBuildContext",
+    "ModelBuilder",
+    "ResidualCNN",
+    "ResidualCNNBuilder",
+    "ResidualCNNParams",
+    "SimpleCNN",
+    "SimpleCNNBuilder",
+    "SimpleCNNParams",
+    "StageConfig",
+    "StemConfig",
+    "TorchvisionBuilder",
+    "TorchvisionParams",
+    "apply_stem_adaptation",
+    "apply_transfer",
+    "build_model",
+    "build_param_groups",
+    "model_summary",
+    "register_model",
+    "replace_classifier",
+    "unfreeze_modules",
+]

cnn_classifier/models/base.py

@@ -0,0 +1,62 @@
+"""Model builder protocol, registry, and construction entry point."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from pydantic import BaseModel
+from torch import nn
+
+from cnn_classifier.config.schema import ModelConfig, TransferConfig
+from cnn_classifier.core.errors import ConfigError, RegistryError
+from cnn_classifier.core.registry import Registry
+
+
+@dataclass(frozen=True)
+class ModelBuildContext:
+    num_classes: int
+    in_channels: int
+    image_size: tuple[int, int]
+    transfer: TransferConfig
+
+
+class ModelBuilder(Protocol):
+    Params: type[BaseModel]
+
+    @staticmethod
+    def build(params: Any, context: ModelBuildContext) -> nn.Module:
+        """Construct a model from validated parameters and context."""
+        ...
+
+
+MODELS: Registry[type[Any]] = Registry("model")
+
+
+def register_model(name: str) -> Callable[[type[Any]], type[Any]]:
+    """Register a model builder class under ``name``."""
+    return MODELS.register(name)
+
+
+def build_model(
+    config: ModelConfig,
+    *,
+    num_classes: int,
+    in_channels: int,
+    image_size: tuple[int, int],
+) -> nn.Module:
+    """Resolve, validate, and construct the configured model."""
+    try:
+        builder = MODELS.get(config.name)
+    except RegistryError as exc:
+        raise ConfigError(f"unknown model {config.name!r}") from exc
+    params = builder.Params.model_validate(config.params)
+    context = ModelBuildContext(
+        num_classes=num_classes,
+        in_channels=in_channels,
+        image_size=image_size,
+        transfer=config.transfer,
+    )
+    model: nn.Module = builder.build(params, context)
+    return model

cnn_classifier/models/residual_cnn.py

@@ -0,0 +1,138 @@
+"""Configurable residual CNN with an explicit stem and stage description."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+import torch
+from pydantic import BaseModel, ConfigDict, Field
+from torch import nn
+
+from cnn_classifier.models.base import ModelBuildContext, register_model
+
+
+class StemConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
+
+    channels: int = Field(gt=0)
+    kernel_size: int = Field(default=3, ge=1, le=7)
+    stride: int = Field(default=1, ge=1)
+    pool: bool = False
+
+
+class StageConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
+
+    channels: int = Field(gt=0)
+    conv: bool = True
+    blocks: int = Field(default=1, ge=1)
+    downsample: Literal["none", "maxpool", "stride"] = "none"
+
+
+class ResidualCNNParams(BaseModel):
+    model_config = ConfigDict(extra="forbid", allow_inf_nan=False)
+
+    stem: StemConfig
+    stages: tuple[StageConfig, ...] = Field(min_length=1)
+    dropout: float = Field(default=0.3, ge=0.0, lt=1.0)
+    norm: Literal["batch", "none"] = "batch"
+    residual: bool = True
+
+
+def _norm_layer(channels: int, norm: str) -> nn.Module:
+    if norm == "batch":
+        return nn.BatchNorm2d(channels)
+    return nn.Identity()
+
+
+class ResidualBlock(nn.Module):
+    def __init__(self, channels: int, norm: str, residual: bool) -> None:
+        super().__init__()
+        bias = norm == "none"
+        self.residual = residual
+        self.conv1 = nn.Conv2d(channels, channels, 3, 1, 1, bias=bias)
+        self.norm1 = _norm_layer(channels, norm)
+        self.conv2 = nn.Conv2d(channels, channels, 3, 1, 1, bias=bias)
+        self.norm2 = _norm_layer(channels, norm)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        out = torch.relu(self.norm1(self.conv1(x)))
+        out = self.norm2(self.conv2(out))
+        if self.residual:
+            out = out + x
+        return torch.relu(out)
+
+
+class ResidualCNN(nn.Module):
+    def __init__(self, params: ResidualCNNParams, context: ModelBuildContext) -> None:
+        super().__init__()
+        layers: list[nn.Module] = []
+        stem = params.stem
+        layers.append(
+            nn.Conv2d(
+                context.in_channels,
+                stem.channels,
+                stem.kernel_size,
+                stem.stride,
+                stem.kernel_size // 2,
+                bias=params.norm == "none",
+            )
+        )
+        layers.append(_norm_layer(stem.channels, params.norm))
+        layers.append(nn.ReLU(inplace=True))
+        if stem.pool:
+            layers.append(nn.MaxPool2d(2))
+        stage_spans: list[tuple[str, int, int, bool]] = [("stem", 0, len(layers), False)]
+        current = stem.channels
+        for stage_index, stage in enumerate(params.stages, start=1):
+            start = len(layers)
+            if stage.conv:
+                layers.append(
+                    nn.Conv2d(current, stage.channels, 3, 1, 1, bias=params.norm == "none")
+                )
+                layers.append(_norm_layer(stage.channels, params.norm))
+                layers.append(nn.ReLU(inplace=True))
+                current = stage.channels
+            elif stage.channels != current:
+                raise ValueError("a stage without conv must keep the channel count")
+            for _ in range(stage.blocks):
+                layers.append(ResidualBlock(current, params.norm, params.residual))
+            if stage.downsample == "maxpool":
+                layers.append(nn.MaxPool2d(2))
+            elif stage.downsample == "stride":
+                layers.append(
+                    nn.Sequential(
+                        nn.Conv2d(current, current, 3, 2, 1, bias=params.norm == "none"),
+                        _norm_layer(current, params.norm),
+                        nn.ReLU(inplace=True),
+                    )
+                )
+            stage_spans.append(
+                (
+                    f"stage {stage_index}",
+                    start,
+                    len(layers),
+                    params.residual and stage.blocks > 0,
+                )
+            )
+        self.stage_spans = tuple(stage_spans)
+        self.features = nn.Sequential(*layers)
+        self.head = nn.Sequential(
+            nn.AdaptiveAvgPool2d(1),
+            nn.Flatten(),
+            nn.Dropout(params.dropout),
+            nn.Linear(current, context.num_classes),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        result: torch.Tensor = self.head(self.features(x))
+        return result
+
+
+@register_model("residual_cnn")
+class ResidualCNNBuilder:
+    Params = ResidualCNNParams
+
+    @staticmethod
+    def build(params: ResidualCNNParams, context: ModelBuildContext) -> nn.Module:
+        return ResidualCNN(params, context)