cnn-classifier-toolkit 0.1.0__py3-none-any.whl

cnn_classifier/diagnostics/architecture.py

@@ -0,0 +1,172 @@
+"""Stage-level architecture descriptors for visualization."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import cast
+
+from torch import nn
+
+from cnn_classifier.diagnostics.tracing import LayerShape, trace_shapes
+
+
+@dataclass(frozen=True)
+class ArchitectureStage:
+    name: str
+    kind: str
+    output_shape: tuple[int, ...]
+    params: int
+    trainable_params: int
+    residual: bool = False
+
+    @property
+    def frozen(self) -> bool:
+        return self.trainable_params == 0
+
+
+def _output_shape(record: LayerShape | None) -> tuple[int, ...]:
+    if record is None:
+        return ()
+    shape = tuple(int(value) for value in record.output_shape)
+    if len(shape) > 1 and shape[0] == 1:
+        return shape[1:]
+    return shape
+
+
+def _stage(
+    name: str,
+    kind: str,
+    modules: list[nn.Module],
+    records: dict[str, LayerShape],
+    record_name: str,
+    *,
+    residual: bool = False,
+) -> ArchitectureStage:
+    params = sum(parameter.numel() for module in modules for parameter in module.parameters())
+    trainable = sum(
+        parameter.numel()
+        for module in modules
+        for parameter in module.parameters()
+        if parameter.requires_grad
+    )
+    return ArchitectureStage(
+        name=name,
+        kind=kind,
+        output_shape=_output_shape(records.get(record_name)),
+        params=params,
+        trainable_params=trainable,
+        residual=residual,
+    )
+
+
+def _sequential_stages(model: nn.Module, records: dict[str, LayerShape]) -> list[ArchitectureStage]:
+    features = cast(nn.Sequential, model.features)
+    groups: list[list[tuple[int, nn.Module]]] = []
+    current: list[tuple[int, nn.Module]] = []
+    for index, module in enumerate(features.children()):
+        current.append((index, module))
+        if isinstance(module, nn.MaxPool2d):
+            groups.append(current)
+            current = []
+    if current:
+        groups.append(current)
+    stages = [
+        _stage(
+            f"stage {index}",
+            "stage",
+            [module for _, module in group],
+            records,
+            f"features.{group[-1][0]}",
+        )
+        for index, group in enumerate(groups, start=1)
+    ]
+    stages.append(_stage("head", "head", _children(cast(nn.Module, model.head)), records, "head"))
+    return stages
+
+
+def _children(module: nn.Module) -> list[nn.Module]:
+    if isinstance(module, nn.Sequential):
+        return list(module.children())
+    return [module]
+
+
+def _residual_stages(model: nn.Module, records: dict[str, LayerShape]) -> list[ArchitectureStage]:
+    features = cast(nn.Sequential, model.features)
+    spans = cast(tuple[tuple[str, int, int, bool], ...], model.stage_spans)
+    stages = [
+        _stage(
+            name,
+            "stem" if name == "stem" else "stage",
+            [features[index] for index in range(start, end)],
+            records,
+            f"features.{end - 1}",
+            residual=residual,
+        )
+        for name, start, end, residual in spans
+    ]
+    stages.append(_stage("head", "head", _children(cast(nn.Module, model.head)), records, "head"))
+    return stages
+
+
+def _torchvision_stages(
+    model: nn.Module, records: dict[str, LayerShape]
+) -> list[ArchitectureStage]:
+    children = dict(model.named_children())
+    stem_names = [name for name in ("conv1", "bn1", "relu", "maxpool") if name in children]
+    stages = [
+        _stage(
+            "stem",
+            "stem",
+            [children[name] for name in stem_names],
+            records,
+            stem_names[-1],
+        )
+    ]
+    for name in ("layer1", "layer2", "layer3", "layer4"):
+        if name in children:
+            stages.append(
+                _stage(
+                    name.replace("layer", "layer "),
+                    "stage",
+                    [children[name]],
+                    records,
+                    name,
+                    residual=True,
+                )
+            )
+    head_names: list[str] = [name for name in ("avgpool", "fc") if name in children]
+    if not head_names:
+        head_names = list(children)
+    stages.append(
+        _stage(
+            "head",
+            "head",
+            [children[name] for name in head_names],
+            records,
+            head_names[-1],
+        )
+    )
+    return stages
+
+
+def architecture_stages(
+    model: nn.Module, input_shape: tuple[int, int, int]
+) -> list[ArchitectureStage]:
+    """Return concise stage descriptors for a built-in CNN.
+
+    The built-in ``simple_cnn`` and ``residual_cnn`` families and torchvision
+    ResNet models get precise grouping. Other models fall back to their
+    top-level children.
+    """
+    records = {record.name: record for record in trace_shapes(model, input_shape)}
+    if hasattr(model, "stage_spans") and hasattr(model, "features"):
+        return _residual_stages(model, records)
+    if hasattr(model, "features") and isinstance(model.features, nn.Sequential):
+        return _sequential_stages(model, records)
+    children = dict(model.named_children())
+    if "conv1" in children and "layer1" in children:
+        return _torchvision_stages(model, records)
+    stages = [_stage(name, "stage", [module], records, name) for name, module in children.items()]
+    if not stages:
+        raise ValueError("model has no submodules to visualize")
+    return stages

cnn_classifier/diagnostics/determinism.py

@@ -0,0 +1,25 @@
+"""Determinism check for a seeded callable."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+import torch
+
+from cnn_classifier.core.seeding import seed_everything
+
+
+def _equal(left: Any, right: Any) -> bool:
+    if torch.is_tensor(left) or torch.is_tensor(right):
+        return bool(torch.equal(torch.as_tensor(left), torch.as_tensor(right)))
+    return bool(left == right)
+
+
+def determinism_check(train_fn: Callable[[], Any], seed: int) -> bool:
+    """Run a callable twice with the same seed and compare its result."""
+    seed_everything(seed)
+    first = train_fn()
+    seed_everything(seed)
+    second = train_fn()
+    return _equal(first, second)

cnn_classifier/diagnostics/gradients.py

@@ -0,0 +1,32 @@
+"""Gradient statistics."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+
+
+@dataclass(frozen=True)
+class GradientStat:
+    name: str
+    norm: float
+    zero_fraction: float
+
+
+def gradient_report(model: nn.Module) -> list[GradientStat]:
+    """Return norm and zero fraction for every parameter with a gradient."""
+    report: list[GradientStat] = []
+    for name, parameter in model.named_parameters():
+        if parameter.grad is None:
+            continue
+        gradient = parameter.grad.detach()
+        report.append(
+            GradientStat(
+                name=name,
+                norm=float(torch.linalg.vector_norm(gradient).item()),
+                zero_fraction=float((gradient == 0).float().mean().item()),
+            )
+        )
+    return report

cnn_classifier/diagnostics/sanity.py

@@ -0,0 +1,49 @@
+"""One-batch overfit sanity check."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+from torch import nn
+
+
+@dataclass(frozen=True)
+class OverfitResult:
+    initial_loss: float
+    final_loss: float
+    steps: int
+
+
+def overfit_batch(
+    model: nn.Module,
+    batch: tuple[torch.Tensor, ...],
+    loss_fn: nn.Module,
+    optimizer: torch.optim.Optimizer,
+    device: torch.device,
+    max_steps: int = 100,
+    target_loss: float = 0.01,
+) -> OverfitResult:
+    """Train one batch to a small loss, restoring the model mode."""
+    images = batch[0].to(device)
+    labels = batch[1].to(device)
+    was_training = model.training
+    model.train()
+    initial_loss = 0.0
+    final_loss = 0.0
+    steps = 0
+    try:
+        for step in range(1, max_steps + 1):
+            optimizer.zero_grad(set_to_none=True)
+            loss = loss_fn(model(images), labels)
+            if step == 1:
+                initial_loss = float(loss.item())
+            loss.backward()
+            optimizer.step()
+            final_loss = float(loss.item())
+            steps = step
+            if final_loss <= target_loss:
+                break
+    finally:
+        model.train(was_training)
+    return OverfitResult(initial_loss=initial_loss, final_loss=final_loss, steps=steps)

cnn_classifier/diagnostics/tracing.py

@@ -0,0 +1,56 @@
+"""Shape tracing with forward hooks."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+from torch import nn
+
+
+@dataclass(frozen=True)
+class LayerShape:
+    name: str
+    type: str
+    input_shape: tuple[int, ...]
+    output_shape: tuple[int, ...]
+    params: int
+
+
+def trace_shapes(model: nn.Module, input_shape: tuple[int, int, int]) -> list[LayerShape]:
+    """Run one dummy input and report each module's shapes."""
+    parameter = next(model.parameters(), None)
+    device = parameter.device if parameter is not None else torch.device("cpu")
+    was_training = model.training
+    records: list[LayerShape] = []
+    handles: list[Any] = []
+
+    def hook(name: str, module: nn.Module) -> Callable[[nn.Module, Any, Any], None]:
+        def _record(_module: nn.Module, inputs: Any, output: Any) -> None:
+            input_tensor = inputs[0] if isinstance(inputs, tuple) else inputs
+            output_tensor = output[0] if isinstance(output, tuple) else output
+            records.append(
+                LayerShape(
+                    name=name or "model",
+                    type=type(module).__name__,
+                    input_shape=tuple(int(value) for value in input_tensor.shape),
+                    output_shape=tuple(int(value) for value in output_tensor.shape),
+                    params=sum(p.numel() for p in module.parameters()),
+                )
+            )
+
+        return _record
+
+    for name, module in model.named_modules():
+        handles.append(module.register_forward_hook(hook(name, module)))
+    try:
+        model.eval()
+        with torch.no_grad():
+            model(torch.zeros(1, *input_shape, device=device))
+    finally:
+        for handle in handles:
+            handle.remove()
+        model.train(was_training)
+    return records

cnn_classifier/evaluation/__init__.py

@@ -0,0 +1,53 @@
+"""Evaluation metrics, calibration, bootstrap, comparisons, and inference."""
+
+from cnn_classifier.evaluation.bootstrap import (
+    ConfidenceInterval,
+    bootstrap_ci,
+    paired_bootstrap_difference,
+)
+from cnn_classifier.evaluation.calibration import (
+    CalibrationBin,
+    CalibrationReport,
+    expected_calibration_error,
+)
+from cnn_classifier.evaluation.error_analysis import (
+    ErrorExample,
+    confusion_pairs,
+    most_confident_errors,
+)
+from cnn_classifier.evaluation.evaluator import (
+    EvaluationResult,
+    evaluate_model,
+    save_predictions,
+)
+from cnn_classifier.evaluation.latency import LatencyReport, measure_latency
+from cnn_classifier.evaluation.metrics import (
+    ClassMetrics,
+    MetricsReport,
+    accuracy,
+    classification_metrics,
+)
+from cnn_classifier.evaluation.predict import Predictions, predict_paths
+
+__all__ = [
+    "CalibrationBin",
+    "CalibrationReport",
+    "ClassMetrics",
+    "ConfidenceInterval",
+    "ErrorExample",
+    "EvaluationResult",
+    "LatencyReport",
+    "MetricsReport",
+    "Predictions",
+    "accuracy",
+    "bootstrap_ci",
+    "classification_metrics",
+    "confusion_pairs",
+    "evaluate_model",
+    "expected_calibration_error",
+    "measure_latency",
+    "most_confident_errors",
+    "paired_bootstrap_difference",
+    "predict_paths",
+    "save_predictions",
+]

cnn_classifier/evaluation/bootstrap.py

@@ -0,0 +1,96 @@
+"""Bootstrap confidence intervals and paired comparisons."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+import numpy as np
+
+MetricFunction = Callable[[np.ndarray, np.ndarray], float]
+
+
+@dataclass(frozen=True)
+class ConfidenceInterval:
+    low: float
+    high: float
+    estimate: float
+
+
+def _validate(y_true: np.ndarray, y_pred: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+    truth = np.asarray(y_true)
+    predictions = np.asarray(y_pred)
+    if len(truth) != len(predictions):
+        raise ValueError("y_true and y_pred must have the same length")
+    return truth, predictions
+
+
+def _resample_indices(
+    truth: np.ndarray,
+    rng: np.random.Generator,
+    stratify: bool,
+) -> np.ndarray:
+    if not stratify:
+        return rng.integers(0, len(truth), size=len(truth))
+    pieces = []
+    for label in np.unique(truth):
+        class_indices = np.flatnonzero(truth == label)
+        pieces.append(rng.choice(class_indices, size=len(class_indices), replace=True))
+    return np.concatenate(pieces)
+
+
+def bootstrap_ci(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    metric_fn: MetricFunction,
+    n_resamples: int = 1000,
+    confidence: float = 0.95,
+    seed: int = 0,
+    stratify: bool = False,
+) -> ConfidenceInterval:
+    """Return a percentile bootstrap interval for one prediction set."""
+    truth, predictions = _validate(y_true, y_pred)
+    if n_resamples <= 0:
+        raise ValueError("n_resamples must be positive")
+    if not 0.0 < confidence < 1.0:
+        raise ValueError("confidence must be in (0, 1)")
+    rng = np.random.default_rng(seed)
+    values = []
+    for _ in range(n_resamples):
+        indices = _resample_indices(truth, rng, stratify)
+        values.append(metric_fn(truth[indices], predictions[indices]))
+    alpha = (1.0 - confidence) / 2.0
+    low, high = np.percentile(values, [100 * alpha, 100 * (1 - alpha)])
+    return ConfidenceInterval(
+        low=float(low), high=float(high), estimate=float(metric_fn(truth, predictions))
+    )
+
+
+def paired_bootstrap_difference(
+    y_true: np.ndarray,
+    pred_a: np.ndarray,
+    pred_b: np.ndarray,
+    metric_fn: MetricFunction,
+    n_resamples: int = 1000,
+    confidence: float = 0.95,
+    seed: int = 0,
+    stratify: bool = False,
+) -> ConfidenceInterval:
+    """Return a bootstrap interval for ``metric(a) - metric(b)``."""
+    truth, predictions_a = _validate(y_true, pred_a)
+    _, predictions_b = _validate(y_true, pred_b)
+    rng = np.random.default_rng(seed)
+    values = []
+    for _ in range(n_resamples):
+        indices = _resample_indices(truth, rng, stratify)
+        values.append(
+            metric_fn(truth[indices], predictions_a[indices])
+            - metric_fn(truth[indices], predictions_b[indices])
+        )
+    alpha = (1.0 - confidence) / 2.0
+    low, high = np.percentile(values, [100 * alpha, 100 * (1 - alpha)])
+    return ConfidenceInterval(
+        low=float(low),
+        high=float(high),
+        estimate=float(metric_fn(truth, predictions_a) - metric_fn(truth, predictions_b)),
+    )

cnn_classifier/evaluation/calibration.py

@@ -0,0 +1,74 @@
+"""Calibration metrics: ECE, MCE, Brier score, and reliability bins."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+
+
+@dataclass(frozen=True)
+class CalibrationBin:
+    lower: float
+    upper: float
+    count: int
+    confidence: float
+    accuracy: float
+
+
+@dataclass(frozen=True)
+class CalibrationReport:
+    ece: float
+    mce: float
+    bins: list[CalibrationBin]
+    brier: float
+
+
+def expected_calibration_error(
+    probabilities: np.ndarray,
+    y_true: np.ndarray,
+    n_bins: int = 15,
+) -> CalibrationReport:
+    """Compute ECE, MCE, Brier score, and reliability bins."""
+    probs = np.asarray(probabilities, dtype=np.float64)
+    truth = np.asarray(y_true, dtype=np.int64)
+    if probs.ndim != 2 or len(probs) != len(truth):
+        raise ValueError("probabilities must be [N,C] and match y_true")
+    if n_bins <= 0:
+        raise ValueError("n_bins must be positive")
+    confidence = probs.max(axis=1)
+    predictions = probs.argmax(axis=1)
+    correct = (predictions == truth).astype(np.float64)
+    edges = np.linspace(0.0, 1.0, n_bins + 1)
+    bin_indices = np.clip(np.digitize(confidence, edges, right=False) - 1, 0, n_bins - 1)
+    bins: list[CalibrationBin] = []
+    ece = 0.0
+    mce = 0.0
+    total = len(truth)
+    for index in range(n_bins):
+        mask = bin_indices == index
+        count = int(mask.sum())
+        if count:
+            mean_conf = float(confidence[mask].mean())
+            mean_acc = float(correct[mask].mean())
+            gap = abs(mean_acc - mean_conf)
+        else:
+            mean_conf = 0.0
+            mean_acc = 0.0
+            gap = 0.0
+        bins.append(
+            CalibrationBin(
+                lower=float(edges[index]),
+                upper=float(edges[index + 1]),
+                count=count,
+                confidence=mean_conf,
+                accuracy=mean_acc,
+            )
+        )
+        if total:
+            ece += (count / total) * gap
+            mce = max(mce, gap)
+    one_hot = np.zeros_like(probs)
+    one_hot[np.arange(len(truth)), truth] = 1.0
+    brier = float(np.mean(np.sum((probs - one_hot) ** 2, axis=1)))
+    return CalibrationReport(ece=float(ece), mce=float(mce), bins=bins, brier=brier)

cnn_classifier/evaluation/compare.py

@@ -0,0 +1,111 @@
+"""Fair multi-run comparison with test-fingerprint enforcement."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from cnn_classifier.core.errors import RunError
+
+
+@dataclass(frozen=True)
+class ComparisonReport:
+    rows: list[dict[str, Any]]
+    warnings: list[str]
+    markdown: str
+    pareto_data: list[dict[str, Any]]
+
+
+def _load_run(run_dir: Path) -> tuple[str, dict[str, Any]]:
+    summary_path = run_dir / "summary.json"
+    fingerprint_path = run_dir / "data" / "fingerprint.json"
+    try:
+        summary = json.loads(summary_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise RunError(f"cannot read summary for {run_dir}: {exc}") from exc
+    fingerprint = summary.get("test_fingerprint") or summary.get("dataset_fingerprint")
+    if fingerprint_path.is_file():
+        try:
+            fingerprint = json.loads(fingerprint_path.read_text(encoding="utf-8")).get(
+                "fingerprint", fingerprint
+            )
+        except (OSError, json.JSONDecodeError) as exc:
+            raise RunError(f"cannot read fingerprint for {run_dir}: {exc}") from exc
+    if not fingerprint:
+        raise RunError(f"run {run_dir} has no test fingerprint")
+    return str(fingerprint), summary
+
+
+def _row(run_dir: Path, summary: dict[str, Any]) -> dict[str, Any]:
+    interval = summary.get("accuracy_ci") or [None, None]
+    return {
+        "run": run_dir.name,
+        "accuracy": summary.get("test_accuracy", summary.get("best_metric")),
+        "macro_f1": summary.get("macro_f1"),
+        "ci_low": interval[0],
+        "ci_high": interval[1],
+        "params": summary.get("params"),
+        "trainable_params": summary.get("trainable_params"),
+        "train_seconds": summary.get("train_seconds"),
+        "latency_ms": summary.get("latency_ms"),
+        "peak_memory_mb": summary.get("peak_memory_mb"),
+    }
+
+
+def compare_runs(
+    run_dirs: list[Path] | tuple[Path, ...],
+    output_dir: Path,
+) -> ComparisonReport:
+    """Compare runs only when they share the same test fingerprint."""
+    if not run_dirs:
+        raise RunError("compare_runs requires at least one run")
+    fingerprints: dict[str, str] = {}
+    summaries: list[tuple[Path, dict[str, Any]]] = []
+    for run_dir in run_dirs:
+        fingerprint, summary = _load_run(Path(run_dir))
+        fingerprints[run_dir.name] = fingerprint
+        summaries.append((Path(run_dir), summary))
+    if len(set(fingerprints.values())) != 1:
+        details = ", ".join(f"{name}={fingerprint}" for name, fingerprint in fingerprints.items())
+        raise RunError(f"test fingerprint mismatch across runs: {details}")
+    rows = sorted(
+        (_row(run_dir, summary) for run_dir, summary in summaries),
+        key=lambda row: float(row.get("accuracy") or 0.0),
+        reverse=True,
+    )
+    warnings: list[str] = []
+    for row in rows:
+        if row["accuracy"] is None:
+            warnings.append(f"{row['run']} has no accuracy")
+    header = "| Run | Accuracy | Macro F1 | Params | Latency (ms) |"
+    separator = "|---|---:|---:|---:|---:|"
+    lines = [header, separator]
+    for row in rows:
+        accuracy = row["accuracy"]
+        macro_f1 = row["macro_f1"]
+        lines.append(
+            f"| {row['run']} | "
+            f"{accuracy if accuracy is not None else 'n/a'} | "
+            f"{macro_f1 if macro_f1 is not None else 'n/a'} | "
+            f"{row['params'] if row['params'] is not None else 'n/a'} | "
+            f"{row['latency_ms'] if row['latency_ms'] is not None else 'n/a'} |"
+        )
+    markdown = "\n".join(lines) + "\n"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    (output_dir / "comparison.md").write_text(markdown, encoding="utf-8")
+    (output_dir / "comparison.json").write_text(
+        json.dumps({"rows": rows, "warnings": warnings}, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    pareto = [
+        {
+            "run": row["run"],
+            "accuracy": row["accuracy"],
+            "params": row["params"],
+            "latency_ms": row["latency_ms"],
+        }
+        for row in rows
+    ]
+    return ComparisonReport(rows=rows, warnings=warnings, markdown=markdown, pareto_data=pareto)