quadra 0.0.1__py3-none-any.whl → 2.2.7__py3-none-any.whl

quadra/trainers/classification.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from typing import cast
+
+import numpy as np
+import pandas as pd
+import torch
+from numpy import ndarray
+from pandas import DataFrame
+from sklearn.linear_model import LogisticRegression
+from sklearn.linear_model._base import ClassifierMixin
+from torch.utils.data import DataLoader
+
+from quadra.utils import utils
+from quadra.utils.classification import get_results
+from quadra.utils.models import get_feature
+
+log = utils.get_logger(__name__)
+
+
+class SklearnClassificationTrainer:
+    """Class to configure and run a classification using torch for feature extraction and sklearn to fit a classifier.
+
+    Args:
+        input_shape: [H, W, C]
+        random_state: seed to fix randomness
+        classifier: classification model
+        iteration_over_training: the number of iteration over training during feature extraction
+        backbone: the feature extractor
+    """
+
+    def __init__(
+        self,
+        input_shape: list,
+        backbone: torch.nn.Module,
+        random_state: int = 42,
+        classifier: ClassifierMixin = LogisticRegression,
+        iteration_over_training: int = 1,
+    ) -> None:
+        super().__init__()
+
+        try:
+            self.classifier = classifier(max_iter=1e4, random_state=random_state)
+        except Exception:
+            self.classifier = classifier
+
+        self.input_shape = input_shape
+        self.random_state = random_state
+        self.iteration_over_training = iteration_over_training
+        self.backbone = backbone
+
+    def change_backbone(self, backbone: torch.nn.Module):
+        """Update feature extractor."""
+        self.backbone = backbone
+        self.backbone.eval()
+
+    def change_classifier(self, classifier: ClassifierMixin):
+        """Update classifier."""
+        self.classifier = classifier
+
+    def fit(
+        self,
+        train_dataloader: DataLoader | None = None,
+        train_features: ndarray | None = None,
+        train_labels: ndarray | None = None,
+    ):
+        """Fit classifier on training set."""
+        # Extract feature
+        if self.backbone is None:
+            raise AssertionError("You must set a model before running execution")
+
+        if train_dataloader is not None:  # train_features is None or train_labels is None:
+            log.info("Extracting features from training set")
+            train_features, train_labels, _ = get_feature(
+                feature_extractor=self.backbone,
+                dl=train_dataloader,
+                iteration_over_training=self.iteration_over_training,
+                gradcam=False,
+            )
+        else:
+            log.info("Using cached features for training set")
+            # With the current implementation cached features are not sorted
+            # Even though it doesn't seem to change anything
+            if train_features is None or train_labels is None:
+                raise AssertionError("Train features and labels must be provided when using cached data")
+            permuted_indices = np.random.RandomState(seed=self.random_state).permutation(train_features.shape[0])
+            train_features = train_features[permuted_indices]
+            train_labels = train_labels[permuted_indices]
+
+        log.info("Fitting classifier on %d features", len(train_features))  # type: ignore[arg-type]
+        self.classifier.fit(train_features, train_labels)
+
+    def test(
+        self,
+        test_dataloader: DataLoader,
+        test_labels: ndarray | None = None,
+        test_features: ndarray | None = None,
+        class_to_keep: list[int] | None = None,
+        idx_to_class: dict[int, str] | None = None,
+        predict_proba: bool = True,
+        gradcam: bool = False,
+    ) -> (
+        tuple[str | dict, DataFrame, float, DataFrame, np.ndarray | None]
+        | tuple[None, None, None, DataFrame, np.ndarray | None]
+    ):
+        """Test classifier on test set.
+
+        Args:
+            test_dataloader: Test dataloader
+            test_labels: test labels
+            test_features: Optional test features used when cache data is available
+            class_to_keep: list of class to keep
+            idx_to_class: dictionary mapping class index to class name
+            predict_proba: if True, predict also probability for each test image
+            gradcam: Whether to compute gradcam
+
+        Returns:
+            cl_rep: Classification report
+            pd_cm: Confusion matrix dataframe
+            accuracy: Test accuracy
+            res: Test results
+            cams: Gradcams
+        """
+        cams = None
+        # Extract feature
+        if test_features is None:
+            log.info("Extracting features from test set")
+            test_features, final_test_labels, cams = get_feature(
+                feature_extractor=self.backbone,
+                dl=test_dataloader,
+                gradcam=gradcam,
+                classifier=self.classifier,
+                input_shape=(self.input_shape[2], self.input_shape[0], self.input_shape[1]),
+            )
+        else:
+            if test_labels is None:
+                raise ValueError("Test labels must be provided when using cached data")
+            log.info("Using cached features for test set")
+            final_test_labels = test_labels
+
+        # Run classifier
+        log.info("Predict classifier on test set")
+        test_prediction_label = self.classifier.predict(test_features)
+        if predict_proba:
+            test_probability = self.classifier.predict_proba(test_features)
+            test_probability = test_probability.max(axis=1)
+
+        if class_to_keep is not None:
+            if idx_to_class is None:
+                raise ValueError("You must provide `idx_to_class` and `test_labels` when using `class_to_keep`")
+            filtered_test_labels = [int(x) if idx_to_class[x] in class_to_keep else -1 for x in final_test_labels]
+        else:
+            filtered_test_labels = cast(list[int], final_test_labels.tolist())
+
+        if not hasattr(test_dataloader.dataset, "x"):
+            raise ValueError("Current dataset doesn't provide an `x` attribute")
+
+        res = pd.DataFrame(
+            {
+                "sample": list(test_dataloader.dataset.x),
+                "real_label": final_test_labels,
+                "pred_label": test_prediction_label,
+            }
+        )
+
+        if not all(t == -1 for t in filtered_test_labels):
+            test_real_label_cm = np.array(filtered_test_labels)
+            if cams is not None:
+                cams = cams[test_real_label_cm != -1]  # TODO: Is class_to_keep still used?
+            pred_labels_cm = np.array(test_prediction_label)[test_real_label_cm != -1]
+            test_real_label_cm = test_real_label_cm[test_real_label_cm != -1].astype(pred_labels_cm.dtype)
+            cl_rep, pd_cm, accuracy = get_results(test_real_label_cm, pred_labels_cm, idx_to_class)
+
+            if predict_proba:
+                res["probability"] = test_probability
+
+            return cl_rep, pd_cm, accuracy, res, cams
+
+        return None, None, None, res, cams

quadra/utils/anomaly.py

@@ -0,0 +1,112 @@
+"""Anomaly Score Normalization Callback that uses min-max normalization."""
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+try:
+    from typing import Any, TypeAlias
+except ImportError:
+    from typing import Any
+
+    from typing_extensions import TypeAlias  # noqa
+
+
+# MyPy wants TypeAlias, but pylint has problems dealing with it
+import numpy as np  # pylint: disable=unused-import
+import pytorch_lightning as pl
+import torch  # pylint: disable=unused-import
+from anomalib.models.components import AnomalyModule
+from pytorch_lightning import Callback
+from pytorch_lightning.utilities.types import STEP_OUTPUT
+
+# https://github.com/python/cpython/issues/90015#issuecomment-1172996118
+MapOrValue: TypeAlias = "float | torch.Tensor | np.ndarray"
+
+
+def normalize_anomaly_score(raw_score: MapOrValue, threshold: float) -> MapOrValue:
+    """Normalize anomaly score value or map based on threshold.
+
+    Args:
+        raw_score: Raw anomaly score valure or map
+        threshold: Threshold for anomaly detection
+
+    Returns:
+        Normalized anomaly score value or map clipped between 0 and 1000
+    """
+    if threshold > 0:
+        normalized_score = (raw_score / threshold) * 100.0
+    elif threshold == 0:
+        # TODO: Is this the best way to handle this case?
+        normalized_score = (raw_score + 1) * 100.0
+    else:
+        normalized_score = 200.0 - ((raw_score / threshold) * 100.0)
+
+    if isinstance(normalized_score, torch.Tensor):
+        return torch.clamp(normalized_score, 0.0, 1000.0)
+
+    return np.clip(normalized_score, 0.0, 1000.0)
+
+
+class ThresholdNormalizationCallback(Callback):
+    """Callback that normalizes the image-level and pixel-level anomaly scores dividing by the threshold value.
+
+    Args:
+        threshold_type: Threshold used to normalize pixel level anomaly scores, either image or pixel (default)
+    """
+
+    def __init__(self, threshold_type: str = "pixel"):
+        super().__init__()
+        self.threshold_type = threshold_type
+
+    def on_test_start(self, trainer: pl.Trainer, pl_module: AnomalyModule) -> None:
+        """Called when the test begins."""
+        del trainer  # `trainer` variable is not used.
+
+        for metric in (pl_module.image_metrics, pl_module.pixel_metrics):
+            if metric is not None:
+                metric.set_threshold(100.0)
+
+    def on_test_batch_end(
+        self,
+        trainer: pl.Trainer,
+        pl_module: AnomalyModule,
+        outputs: STEP_OUTPUT | None,
+        batch: Any,
+        batch_idx: int,
+        dataloader_idx: int = 0,
+    ) -> None:
+        """Called when the test batch ends, normalizes the predicted scores and anomaly maps."""
+        del trainer, batch, batch_idx, dataloader_idx  # These variables are not used.
+
+        self._normalize_batch(outputs, pl_module)
+
+    def on_predict_batch_end(
+        self,
+        trainer: pl.Trainer,
+        pl_module: AnomalyModule,
+        outputs: Any,
+        batch: Any,
+        batch_idx: int,
+        dataloader_idx: int = 0,
+    ) -> None:
+        """Called when the predict batch ends, normalizes the predicted scores and anomaly maps."""
+        del trainer, batch, batch_idx, dataloader_idx  # These variables are not used.
+
+        self._normalize_batch(outputs, pl_module)
+
+    def _normalize_batch(self, outputs, pl_module):
+        """Normalize a batch of predictions."""
+        image_threshold = pl_module.image_threshold.value.cpu()
+        pixel_threshold = pl_module.pixel_threshold.value.cpu()
+        outputs["pred_scores"] = normalize_anomaly_score(outputs["pred_scores"], image_threshold.item())
+
+        threshold = pixel_threshold if self.threshold_type == "pixel" else image_threshold
+        threshold = threshold.item()
+
+        if "anomaly_maps" in outputs:
+            outputs["anomaly_maps"] = normalize_anomaly_score(outputs["anomaly_maps"], threshold)
+
+        if "box_scores" in outputs:
+            outputs["box_scores"] = [normalize_anomaly_score(scores, threshold) for scores in outputs["box_scores"]]