quadra 2.5.0__tar.gz → 2.5.1__tar.gz

{quadra-2.5.0 → quadra-2.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quadra
-Version: 2.5.0
+Version: 2.5.1
 Summary: Deep Learning experiment orchestration library
 License: Apache-2.0
 License-File: LICENSE

{quadra-2.5.0 → quadra-2.5.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "quadra"
-version = "2.5.0"
+version = "2.5.1"
 description = "Deep Learning experiment orchestration library"
 authors = [
 	"Federico Belotti <federico.belotti@orobix.com>",

{quadra-2.5.0 → quadra-2.5.1}/quadra/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.5.0"
+__version__ = "2.5.1"
 
 
 def get_version():

{quadra-2.5.0 → quadra-2.5.1}/quadra/tasks/anomaly.py

@@ -244,7 +244,7 @@ class AnomalibDetection(Generic[AnomalyDataModuleT], LightningTask[AnomalyDataMo
         ):
             threshold = torch.tensor(100.0)
         else:
-            threshold = self.module.image_metrics.F1Score.threshold
+            threshold = self.module.image_metrics.F1Score.threshold  # type: ignore[union-attr,assignment]
 
         # The output of the prediction is a normalized score so the cumulative histogram is displayed with the
         # normalized scores
@@ -328,6 +328,22 @@ class AnomalibDetection(Generic[AnomalyDataModuleT], LightningTask[AnomalyDataMo
                 else:
                     utils.upload_file_tensorboard(a, tensorboard_logger)
 
+    def execute(self):
+        """Execute the experiment and all the steps."""
+        self.prepare()
+        self.train()
+        # When training in fp16 mixed precision, export function casts model weights from fp32 to fp16,
+        # for this reason, predictions logits could slightly change and predictions could be inconsistent between
+        # test and generated report.
+        # Performing export before test allows to have consistent results in test metrics and generated report.
+        if self.config.export is not None and len(self.config.export.types) > 0:
+            self.export()
+        if self.run_test:
+            self.test()
+        if self.report:
+            self.generate_report()
+        self.finalize()
+
 
 class AnomalibEvaluation(Evaluation[AnomalyDataModule]):
     """Evaluation task for Anomalib.
@@ -445,12 +461,22 @@ class AnomalibEvaluation(Evaluation[AnomalyDataModule]):
         training_threshold = self.model_data[f"{self.training_threshold_type}_threshold"]
         optimal_threshold = self.metadata["threshold"]
 
-        normalized_optimal_threshold = cast(float, normalize_anomaly_score(optimal_threshold, training_threshold))
-
         os.makedirs(os.path.join(self.report_path, "predictions"), exist_ok=True)
         os.makedirs(os.path.join(self.report_path, "heatmaps"), exist_ok=True)
 
         anomaly_scores = self.metadata["anomaly_scores"].cpu().numpy()
+
+        # The reason I have to expand dims and cast the optimal threshold to anomaly_scores dtype is because
+        # of internal roundings performed differently by numpy and python
+        # Particularly the normalized_optimal_threshold computed directly using float values might be higher than the
+        # actual value obtained by the anomaly_scores
+        normalized_optimal_threshold = cast(
+            np.ndarray,
+            normalize_anomaly_score(
+                np.expand_dims(np.array(optimal_threshold, dtype=anomaly_scores.dtype), -1), training_threshold
+            ),
+        ).item()
+
         anomaly_scores = normalize_anomaly_score(anomaly_scores, training_threshold)
 
         if not isinstance(anomaly_scores, np.ndarray):

{quadra-2.5.0 → quadra-2.5.1}/quadra/utils/anomaly.py

@@ -5,6 +5,8 @@
 
 from __future__ import annotations
 
+from typing import cast
+
 try:
     from typing import Any, TypeAlias
 except ImportError:
@@ -43,6 +45,39 @@ def normalize_anomaly_score(raw_score: MapOrValue, threshold: float) -> MapOrVal
     else:
         normalized_score = 200.0 - ((raw_score / threshold) * 100.0)
 
+    # Ensures that the normalized scores are consistent with the raw scores
+    # For all the items whose prediction changes after normalization, force the normalized score to be
+    # consistent with the prediction made on the raw score by clipping the score:
+    #   - to 100.0 if the prediction was "anomaly" on the raw score and "good" on the normalized score
+    #   - to 99.99 if the prediction was "good" on the raw score and "anomaly" on the normalized score
+    score = raw_score
+    if isinstance(score, torch.Tensor):
+        score = score.cpu().numpy()
+    # Anomalib classify as anomaly if anomaly_score gte threshold
+    is_anomaly_mask = score >= threshold
+    is_not_anomaly_mask = np.bitwise_not(is_anomaly_mask)
+    if isinstance(normalized_score, torch.Tensor):
+        if normalized_score.dim() == 0:
+            normalized_score = (
+                normalized_score.clamp(min=100.0) if is_anomaly_mask else normalized_score.clamp(max=99.99)
+            )
+        else:
+            normalized_score[is_anomaly_mask] = normalized_score[is_anomaly_mask].clamp(min=100.0)
+            normalized_score[is_not_anomaly_mask] = normalized_score[is_not_anomaly_mask].clamp(max=99.99)
+    elif isinstance(normalized_score, np.ndarray) or np.isscalar(normalized_score):
+        if np.isscalar(normalized_score) or normalized_score.ndim == 0:  # type: ignore[union-attr]
+            normalized_score = (
+                np.clip(normalized_score, a_min=100.0, a_max=None)
+                if is_anomaly_mask
+                else np.clip(normalized_score, a_min=None, a_max=99.99)
+            )
+        else:
+            normalized_score = cast(np.ndarray, normalized_score)
+            normalized_score[is_anomaly_mask] = np.clip(normalized_score[is_anomaly_mask], a_min=100.0, a_max=None)
+            normalized_score[is_not_anomaly_mask] = np.clip(
+                normalized_score[is_not_anomaly_mask], a_min=None, a_max=99.99
+            )
+
     if isinstance(normalized_score, torch.Tensor):
         return torch.clamp(normalized_score, 0.0, 1000.0)