ssad 0.1.2__py3-none-any.whl

ssad/__init__.py

@@ -0,0 +1,39 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""
+Initializes the main package API by exposing public interfaces from submodules.
+
+This module re-exports the core components of the package to simplify access and
+maintain a clean and consistent public API.
+
+Available namespaces:
+- confidence_estimators
+- datamodules
+- datasets
+- distribution_analyzers
+- models
+- modules
+- loggers
+
+Usage:
+    from mypackage import SomeModel, SomeDataModule, ConfidenceEstimator
+
+Note:
+    This file uses wildcard imports (`*`) to expose only the public symbols defined
+    in each submodule's `__all__` list.
+"""
+from .confidence_estimators.api import *
+from .datamodules.api import *
+from .datasets.api import *
+from .distribution_analyzers.api import *
+from .models.api import *
+from .modules.api import *
+from .loggers.api import *

ssad/confidence_estimators/__init__.py

@@ -0,0 +1,15 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+r"""
+# What is confidence estimator
+A confidence estimator is a module that, given a value such as a reconstruction error,
+or a gradient value, returns a confidence score comprised between -1 and 1.
+"""

ssad/confidence_estimators/api.py

@@ -0,0 +1,16 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""Public API for self-supervision confidence estimators."""
+from .binary_confidence import BinaryConfidence
+from .supports_confidence_estimation import SupportsConfidenceEstimation
+from .confidence_intervals_configuration import ConfidenceIntervalsConfiguration
+
+__all__ = ["SupportsConfidenceEstimation", "BinaryConfidence", "ConfidenceIntervalsConfiguration"]

ssad/confidence_estimators/binary_confidence.py

@@ -0,0 +1,30 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""
+Implements a binary confidence estimator.
+A sample is either normal or abnormal, otherwise it is omitted (zero confidence).
+"""
+
+import torch
+from .confidence_estimator import BaseConfidenceEstimator
+
+
+class BinaryConfidence(BaseConfidenceEstimator):
+    """Binary confidence estimator"""
+
+    def _confidence_normal(self, score):
+        return torch.ones_like(score)
+
+    def _confidence_abnormal(self, score):
+        return torch.full_like(score, -1)
+
+    def _confidence_unknown(self, score):
+        return torch.zeros_like(score)

ssad/confidence_estimators/confidence_estimator.py

@@ -0,0 +1,94 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""
+Provides the base class for confidence estimators.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Optional
+import torch
+
+from .confidence_intervals_configuration import (
+    ConfidenceIntervalsConfiguration,
+)
+from .supports_confidence_estimation import SupportsConfidenceEstimation
+
+
+class BaseConfidenceEstimator(ABC, SupportsConfidenceEstimation):
+    """Base class for confidence estimators.
+    A confidence estimator relies on four intervals with associated scoring functions
+    to provide a confidence score given a criterion score for a sample.
+
+    The criterion score can be for instance a reconstruction score, or the norm of
+    the gradient of the reconstruction error.
+
+    The intervals define the domain for model scores for the four different confidence behaviors:
+    - normal: samples with model scores in this interval are considered as normal
+    - abnormal: samples with model scores in this interval are considered as abnormal
+    - unknown_positive: samples with model scores in this interval are
+        considered as unknown, but leaning towards a normal sample.
+    - unknown_negative: samples with model scores in this interval are
+        considered as unknown, but leaning towards an abnormal sample.
+
+    Each interval is associated with a "criterion score to confidence score" conversion function.
+    These functions should be implemented in the _estimate_confidence_from_model_score method.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.configuration: Optional[ConfidenceIntervalsConfiguration] = None
+        self.distribution: Optional[torch.Tensor] = None
+
+    @abstractmethod
+    def _confidence_normal(self, score):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _confidence_abnormal(self, score) -> torch.Tensor:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def _confidence_unknown(self, score) -> torch.Tensor:
+        raise NotImplementedError()
+
+    @torch.no_grad()
+    def estimate_confidence(self, scores_batch: torch.Tensor) -> torch.Tensor:
+        """Estimates the confidence in a batch by retrieving the criterion score
+        and translating into a confidence score.
+
+        Args:
+            scores_batch (torch.Tensor): batch whose confidence is to be estimated.
+
+        Returns:
+            torch.Tensor: confidence score
+        """
+        # TODO: check this order of computation
+        confidence = self._confidence_unknown(scores_batch)
+
+        if self.configuration is None:
+            raise ValueError("Confidence estimator configuration is None")
+
+        # TODO: rework signatures of confidence normal/abnormal/unknown
+        normal_confidences = self._confidence_normal(scores_batch)
+        abnormal_confidences = self._confidence_abnormal(scores_batch)
+
+        confidence = torch.where(
+            self.configuration.normal.contains_tensor_mask(scores_batch),
+            normal_confidences,
+            confidence,
+        )
+        confidence = torch.where(
+            self.configuration.abnormal.contains_tensor_mask(scores_batch),
+            abnormal_confidences,
+            confidence,
+        )
+
+        return confidence

ssad/confidence_estimators/confidence_intervals_configuration.py

@@ -0,0 +1,67 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""
+A Confidence Intervals configuration consists in a set of three intervals
+for the value of the analyzed distribution (e.g., reconstruction scores).
+These three intervals respectively define the intervals that correspond to
+normal samples, abnormal samples, and samples for which it is impossible to decide,
+labelled "unknown".
+"""
+
+from dataclasses import dataclass
+
+import pandas as pd
+import torch
+
+
+class Interval(pd.Interval):
+    """Extends pandas Intervals to add a method to get tensor masks corresponding to values
+    within the interval.
+    """
+
+    def contains_tensor_mask(self, item: torch.Tensor) -> torch.Tensor:
+        """Computes the mask corresponding to the tensor indexes
+        with a value contained within the interval.
+
+        Args:
+            item (torch.Tensor): tensor whose mask is to be computed
+
+        Returns:
+            torch.Tensor: mask, boolean tensor with the same shape as item,
+                where values are True when the corresponding value (same index) in item
+                is within the Interval.
+        """
+        left_tensor = item >= self.left if self.closed_left else item > self.left
+        right_tensor = item <= self.right if self.closed_right else item > self.right
+        return torch.logical_and(left_tensor, right_tensor)
+
+
+@dataclass
+class ConfidenceIntervalsConfiguration:
+    """Represents the configuration of intervals for a confidence estimator."""
+
+    normal: Interval
+    abnormal: Interval
+    unknown: Interval
+
+    def as_dict(self) -> dict[str, str]:
+        """Provides a dict representation of the intervals configuration
+
+        Returns:
+            dict[IntervalLiteral, str]:
+                dictionary of string representations for intervals in the configuration.
+        """
+        dict_repr = {
+            "normal": str(self.normal),
+            "abnormal": str(self.abnormal),
+            "unknown": str(self.unknown),
+        }
+        return dict_repr

ssad/confidence_estimators/hybrid_confidence.py

@@ -0,0 +1,49 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+from typing import List, Optional
+
+import torch
+
+from ssad.datamodules.dataset_with_confidence import DatasetWithConfidence
+from .confidence_estimator import BaseConfidenceEstimator, SupportsConfidenceEstimation
+
+
+class HybridEstimator(SupportsConfidenceEstimation):
+
+    def __init__(
+        self,
+        training_dataset: DatasetWithConfidence,
+        estimators: List[BaseConfidenceEstimator],
+        weights: Optional[List[float]] = None,
+    ):
+
+        if estimators is None:
+            raise ValueError(
+                "Empty estimator list during HybridEstimator construction."
+            )
+
+        if weights is None:
+            self.weights = torch.ones(len(estimators)) / len(estimators)
+        else:
+            self.weights = torch.FloatTensor(weights)
+
+        if len(self.weights) != len(estimators):
+            raise ValueError("Different number of weights and confidence estimators")
+
+        self.training_dataset = training_dataset
+        self.estimators = estimators
+
+    def estimate_confidence(self, scores_batch: torch.Tensor):
+        confidence = torch.Tensor(torch.zeros(len(self.training_dataset)))
+        for idx, estimator in enumerate(self.estimators):
+            estimator_confidence = estimator.estimate_confidence(scores_batch)
+            confidence += self.weights[idx] * estimator_confidence
+        return confidence

ssad/confidence_estimators/supports_confidence_estimation.py

@@ -0,0 +1,26 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+from typing import Optional, Protocol
+
+import torch
+
+from ssad.confidence_estimators.confidence_intervals_configuration import (
+    ConfidenceIntervalsConfiguration,
+)
+
+
+class SupportsConfidenceEstimation(Protocol):
+    """Protocol for confidence estimators used in a SelfSupervisionCallback."""
+
+    configuration: Optional[ConfidenceIntervalsConfiguration]
+
+    def estimate_confidence(self, scores_batch: torch.Tensor) -> torch.Tensor:
+        """Computes the confidence score over a data batch"""

ssad/datamodules/__init__.py

@@ -0,0 +1,10 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.

ssad/datamodules/api.py

@@ -0,0 +1,27 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+from .dataframe_with_labels import DataFrameWithLabels
+from .self_supervision_datamodule import SelfSupervisionDataModule
+from .dataset_with_confidence import (
+    DatasetWithConfidence,
+    init_confidence_from_csv,
+    save_confidence_to_csv,
+)
+from.transforms.dataframe_to_tensor import DataFrameToTensor
+
+__all__ = [
+    "SelfSupervisionDataModule",
+    "DatasetWithConfidence",
+    "init_confidence_from_csv",
+    "save_confidence_to_csv",
+    "DataFrameWithLabels",
+    "DataFrameToTensor",
+]

ssad/datamodules/dataframe_with_labels.py

@@ -0,0 +1,95 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""
+Defines the torch.Dataset specialization for
+pd.Datasets with a 'label' column
+"""
+
+import torch
+from torch.utils.data import Dataset
+
+
+class DataFrameWithLabels(Dataset):
+    """
+    Class that derives from torch.Dataset.
+    Defines the required methods for a dataset that
+    is formed of a pd.DataFrame with a column that corresponds
+    to the row label.
+    """
+
+    def __init__(
+        self,
+        data,
+        label_column_name: str,
+        transform=None,
+        target_transform=None,
+    ):
+        super().__init__()
+        labels = data[label_column_name]
+        features = data.drop(columns=[label_column_name])
+
+        self.data = torch.tensor(features.values, dtype=torch.float32)
+        self.labels = torch.tensor(labels.values, dtype=torch.float32)
+
+        self.transform = transform
+        self.target_transform = target_transform
+
+    def __len__(self) -> int:
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        data = self.data[idx]
+        label = self.labels[idx].unsqueeze(0)
+
+        if self.transform:
+            data = self.transform(data)
+        if self.target_transform:
+            label = self.target_transform(label)
+        return data, label
+
+    def input_dim(self):
+        """Returns the size of the samples of a dataset
+
+        Args:
+            dataset (Dataset): dataset whose row size is required
+
+        Returns:
+            int: length of the dataset samples
+        """
+        # return len(self.data.columns)
+        return self.data.shape[1]
+
+    def collate(self, batch: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+        """Collate function, used to transform a batch into a tensor
+
+        Args:
+            batch (List[int]): list of indexes in the batch
+
+        Returns:
+            torch.Tensor:
+        """
+        data, labels = zip(*batch)
+        data_tensor = torch.stack(data)
+        labels_tensor = torch.stack(labels)
+        return data_tensor, labels_tensor
+
+    def get_stats(self) -> tuple:
+        """
+        Returns the total number of samples, number of normal samples, and number of anomalies.
+        Assumes that labels are 0 for normal and 1 for anomalies.
+
+        Returns:
+            tuple: (total, normal, anomaly)
+        """
+        total = len(self.labels)
+        normal = int((self.labels == 0).sum().item())
+        anomaly = int((self.labels == 1).sum().item())
+        return total, normal, anomaly

ssad/datamodules/dataset_interfaces.py

@@ -0,0 +1,47 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+from typing import Protocol, Any, Tuple, runtime_checkable
+
+
+@runtime_checkable
+class DatasetWithLabels(Protocol):
+    """Interface for a Dataset containing two attributes:
+    data (Any) : data used by a model for its task
+    labels (Any) : ground truth for the data
+
+    """
+
+    labels: Any
+    data: Any
+
+    def __len__(self) -> int: ...
+
+    def __getitem__(self, idx: int) -> Tuple[Any, Any]: ...
+
+
+@runtime_checkable
+class DatasetWithInputDim(Protocol):
+    """Interface for a Dataset with the input_dim method
+    This method is necessary to initialize the input layer of an autoencoder.
+    """
+
+    def input_dim(self) -> int:
+        """Returns the dimension of the data"""
+
+    # def collate(self, batch : List[int]) -> tuple[torch.Tensor, torch.Tensor]:
+    #     """Collate function, used to transform a batch into a tensor
+
+    #     Args:
+    #         batch (List[int]): list of indexes in the batch
+
+    #     Returns:
+    #         torch.Tensor:
+    #     """

ssad/datamodules/dataset_with_confidence.py

@@ -0,0 +1,101 @@
+# Software Name : Self-Supervised Anomaly Detection
+# SPDX-FileCopyrightText: Copyright (c) Orange SA
+# SPDX-License-Identifier: MIT
+#
+# This software is distributed under the MIT License,
+# see the "LICENSE.txt" file for more details or https://spdx.org/licenses/MIT.html
+#
+# Authors: see CONTRIBUTORS
+# Software description: A Python library for autoencoder-based anomaly detection
+#          based on self-supervised training with dynamic sample confidence updates.
+"""
+Defines the torch.Dataset specialization for
+pd.Datasets with a 'label' column
+"""
+
+from collections.abc import Sized
+from pathlib import Path
+from typing import Optional
+
+import pandas as pd
+import torch
+from torch.utils.data import Dataset
+
+from .dataset_interfaces import DatasetWithLabels
+
+CONFIDENCE_COLUMN_NAME = "confidence"
+
+
+class DatasetWithConfidence(Dataset):
+    """
+    Class that derives from torch.Dataset.
+    Defines the required methods for a dataset that
+    is formed of a pd.DataFrame with a column that corresponds
+    to the row label.
+    """
+
+    def __init__(self, dataset: DatasetWithLabels, confidence: torch.Tensor):
+
+        if not isinstance(dataset, Sized) or len(dataset) != confidence.size(dim=0):
+            raise ValueError("Size mismatch between dataset and confidence tensor.")
+
+        self.dataset = dataset
+        self.confidence = confidence
+
+    def __len__(self):
+        return self.dataset.__len__()
+
+    def __getitem__(self, idx):
+        # TODO check if self.dataset[idx] is a tuple or a list
+        # in case there is no label
+        data = self.dataset[idx]
+        return {
+            "data": data[0],
+            "label": data[1],
+            "confidence": self.confidence[idx],
+        }
+
+
+def init_confidence_from_csv(dataset: Dataset, path: Optional[Path]) -> torch.Tensor:
+    """Loads a dataframe with confidence scores for each sample of the dataframe.
+
+    Args:
+        data (pd.Dataframe): dataframe
+        path (Path): path to csv file with a single column containing the confidence scores.
+
+    Returns:
+        confidence (torch.Tensor): tensor with initial confidence values.
+    """
+
+    if not isinstance(dataset, Sized):
+        raise TypeError("Provided dataset does not support the len() method.")
+
+    if path is not None:
+        confidence_df = pd.read_csv(path)
+
+        if CONFIDENCE_COLUMN_NAME not in confidence_df:
+            raise ValueError(f"Missing column: {CONFIDENCE_COLUMN_NAME}")
+
+        confidence = torch.from_numpy(confidence_df[CONFIDENCE_COLUMN_NAME].values)
+    else:
+        confidence = torch.ones(len(dataset))
+    return confidence
+
+
+def save_confidence_to_csv(confidence_dataset: DatasetWithConfidence, path: Path):
+    """Saves confidence scores to csv file.
+    Uses a dataloader to iterate over the dataset and extract confidence scores
+    for each sample into a list.
+    This list is then saved to a csv file by converting it to a pd.DataFrame.
+
+    Args:
+        confidence_dataset (DatasetWithConfidence): dataset with confidence scores to save.
+        path (Path): path to csv file.
+    """
+    confidence = confidence_dataset.confidence.detach().cpu().numpy()
+
+    confidence_df = pd.DataFrame(
+        confidence,
+        columns=[CONFIDENCE_COLUMN_NAME],
+    )
+    confidence_df.to_csv(path)