tsadmetrics 0.1.17__py3-none-any.whl → 1.0.1__py3-none-any.whl

tsadmetrics/evaluation/Runner.py

@@ -0,0 +1,253 @@
+import warnings
+from ..metrics.Registry import Registry
+import numpy as np
+from .Report import Report
+import pandas as pd
+import yaml
+class Runner:
+    """
+    Orchestrates the evaluation of datasets using a set of metrics.
+
+    The `Runner` class provides functionality to:
+    
+    - Load datasets from direct data, file references, or a global YAML configuration file.
+    - Load metrics either directly from a list or from a configuration file.
+    - Evaluate all datasets against all metrics.
+    - Optionally generate a report summarizing the evaluation results.
+
+    Parameters
+    ----------
+    dataset_evaluations : list or str
+        Accepted formats:
+
+        1. **Global config file (str)**
+
+           If a string is provided and `metrics` is None, it is assumed to be
+           the path to a configuration file that defines both datasets and metrics.
+
+        2. **Direct data (list of tuples)**
+
+           Example::
+
+               [
+                   ("dataset1", y_true1, (y_pred_binary1, y_pred_continuous1)),
+                   ("dataset2", y_true2, (y_pred_binary2, y_pred_continuous2)),
+                   ("dataset3", y_true3, y_pred3)
+               ]
+
+           where `y_pred` may be binary or continuous.
+
+        3. **File references (list of tuples)**
+
+           Example::
+
+               [
+                   ("dataset1", "result1.csv"),
+                   ("dataset2", "result2.csv")
+               ]
+
+           Each file must contain:
+
+           - `y_true`
+           - Either:
+             * (`y_pred_binary` and `y_pred_continuous`)
+             * or (`y_pred`)
+
+    metrics : list or str, optional
+        - **List of metrics**: Each element is a tuple:
+        
+          [(metric_name, {param_name: value, ...}), ...]
+
+          Example::
+
+              [
+                  ("pwf", {"beta": 1.0}),
+                  ("rpate", {"alpha": 0.5}),
+                  ("adc", {})
+              ]
+
+        - **Config file (str)**: Path to a YAML file containing metric definitions.
+
+    Attributes
+    ----------
+    dataset_evaluations : list
+        Loaded datasets in normalized format:
+        (name, y_true, y_pred_binary, y_pred_continuous, y_pred)
+
+    metrics : list
+        List of metrics with their configurations.
+
+    Raises
+    ------
+    ValueError
+        If a configuration file is invalid or required fields are missing.
+    """
+    def __init__(self, dataset_evaluations, metrics=None):
+        
+
+        # Case 1: global config file -> load datasets and metrics from config
+        if isinstance(dataset_evaluations, str) and metrics is None:
+            config_file = dataset_evaluations
+            with open(config_file, "r") as f:
+                config = yaml.safe_load(f)
+
+            # unir lista de dicts en un solo dict
+            if isinstance(config, list):
+                merged_config = {}
+                for entry in config:
+                    if not isinstance(entry, dict):
+                        raise ValueError(f"Invalid entry in config file: {entry}")
+                    merged_config.update(entry)
+                config = merged_config
+
+            if not isinstance(config, dict):
+                raise ValueError("Global config file must define datasets and metrics_config as a mapping.")
+
+            if "metrics_config" not in config:
+                raise ValueError("Global config file must contain 'metrics_config'.")
+
+            # separar datasets de la ruta de métricas
+            datasets = [(name, path) for name, path in config.items() if name != "metrics_config"]
+
+            self.dataset_evaluations = self._load_datasets(datasets)
+            self.metrics = Registry.load_metrics_from_file(config["metrics_config"])
+            return
+
+        # Case 2: datasets provided directly, metrics may be list or str
+        self.dataset_evaluations = self._load_datasets(dataset_evaluations)
+
+        if isinstance(metrics, str):
+            self.metrics = Registry.load_metrics_from_file(metrics)
+        else:
+            self.metrics = metrics
+
+    def _load_datasets(self, dataset_evaluations):
+        loaded = []
+        for entry in dataset_evaluations:
+            name, data = entry[0], entry[1:]
+            if len(data) == 1 and isinstance(data[0], str):
+                # Case: File reference
+                df = pd.read_csv(data[0], sep=';')
+                y_true = df["y_true"].values
+
+                if "y_pred_binary" in df.columns and "y_pred_continuous" in df.columns:
+                    y_pred_binary = df["y_pred_binary"].values
+                    y_pred_continuous = df["y_pred_continuous"].values
+                elif "y_pred" in df.columns:
+                    y_pred = df["y_pred"].values
+                    y_pred_binary, y_pred_continuous = None, None
+                else:
+                    raise ValueError(
+                        f"File {data[0]} must contain either "
+                        f"(y_pred_binary, y_pred_continuous) or y_pred column."
+                    )
+            else:
+                # Case: Direct data
+                if len(data) == 2 and isinstance(data[1], tuple):
+                    # Format: y_true, (y_pred_binary, y_pred_continuous)
+                    y_true, (y_pred_binary, y_pred_continuous) = data
+                elif len(data) == 2:
+                    # Format: y_true, y_pred
+                    y_true, y_pred = data
+                    y_pred_binary, y_pred_continuous = None, None
+                else:
+                    raise ValueError("Invalid dataset format.")
+            loaded.append(
+                (name, y_true, y_pred_binary, y_pred_continuous, locals().get("y_pred", None))
+            )
+        return loaded
+
+    def run(self, generate_report=False, report_file="evaluation_report.csv"):
+        """
+        Run the evaluation for all datasets and metrics.
+
+        Returns:
+            pd.DataFrame: DataFrame structured as follows:
+
+                - The **first row** contains the parameters of each metric.
+                - The **subsequent rows** contain the metric values for each dataset.
+                - The **index** column represents the dataset names, with the first row labeled as 'params'.
+
+            Example::
+
+                dataset   | metric1       | metric2
+                ----------|---------------|--------
+                params    | {'param1':0.2}| {}
+                dataset1  | 0.5           | 1.0
+                dataset2  | 0.125         | 1.0
+        """
+        results = {}
+        metric_keys = {}  
+
+        for dataset_name, y_true, y_pred_binary, y_pred_continuous, y_pred in self.dataset_evaluations:
+            dataset_results = {}
+            for metric_name, params in self.metrics:
+                metric = Registry.get_metric(metric_name, **params)
+
+                # Computar valor según tipo de métrica
+                if getattr(metric, "binary_prediction", False):
+                    if y_pred_binary is not None:
+                        value = metric.compute(y_true, y_pred_binary)
+                    elif y_pred is not None and set(np.unique(y_pred)).issubset({0, 1}):
+                        value = metric.compute(y_true, y_pred)
+                    else:
+                        warnings.warn(
+                            f"Metric {metric_name} requires binary input, "
+                            f"but dataset {dataset_name} provided non-binary predictions. Skipped.",
+                            UserWarning
+                        )
+                        value = None
+                else:
+                    if y_pred_continuous is not None:
+                        value = metric.compute(y_true, y_pred_continuous)
+                    elif y_pred is not None and not set(np.unique(y_pred)).issubset({0, 1}):
+                        value = metric.compute(y_true, y_pred)
+                    else:
+                        warnings.warn(
+                            f"Metric {metric_name} requires continuous input, "
+                            f"but dataset {dataset_name} provided binary predictions. Skipped.",
+                            UserWarning
+                        )
+                        value = None
+
+                # Generar clave única usando parámetros si ya existe
+                base_key = metric_name
+                key = f"{metric_name}({params})" if params else metric_name
+                if key in metric_keys and metric_keys[key] != params:
+                    key = f"{metric_name}({params})"
+                metric_keys[key] = params
+
+                dataset_results[key] = value
+
+            results[dataset_name] = dataset_results
+
+        # Construir DataFrame con primera fila = parámetros
+        if results:
+            first_dataset = next(iter(results.values()))
+            metric_names = []
+            metric_params = []
+
+            for metric_config in first_dataset.keys():
+                if "(" in metric_config:
+                    name, params_str = metric_config.split("(", 1)
+                    params_str = params_str.rstrip(")")
+                else:
+                    name, params_str = metric_config, ""
+                metric_names.append(name)
+                metric_params.append(params_str if params_str != "{}" else "")
+
+            df_data = [metric_params]  # primera fila = parámetros
+            for dataset_metrics in results.values():
+                df_data.append([dataset_metrics[m] for m in dataset_metrics.keys()])
+
+            df = pd.DataFrame(df_data, columns=metric_names)
+            df.index = ["params"] + list(results.keys())
+            df.index.name = 'dataset'
+        else:
+            df = pd.DataFrame()
+
+        if generate_report:
+            report = Report()
+            report.generate_report(df, report_file)
+
+        return df

tsadmetrics/metrics/Registry.py

@@ -0,0 +1,141 @@
+from typing import Type
+import yaml
+from ..base.Metric import Metric
+from .spm import *
+from .tem.tpdm import *
+from .tem.ptdm import *
+from .tem.tmem import *
+from .tem.dpm import *
+from .tem.tstm import *
+
+class Registry:
+    """
+    Central registry for anomaly detection metrics.
+
+    This class provides a centralized interface to register, retrieve,
+    and load metric classes for anomaly detection tasks.
+    """
+
+    _registry = {}
+
+    @classmethod
+    def register(cls, metric_cls: Type[Metric]):
+        """
+        Register a metric class using its `name` attribute.
+
+        Args:
+            metric_cls (Type[Metric]): The metric class to register. 
+                The class must define a ``name`` attribute.
+
+        Raises:
+            ValueError: If the metric class does not define a ``name`` 
+                attribute or if a metric with the same name is already registered.
+        """
+        if not hasattr(metric_cls, "name"):
+            raise ValueError(f"Metric class {metric_cls.__name__} must define a 'name' attribute.")
+
+        name = metric_cls.name
+        if name in cls._registry:
+            raise ValueError(f"Metric '{name}' is already registered.")
+
+        cls._registry[name] = metric_cls
+
+    @classmethod
+    def get_metric(cls, name: str, **params) -> Metric:
+        """
+        Retrieve and instantiate a registered metric by name.
+
+        Args:
+            name (str): Name of the metric to retrieve.
+            \*\*params: Parameters to initialize the metric instance.
+
+        Returns:
+            Metric: An instance of the requested metric.
+
+        Raises:
+            ValueError: If the metric name is not registered.
+        """
+        if name not in cls._registry:
+            raise ValueError(f"Metric '{name}' is not registered.")
+        return cls._registry[name](**params)
+
+    @classmethod
+    def available_metrics(cls):
+        """
+        List all registered metric names.
+
+        Returns:
+            list[str]: A list of registered metric names.
+        """
+        return list(cls._registry.keys())
+    
+    @classmethod
+    def load_metrics_info_from_file(cls, filepath: str):
+        """
+        Load metric definitions (names and parameters) from a YAML configuration file.
+
+        Args:
+            filepath (str): Path to the YAML file.
+
+        Returns:
+            list[tuple[str, dict]]: A list of tuples containing the metric name and 
+            its parameters, e.g. ``[("metric_name", {"param1": value, ...}), ...]``.
+
+        Raises:
+            ValueError: If the YAML file contains invalid entries or unsupported format.
+        """
+        with open(filepath, "r") as f:
+            config = yaml.safe_load(f)
+
+        metrics_info = []
+
+        if isinstance(config, list):
+            # If YAML is a list of metric names or dicts
+            for entry in config:
+                if isinstance(entry, str):
+                    metrics_info.append((entry, {}))
+                elif isinstance(entry, dict):
+                    for name, params in entry.items():
+                        metrics_info.append((name, params or {}))
+                else:
+                    raise ValueError(f"Invalid metric entry: {entry}")
+        elif isinstance(config, dict):
+            # If YAML is a dictionary: {metric: params}
+            for name, params in config.items():
+                if params is None:
+                    params = {}
+                metrics_info.append((name, params))
+        else:
+            raise ValueError("YAML format must be a list or dict.")
+        return metrics_info
+
+    @classmethod
+    def load_metrics_from_file(cls, filepath: str):
+        """
+        Load and instantiate metrics from a YAML configuration file.
+
+        Args:
+            filepath (str): Path to the YAML configuration file.
+
+        Returns:
+            list[tuple[str, dict]]: A list of tuples containing the metric name and 
+            the parameters used to instantiate it.
+        """
+        metrics_info = cls.load_metrics_info_from_file(filepath)
+        metrics = []
+        for name, params in metrics_info:
+            metric = cls.get_metric(name, **params)
+            metrics.append((name, params))
+        return metrics
+
+
+# --- Auto-discovery
+def auto_register():
+    """
+    Automatically register all subclasses of ``Metric`` found in the project.
+
+    This function inspects the current inheritance tree of ``Metric`` and 
+    registers each subclass in the central registry.
+    """
+    for metric_cls in Metric.__subclasses__():
+        Registry.register(metric_cls)

tsadmetrics/metrics/__init__.py

@@ -0,0 +1,2 @@
+from .Registry import auto_register
+auto_register()

tsadmetrics/metrics/spm/PointwiseAucPr.py

@@ -0,0 +1,62 @@
+from ...base.Metric import Metric
+import numpy as np
+from ...utils.functions_auc import precision_recall_curve
+
+class PointwiseAucPr(Metric):
+    """
+    Point-wise Area Under the Precision-Recall Curve (AUC-PR) for anomaly detection.
+
+    This metric computes the standard Area Under the Precision-Recall Curve (AUC-PR)
+    in a **point-wise manner**. Each time-series data point is treated independently
+    when calculating precision and recall, making this suitable for anomaly detection tasks
+    where anomalies are labeled at the individual point level.
+
+    Reference:
+        Implementation based on:
+        https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    Attributes:
+        name (str):
+            Fixed name identifier for this metric: `"pw_auc_pr"`.
+        binary_prediction (bool):
+            Indicates whether this metric expects binary predictions. Always `False`
+            since it requires continuous anomaly scores.
+
+    Raises:
+        ValueError:
+            If input arrays are invalid or improperly shaped (handled by the base class).
+        TypeError:
+            If inputs are not array-like.
+    """
+    
+    name = "pw_auc_pr"
+    binary_prediction = False
+    def __init__(self, **kwargs):
+        """
+        Initialize the PointwiseAucPr metric.
+
+        Parameters:
+            **kwargs:
+                Additional keyword arguments passed to the base `Metric` class.
+                These may include configuration parameters or overrides.
+        """
+        super().__init__(name="pw_auc_pr", **kwargs)
+
+    def _compute(self, y_true, y_anomaly_scores):
+        """
+        Compute the point-wise AUC-PR score.
+
+        Parameters:
+            y_true (np.ndarray):
+                Ground-truth binary labels for the time series.
+                Values must be 0 (normal) or 1 (anomaly).
+            y_anomaly_scores (np.ndarray):
+                Continuous anomaly scores assigned to each point in the series.
+
+        Returns:
+            float:
+                The computed point-wise AUC-PR score.
+        """
+
+        precision, recall, _ = precision_recall_curve(y_true, y_anomaly_scores)
+        return -np.sum(np.diff(recall) * np.array(precision)[:-1])

tsadmetrics/metrics/spm/PointwiseAucRoc.py

@@ -0,0 +1,63 @@
+from ...base.Metric import Metric
+import numpy as np
+from ...utils.functions_auc import roc_curve, auc
+
+class PointwiseAucRoc(Metric):
+    """
+    Point-wise Area Under the Receiver Operating Characteristic Curve (AUC-ROC) for anomaly detection.
+
+    This metric computes the standard Area Under the ROC Curve (AUC-ROC)
+    in a **point-wise manner**. Each time-series data point is treated independently
+    when calculating true positives, false positives, and false negatives.
+    It is widely used to evaluate the ability of anomaly scoring functions
+    to distinguish between normal and anomalous points.
+
+    Reference:
+        Implementation based on:
+        https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    Attributes:
+        name (str):
+            Fixed name identifier for this metric: `"pw_auc_roc"`.
+        binary_prediction (bool):
+            Indicates whether this metric expects binary predictions. Always `False`
+            since it requires continuous anomaly scores.
+
+    Raises:
+        ValueError:
+            If input arrays are invalid or improperly shaped (validated in the base class).
+        TypeError:
+            If inputs are not array-like.
+    """
+
+    name = "pw_auc_roc"
+    binary_prediction = False
+
+    def __init__(self, **kwargs):
+        """
+        Initialize the PointwiseAucRoc metric.
+
+        Parameters:
+            **kwargs:
+                Additional keyword arguments passed to the base `Metric` class.
+                These may include configuration parameters or overrides.
+        """
+        super().__init__(name="pw_auc_roc", **kwargs)
+
+    def _compute(self, y_true, y_anomaly_scores):
+        """
+        Compute the point-wise AUC-ROC score.
+
+        Parameters:
+            y_true (np.ndarray):
+                Ground-truth binary labels for the time series.
+                Values must be 0 (normal) or 1 (anomaly).
+            y_anomaly_scores (np.ndarray):
+                Continuous anomaly scores assigned to each point in the series.
+
+        Returns:
+            float:
+                The computed point-wise AUC-ROC score.
+        """
+        fpr, tpr, _ = roc_curve(y_true, y_anomaly_scores)
+        return auc(fpr, tpr)

tsadmetrics/metrics/spm/PointwiseFScore.py

@@ -0,0 +1,86 @@
+from ...base.Metric import Metric
+import numpy as np
+
+class PointwiseFScore(Metric):
+    """
+    Point-wise F-score for anomaly detection in time series.
+
+    This metric computes the classical F-score without considering temporal context,
+    treating each time-series point independently. It balances precision and recall
+    according to the configurable parameter `beta`.
+
+    Reference:
+        Implementation based on:
+        https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    Parameters:
+        beta (float, optional):
+            The beta value determines the relative weight of recall compared to precision.
+            A value of 1.0 gives equal weight (F1-score).
+            Default is 1.0.
+
+    Attributes:
+        name (str):
+            Fixed name identifier for this metric: `"pwf"`.
+        binary_prediction (bool):
+            Indicates whether this metric expects binary predictions. Always `True`.
+        param_schema (dict):
+            Schema for supported parameters:
+            - `beta` (float, default=1.0).
+
+    Raises:
+        ValueError:
+            If required parameters are missing (validated by the base class).
+        TypeError:
+            If parameter types do not match the schema (validated by the base class).
+    """
+
+    name = "pwf"
+    binary_prediction = True
+    param_schema = {
+        "beta": {
+            "default": 1.0,
+            "type": float
+        }
+    }
+
+    def __init__(self, **kwargs):
+        """
+        Initialize the PointwiseFScore metric.
+
+        Parameters:
+            **kwargs:
+                Additional keyword arguments passed to the base `Metric` class.
+                These may include configuration parameters such as `beta`.
+        """
+        super().__init__(name="pwf", **kwargs)
+
+    def _compute(self, y_true, y_pred):
+        """
+        Compute the point-wise F-score.
+
+        Parameters:
+            y_true (np.ndarray):
+                Ground-truth binary labels for the time series.
+                Values must be 0 (normal) or 1 (anomaly).
+            y_pred (np.ndarray):
+                Predicted binary labels for the time series.
+                Values must be 0 (normal) or 1 (anomaly).
+
+        Returns:
+            float:
+                The computed point-wise F-score.
+                Returns 0 if either precision or recall is 0.
+        """
+        tp = np.sum(y_pred * y_true)
+        fp = np.sum(y_pred * (1 - y_true))
+        fn = np.sum((1 - y_pred) * y_true)
+
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0
+
+        if precision == 0 or recall == 0:
+            return 0
+        
+        beta = self.params['beta']
+        return ((1 + beta**2) * precision * recall) / (beta**2 * precision + recall)

tsadmetrics/metrics/spm/PrecisionAtK.py

@@ -0,0 +1,81 @@
+from ...base.Metric import Metric
+import numpy as np
+
+class PrecisionAtK(Metric):
+    """
+    Precision at K (P@K) for anomaly detection in time series.
+
+    This metric evaluates how many of the top-`k` points with the highest anomaly
+    scores correspond to true anomalies. It is particularly useful when focusing
+    on identifying the most anomalous points rather than setting a global threshold.
+
+    By definition, `k` is automatically set to the number of true anomalies present
+    in `y_true`.
+
+    .. math::
+        k = \sum(y\_true)
+
+    Reference:
+        Implementation based on:
+        https://link.springer.com/article/10.1007/s10618-023-00988-8
+
+    Attributes:
+        name (str):
+            Fixed name identifier for this metric: `"pak"`.
+        binary_prediction (bool):
+            Indicates whether this metric expects binary predictions. Always `False`
+            since it requires continuous anomaly scores.
+
+    Raises:
+        AssertionError:
+            - If the number of true anomalies (`k`) is zero.
+            - If the number of predicted positives is less than `k`.
+        ValueError:
+            If input arrays are invalid or improperly shaped (validated in the base class).
+        TypeError:
+            If inputs are not array-like.
+    """
+
+    name = "pak"
+    binary_prediction = False
+
+    def __init__(self, **kwargs):
+        """
+        Initialize the PrecisionAtK metric.
+
+        Parameters:
+            **kwargs:
+                Additional keyword arguments passed to the base `Metric` class.
+                These may include configuration parameters or overrides.
+        """
+        super().__init__(name="pak", **kwargs)
+
+    def _compute(self, y_true, y_anomaly_scores):
+        """
+        Compute the Precision at K (P@K) score.
+
+        Parameters:
+            y_true (np.ndarray):
+                Ground-truth binary labels for the time series.
+                Values must be 0 (normal) or 1 (anomaly).
+            y_anomaly_scores (np.ndarray):
+                Continuous anomaly scores assigned to each point in the series.
+
+        Returns:
+            float:
+                The precision at K score, where K = number of anomalies in `y_true`.
+
+        Raises:
+            AssertionError:
+                If `k = sum(y_true)` is 0.
+                If fewer than `k` points are predicted as anomalies.
+        """
+        k = int(sum(y_true))
+        assert k > 0, "The number of true anomalies (k) must be greater than zero."
+        threshold = np.sort(y_anomaly_scores)[-k]
+
+        pred = y_anomaly_scores >= threshold
+        assert sum(pred) >= k, (
+            f"Number of predicted positives ({sum(pred)}) should be >= k ({k})."
+        )
+        return np.dot(pred, y_true) / sum(pred)