tsadmetrics 0.1.16__py3-none-any.whl → 1.0.0__py3-none-any.whl

tsadmetrics/__init__.py

@@ -1,21 +0,0 @@
-from .binary_metrics import *
-from .non_binary_metrics import *
-from .utils import *
-
-
-
-
-__author__ = 'Pedro Rafael Velasco Priego i12veprp@uco.es'
-__version__ = "0.1.3"
-__all__ = ['point_wise_recall', 'point_wise_precision', 'point_wise_f_score','point_adjusted_recall',
-            'point_adjusted_precision', 'point_adjusted_f_score', 'segment_wise_recall', 'segment_wise_precision',
-            'segment_wise_f_score','delay_th_point_adjusted_recall', 'delay_th_point_adjusted_precision',
-              'delay_th_point_adjusted_f_score','point_adjusted_at_k_recall','point_adjusted_at_k_precision',
-              'point_adjusted_at_k_f_score','latency_sparsity_aw_recall', 'latency_sparsity_aw_precision',
-                'latency_sparsity_aw_f_score','composite_f_score','time_tolerant_recall','time_tolerant_precision',
-                'time_tolerant_f_score','range_based_recall','range_based_precision','range_based_f_score','ts_aware_recall',
-                'ts_aware_precision','ts_aware_f_score','enhanced_ts_aware_recall','enhanced_ts_aware_precision','enhanced_ts_aware_f_score',
-                'affiliation_based_recall','affiliation_based_precision','affiliation_based_f_score','nab_score','temporal_distance',
-                'average_detection_count','absolute_detection_distance','total_detected_in_range','detection_accuracy_in_range','weighted_detection_difference',
-                'binary_pate','real_pate','mean_time_to_detect',
-                'precision_at_k','auc_roc_pw','auc_pr_pw','auc_pr_pa','auc_pr_sw','vus_roc','vus_pr', 'compute_metrics', 'compute_metrics_from_file']

tsadmetrics/base/Metric.py

@@ -0,0 +1,188 @@
+import yaml
+import numpy as np
+
+class Metric:
+    """
+    Base class for time series anomaly detection metrics.
+
+    This class provides common functionality for metric configuration, including
+    parameter validation from a YAML configuration file and support for a parameter
+    schema defined in each subclass.
+
+    Parameters:
+        name (str, optional):
+            The name of the metric. If not provided, it defaults to the lowercase
+            name of the subclass.
+        config_file (str, optional):
+            Path to a YAML configuration file. Parameters defined in the file under
+            the metric's name will be loaded automatically.
+        \*\*params:
+            Additional parameters passed directly to the metric. These override
+            those loaded from the configuration file.
+
+    Attributes:
+        name (str):
+            Name of the metric instance.
+        params (dict):
+            Dictionary of parameters used by the metric.
+        binary_prediction (bool):
+            Whether the metric expects binary predictions (True) or continuous scores (False).
+
+    Raises:
+        ValueError:
+            If a required parameter is missing or if the configuration file is not found.
+        TypeError:
+            If a parameter does not match its expected type as defined in the schema.
+    """
+
+    def __init__(self, name=None, config_file=None, **params):
+        self.name = name or self.__class__.__name__.lower()
+
+        # Ensure subclasses define binary_prediction
+        if not hasattr(self.__class__, "binary_prediction"):
+            raise ValueError(
+                f"Subclass {self.__class__.__name__} must define class attribute 'binary_prediction' (True/False)."
+            )
+        if not isinstance(self.__class__.binary_prediction, bool):
+            raise TypeError(
+                f"'binary_prediction' in {self.__class__.__name__} must be of type bool."
+            )
+
+        self.binary_prediction = self.__class__.binary_prediction
+        self.params = {}
+        self.configure(config_file=config_file, **params)
+
+
+    def configure(self, config_file=None, **params):
+        """
+        Load and validate metric parameters from a YAML configuration file
+        and/or from explicit keyword arguments.
+
+        Parameters:
+            config_file (str, optional):
+                Path to the configuration file. If provided, it will load parameters
+                under the section with the metric's name.
+            \*\*params:
+                Parameters passed directly to the metric instance.
+
+        Raises:
+            ValueError:
+                If a required parameter is not specified or the configuration file is missing.
+            TypeError:
+                If a parameter value does not match the expected type.
+        """
+        if config_file:
+            try:
+                with open(config_file, 'r') as f:
+                    config = yaml.safe_load(f)
+                    file_params = config.get(self.name.lower(), {})
+                    self.params.update(file_params)
+            except FileNotFoundError:
+                raise ValueError(f"Configuration file '{config_file}' not found.")
+
+        self.params.update(params)
+
+        schema = getattr(self.__class__, 'param_schema', {})
+        for key, rules in schema.items():
+            if key not in self.params:
+                if 'default' in rules:
+                    self.params[key] = rules['default']
+                else:
+                    raise ValueError(f"Required parameter '{key}' not specified.")
+
+            if 'type' in rules and key in self.params:
+                expected_type = rules['type']
+                if expected_type is float:
+                    if not isinstance(self.params[key], (float, int)):
+                        raise TypeError(f"Parameter '{key}' must be of type float, got {type(self.params[key]).__name__} instead.")
+                else:
+                    if not isinstance(self.params[key], expected_type):
+                        raise TypeError(f"Parameter '{key}' must be of type {expected_type.__name__}, got {type(self.params[key]).__name__} instead.")
+
+    def _validate_inputs(self, y_true, y_pred):
+        """
+        Validate that y_true and y_pred are valid sequences of the same length.
+
+        If binary_prediction = True:
+            Both y_true and y_pred must be binary (0 or 1).
+        If binary_prediction = False:
+            y_true must be binary (0 or 1), y_pred can be continuous values.
+
+        Raises:
+            ValueError: If lengths differ or values are not valid.
+            TypeError: If inputs are not array-like.
+        """
+        y_true = np.asarray(y_true)
+        y_pred = np.asarray(y_pred)
+
+        if y_true.shape != y_pred.shape:
+            raise ValueError(
+                f"Shape mismatch: y_true has shape {y_true.shape}, y_pred has shape {y_pred.shape}."
+            )
+
+        if y_true.ndim != 1 or y_pred.ndim != 1:
+            raise ValueError("y_true and y_pred must be 1D arrays.")
+
+        if not np.isin(y_true, [0, 1]).all():
+            raise ValueError("y_true must contain only 0 or 1.")
+
+        if self.binary_prediction:
+            if not np.isin(y_pred, [0, 1]).all():
+                raise ValueError("y_pred must contain only 0 or 1 (binary_prediction=True).")
+
+        return y_true, y_pred
+    
+    def _compute(self, y_true, y_pred):
+        """
+        Compute the value of the metric (core implementation).
+
+        This method contains the actual logic of the metric and must be 
+        implemented by subclasses. It is automatically called by 
+        `compute()` after input validation.
+
+        Parameters:
+            y_true (array-like):
+                Ground truth binary labels.
+            y_pred (array-like):
+                Predicted binary labels.
+
+        Returns:
+            float: The value of the metric.
+
+        Raises:
+            NotImplementedError: If the method is not overridden by a subclass.
+        """
+        raise NotImplementedError("Subclasses must implement _compute().")
+
+
+
+    def compute(self, y_true, y_pred):
+        """
+        Compute the value of the metric (wrapper method).
+
+        This method performs input validation and then calls the internal
+        `_compute()` method, which contains the actual metric logic. 
+
+        **Important:** Subclasses **should not override** this method. 
+        Instead, implement `_compute()` to define the behavior of the metric.
+
+        Parameters
+        ----------
+        y_true : array-like
+            Ground truth binary labels.
+        y_pred : array-like
+            Predicted binary labels.
+
+        Returns
+        -------
+        float
+            The value of the metric.
+
+        Raises
+        ------
+        NotImplementedError
+            If `_compute()` is not implemented by the subclass.
+        """
+        y_true, y_pred = self._validate_inputs(y_true, y_pred)
+        return self._compute(y_true, y_pred)
+

tsadmetrics/evaluation/Report.py

@@ -0,0 +1,25 @@
+import pandas as pd
+
+class Report:
+    def __init__(self):
+        pass
+    def generate_report(self, results, output_file):
+        """
+        Generate a report from the evaluation results.
+
+        Parameters:
+            results (dict):
+                Dictionary containing evaluation results.
+            output_file (str):
+                Path to the output file where the report will be saved.
+        """
+        
+        if type(results) is dict:
+            df = pd.DataFrame.from_dict(results, orient='index')
+
+            df.index.name = 'dataset'
+            df.reset_index(inplace=True)
+
+            df.to_csv(output_file, index=False, sep=';')
+        else:
+            results.to_csv(output_file, sep=';')

tsadmetrics/evaluation/Runner.py

@@ -0,0 +1,253 @@
+import warnings
+from ..metrics.Registry import Registry
+import numpy as np
+from .Report import Report
+import pandas as pd
+import yaml
+class Runner:
+    """
+    Orchestrates the evaluation of datasets using a set of metrics.
+
+    The `Runner` class provides functionality to:
+    
+    - Load datasets from direct data, file references, or a global YAML configuration file.
+    - Load metrics either directly from a list or from a configuration file.
+    - Evaluate all datasets against all metrics.
+    - Optionally generate a report summarizing the evaluation results.
+
+    Parameters
+    ----------
+    dataset_evaluations : list or str
+        Accepted formats:
+
+        1. **Global config file (str)**
+
+           If a string is provided and `metrics` is None, it is assumed to be
+           the path to a configuration file that defines both datasets and metrics.
+
+        2. **Direct data (list of tuples)**
+
+           Example::
+
+               [
+                   ("dataset1", y_true1, (y_pred_binary1, y_pred_continuous1)),
+                   ("dataset2", y_true2, (y_pred_binary2, y_pred_continuous2)),
+                   ("dataset3", y_true3, y_pred3)
+               ]
+
+           where `y_pred` may be binary or continuous.
+
+        3. **File references (list of tuples)**
+
+           Example::
+
+               [
+                   ("dataset1", "result1.csv"),
+                   ("dataset2", "result2.csv")
+               ]
+
+           Each file must contain:
+
+           - `y_true`
+           - Either:
+             * (`y_pred_binary` and `y_pred_continuous`)
+             * or (`y_pred`)
+
+    metrics : list or str, optional
+        - **List of metrics**: Each element is a tuple:
+        
+          [(metric_name, {param_name: value, ...}), ...]
+
+          Example::
+
+              [
+                  ("pwf", {"beta": 1.0}),
+                  ("rpate", {"alpha": 0.5}),
+                  ("adc", {})
+              ]
+
+        - **Config file (str)**: Path to a YAML file containing metric definitions.
+
+    Attributes
+    ----------
+    dataset_evaluations : list
+        Loaded datasets in normalized format:
+        (name, y_true, y_pred_binary, y_pred_continuous, y_pred)
+
+    metrics : list
+        List of metrics with their configurations.
+
+    Raises
+    ------
+    ValueError
+        If a configuration file is invalid or required fields are missing.
+    """
+    def __init__(self, dataset_evaluations, metrics=None):
+        
+
+        # Case 1: global config file -> load datasets and metrics from config
+        if isinstance(dataset_evaluations, str) and metrics is None:
+            config_file = dataset_evaluations
+            with open(config_file, "r") as f:
+                config = yaml.safe_load(f)
+
+            # unir lista de dicts en un solo dict
+            if isinstance(config, list):
+                merged_config = {}
+                for entry in config:
+                    if not isinstance(entry, dict):
+                        raise ValueError(f"Invalid entry in config file: {entry}")
+                    merged_config.update(entry)
+                config = merged_config
+
+            if not isinstance(config, dict):
+                raise ValueError("Global config file must define datasets and metrics_config as a mapping.")
+
+            if "metrics_config" not in config:
+                raise ValueError("Global config file must contain 'metrics_config'.")
+
+            # separar datasets de la ruta de métricas
+            datasets = [(name, path) for name, path in config.items() if name != "metrics_config"]
+
+            self.dataset_evaluations = self._load_datasets(datasets)
+            self.metrics = Registry.load_metrics_from_file(config["metrics_config"])
+            return
+
+        # Case 2: datasets provided directly, metrics may be list or str
+        self.dataset_evaluations = self._load_datasets(dataset_evaluations)
+
+        if isinstance(metrics, str):
+            self.metrics = Registry.load_metrics_from_file(metrics)
+        else:
+            self.metrics = metrics
+
+    def _load_datasets(self, dataset_evaluations):
+        loaded = []
+        for entry in dataset_evaluations:
+            name, data = entry[0], entry[1:]
+            if len(data) == 1 and isinstance(data[0], str):
+                # Case: File reference
+                df = pd.read_csv(data[0], sep=';')
+                y_true = df["y_true"].values
+
+                if "y_pred_binary" in df.columns and "y_pred_continuous" in df.columns:
+                    y_pred_binary = df["y_pred_binary"].values
+                    y_pred_continuous = df["y_pred_continuous"].values
+                elif "y_pred" in df.columns:
+                    y_pred = df["y_pred"].values
+                    y_pred_binary, y_pred_continuous = None, None
+                else:
+                    raise ValueError(
+                        f"File {data[0]} must contain either "
+                        f"(y_pred_binary, y_pred_continuous) or y_pred column."
+                    )
+            else:
+                # Case: Direct data
+                if len(data) == 2 and isinstance(data[1], tuple):
+                    # Format: y_true, (y_pred_binary, y_pred_continuous)
+                    y_true, (y_pred_binary, y_pred_continuous) = data
+                elif len(data) == 2:
+                    # Format: y_true, y_pred
+                    y_true, y_pred = data
+                    y_pred_binary, y_pred_continuous = None, None
+                else:
+                    raise ValueError("Invalid dataset format.")
+            loaded.append(
+                (name, y_true, y_pred_binary, y_pred_continuous, locals().get("y_pred", None))
+            )
+        return loaded
+
+    def run(self, generate_report=False, report_file="evaluation_report.csv"):
+        """
+        Run the evaluation for all datasets and metrics.
+
+        Returns:
+            pd.DataFrame: DataFrame structured as follows:
+
+                - The **first row** contains the parameters of each metric.
+                - The **subsequent rows** contain the metric values for each dataset.
+                - The **index** column represents the dataset names, with the first row labeled as 'params'.
+
+            Example::
+
+                dataset   | metric1       | metric2
+                ----------|---------------|--------
+                params    | {'param1':0.2}| {}
+                dataset1  | 0.5           | 1.0
+                dataset2  | 0.125         | 1.0
+        """
+        results = {}
+        metric_keys = {}  
+
+        for dataset_name, y_true, y_pred_binary, y_pred_continuous, y_pred in self.dataset_evaluations:
+            dataset_results = {}
+            for metric_name, params in self.metrics:
+                metric = Registry.get_metric(metric_name, **params)
+
+                # Computar valor según tipo de métrica
+                if getattr(metric, "binary_prediction", False):
+                    if y_pred_binary is not None:
+                        value = metric.compute(y_true, y_pred_binary)
+                    elif y_pred is not None and set(np.unique(y_pred)).issubset({0, 1}):
+                        value = metric.compute(y_true, y_pred)
+                    else:
+                        warnings.warn(
+                            f"Metric {metric_name} requires binary input, "
+                            f"but dataset {dataset_name} provided non-binary predictions. Skipped.",
+                            UserWarning
+                        )
+                        value = None
+                else:
+                    if y_pred_continuous is not None:
+                        value = metric.compute(y_true, y_pred_continuous)
+                    elif y_pred is not None and not set(np.unique(y_pred)).issubset({0, 1}):
+                        value = metric.compute(y_true, y_pred)
+                    else:
+                        warnings.warn(
+                            f"Metric {metric_name} requires continuous input, "
+                            f"but dataset {dataset_name} provided binary predictions. Skipped.",
+                            UserWarning
+                        )
+                        value = None
+
+                # Generar clave única usando parámetros si ya existe
+                base_key = metric_name
+                key = f"{metric_name}({params})" if params else metric_name
+                if key in metric_keys and metric_keys[key] != params:
+                    key = f"{metric_name}({params})"
+                metric_keys[key] = params
+
+                dataset_results[key] = value
+
+            results[dataset_name] = dataset_results
+
+        # Construir DataFrame con primera fila = parámetros
+        if results:
+            first_dataset = next(iter(results.values()))
+            metric_names = []
+            metric_params = []
+
+            for metric_config in first_dataset.keys():
+                if "(" in metric_config:
+                    name, params_str = metric_config.split("(", 1)
+                    params_str = params_str.rstrip(")")
+                else:
+                    name, params_str = metric_config, ""
+                metric_names.append(name)
+                metric_params.append(params_str if params_str != "{}" else "")
+
+            df_data = [metric_params]  # primera fila = parámetros
+            for dataset_metrics in results.values():
+                df_data.append([dataset_metrics[m] for m in dataset_metrics.keys()])
+
+            df = pd.DataFrame(df_data, columns=metric_names)
+            df.index = ["params"] + list(results.keys())
+            df.index.name = 'dataset'
+        else:
+            df = pd.DataFrame()
+
+        if generate_report:
+            report = Report()
+            report.generate_report(df, report_file)
+
+        return df

tsadmetrics/metrics/Registry.py

@@ -0,0 +1,141 @@
+from typing import Type
+import yaml
+from ..base.Metric import Metric
+from .spm import *
+from .tem.tpdm import *
+from .tem.ptdm import *
+from .tem.tmem import *
+from .tem.dpm import *
+from .tem.tstm import *
+
+class Registry:
+    """
+    Central registry for anomaly detection metrics.
+
+    This class provides a centralized interface to register, retrieve,
+    and load metric classes for anomaly detection tasks.
+    """
+
+    _registry = {}
+
+    @classmethod
+    def register(cls, metric_cls: Type[Metric]):
+        """
+        Register a metric class using its `name` attribute.
+
+        Args:
+            metric_cls (Type[Metric]): The metric class to register. 
+                The class must define a ``name`` attribute.
+
+        Raises:
+            ValueError: If the metric class does not define a ``name`` 
+                attribute or if a metric with the same name is already registered.
+        """
+        if not hasattr(metric_cls, "name"):
+            raise ValueError(f"Metric class {metric_cls.__name__} must define a 'name' attribute.")
+
+        name = metric_cls.name
+        if name in cls._registry:
+            raise ValueError(f"Metric '{name}' is already registered.")
+
+        cls._registry[name] = metric_cls
+
+    @classmethod
+    def get_metric(cls, name: str, **params) -> Metric:
+        """
+        Retrieve and instantiate a registered metric by name.
+
+        Args:
+            name (str): Name of the metric to retrieve.
+            \*\*params: Parameters to initialize the metric instance.
+
+        Returns:
+            Metric: An instance of the requested metric.
+
+        Raises:
+            ValueError: If the metric name is not registered.
+        """
+        if name not in cls._registry:
+            raise ValueError(f"Metric '{name}' is not registered.")
+        return cls._registry[name](**params)
+
+    @classmethod
+    def available_metrics(cls):
+        """
+        List all registered metric names.
+
+        Returns:
+            list[str]: A list of registered metric names.
+        """
+        return list(cls._registry.keys())
+    
+    @classmethod
+    def load_metrics_info_from_file(cls, filepath: str):
+        """
+        Load metric definitions (names and parameters) from a YAML configuration file.
+
+        Args:
+            filepath (str): Path to the YAML file.
+
+        Returns:
+            list[tuple[str, dict]]: A list of tuples containing the metric name and 
+            its parameters, e.g. ``[("metric_name", {"param1": value, ...}), ...]``.
+
+        Raises:
+            ValueError: If the YAML file contains invalid entries or unsupported format.
+        """
+        with open(filepath, "r") as f:
+            config = yaml.safe_load(f)
+
+        metrics_info = []
+
+        if isinstance(config, list):
+            # If YAML is a list of metric names or dicts
+            for entry in config:
+                if isinstance(entry, str):
+                    metrics_info.append((entry, {}))
+                elif isinstance(entry, dict):
+                    for name, params in entry.items():
+                        metrics_info.append((name, params or {}))
+                else:
+                    raise ValueError(f"Invalid metric entry: {entry}")
+        elif isinstance(config, dict):
+            # If YAML is a dictionary: {metric: params}
+            for name, params in config.items():
+                if params is None:
+                    params = {}
+                metrics_info.append((name, params))
+        else:
+            raise ValueError("YAML format must be a list or dict.")
+        return metrics_info
+
+    @classmethod
+    def load_metrics_from_file(cls, filepath: str):
+        """
+        Load and instantiate metrics from a YAML configuration file.
+
+        Args:
+            filepath (str): Path to the YAML configuration file.
+
+        Returns:
+            list[tuple[str, dict]]: A list of tuples containing the metric name and 
+            the parameters used to instantiate it.
+        """
+        metrics_info = cls.load_metrics_info_from_file(filepath)
+        metrics = []
+        for name, params in metrics_info:
+            metric = cls.get_metric(name, **params)
+            metrics.append((name, params))
+        return metrics
+
+
+# --- Auto-discovery
+def auto_register():
+    """
+    Automatically register all subclasses of ``Metric`` found in the project.
+
+    This function inspects the current inheritance tree of ``Metric`` and 
+    registers each subclass in the central registry.
+    """
+    for metric_cls in Metric.__subclasses__():
+        Registry.register(metric_cls)

tsadmetrics/metrics/__init__.py

@@ -0,0 +1,2 @@
+from .Registry import auto_register
+auto_register()