tsadmetrics 0.1.17__py3-none-any.whl → 1.0.1__py3-none-any.whl

examples/specific_examples/PointadjustedFScore_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mdpt.PointadjustedFScore import PointadjustedFScore
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = PointadjustedFScore()
+result = metric.compute(y_true, y_pred)
+print("PointadjustedFScore:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("paf", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/RangebasedFScore_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mdpt.RangebasedFScore import RangebasedFScore
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = RangebasedFScore()
+result = metric.compute(y_true, y_pred)
+print("RangebasedFScore:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("rbf", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/SegmentwiseFScore_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mdpt.SegmentwiseFScore import SegmentwiseFScore
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = SegmentwiseFScore()
+result = metric.compute(y_true, y_pred)
+print("SegmentwiseFScore:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("swf", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/TemporalDistance_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mect.TemporalDistance import TemporalDistance
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = TemporalDistance()
+result = metric.compute(y_true, y_pred)
+print("TemporalDistance:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("td", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/TimeTolerantFScore_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mtdt.TimeTolerantFScore import TimeTolerantFScore
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = TimeTolerantFScore()
+result = metric.compute(y_true, y_pred)
+print("TimeTolerantFScore:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("ttf", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/TimeseriesAwareFScore_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mdtp.TimeseriesAwareFScore import TimeseriesAwareFScore
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = TimeseriesAwareFScore()
+result = metric.compute(y_true, y_pred)
+print("TimeseriesAwareFScore:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("taf", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/TotalDetectedInRange_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mdtp.TotalDetectedInRange import TotalDetectedInRange
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = TotalDetectedInRange()
+result = metric.compute(y_true, y_pred)
+print("TotalDetectedInRange:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("tdir", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/VusPr_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mtdt.VusPr import VusPr
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.99,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+
+# Direct usage
+metric = VusPr()
+result = metric.compute(y_true, y_pred)
+print("VusPr:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("vus_pr", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/VusRoc_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mtdt.VusRoc import VusRoc
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,0.4,0.5,0.6,0.7,0.8,0.9,0.95,0.99,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+
+# Direct usage
+metric = VusRoc()
+result = metric.compute(y_true, y_pred)
+print("VusRoc:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("vus_roc", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

examples/specific_examples/WeightedDetectionDifference_example.py

@@ -0,0 +1,24 @@
+from tsadmetrics.metrics.tem.mdtp.WeightedDetectionDifference import WeightedDetectionDifference
+from tsadmetrics.evaluation.Runner import Runner
+import numpy as np
+
+y_true = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+y_pred = [0,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1]
+
+# Direct usage
+metric = WeightedDetectionDifference()
+result = metric.compute(y_true, y_pred)
+print("WeightedDetectionDifference:", result)
+
+# Usage with Runner
+dataset_evaluations = [
+    ("dataset1", y_true, (y_pred, y_pred))
+]
+
+metrics = [
+    ("wdd", {})
+]
+
+runner = Runner(dataset_evaluations, metrics)
+results = runner.run()
+print(results)

tsadmetrics/__init__.py

@@ -1,21 +0,0 @@
-from .binary_metrics import *
-from .non_binary_metrics import *
-from .utils import *
-
-
-
-
-__author__ = 'Pedro Rafael Velasco Priego i12veprp@uco.es'
-__version__ = "0.1.3"
-__all__ = ['point_wise_recall', 'point_wise_precision', 'point_wise_f_score','point_adjusted_recall',
-            'point_adjusted_precision', 'point_adjusted_f_score', 'segment_wise_recall', 'segment_wise_precision',
-            'segment_wise_f_score','delay_th_point_adjusted_recall', 'delay_th_point_adjusted_precision',
-              'delay_th_point_adjusted_f_score','point_adjusted_at_k_recall','point_adjusted_at_k_precision',
-              'point_adjusted_at_k_f_score','latency_sparsity_aw_recall', 'latency_sparsity_aw_precision',
-                'latency_sparsity_aw_f_score','composite_f_score','time_tolerant_recall','time_tolerant_precision',
-                'time_tolerant_f_score','range_based_recall','range_based_precision','range_based_f_score','ts_aware_recall',
-                'ts_aware_precision','ts_aware_f_score','enhanced_ts_aware_recall','enhanced_ts_aware_precision','enhanced_ts_aware_f_score',
-                'affiliation_based_recall','affiliation_based_precision','affiliation_based_f_score','nab_score','temporal_distance',
-                'average_detection_count','absolute_detection_distance','total_detected_in_range','detection_accuracy_in_range','weighted_detection_difference',
-                'binary_pate','real_pate','mean_time_to_detect',
-                'precision_at_k','auc_roc_pw','auc_pr_pw','auc_roc_pa','auc_pr_pa','vus_roc','vus_pr', 'compute_metrics', 'compute_metrics_from_file']

tsadmetrics/base/Metric.py

@@ -0,0 +1,188 @@
+import yaml
+import numpy as np
+
+class Metric:
+    """
+    Base class for time series anomaly detection metrics.
+
+    This class provides common functionality for metric configuration, including
+    parameter validation from a YAML configuration file and support for a parameter
+    schema defined in each subclass.
+
+    Parameters:
+        name (str, optional):
+            The name of the metric. If not provided, it defaults to the lowercase
+            name of the subclass.
+        config_file (str, optional):
+            Path to a YAML configuration file. Parameters defined in the file under
+            the metric's name will be loaded automatically.
+        \*\*params:
+            Additional parameters passed directly to the metric. These override
+            those loaded from the configuration file.
+
+    Attributes:
+        name (str):
+            Name of the metric instance.
+        params (dict):
+            Dictionary of parameters used by the metric.
+        binary_prediction (bool):
+            Whether the metric expects binary predictions (True) or continuous scores (False).
+
+    Raises:
+        ValueError:
+            If a required parameter is missing or if the configuration file is not found.
+        TypeError:
+            If a parameter does not match its expected type as defined in the schema.
+    """
+
+    def __init__(self, name=None, config_file=None, **params):
+        self.name = name or self.__class__.__name__.lower()
+
+        # Ensure subclasses define binary_prediction
+        if not hasattr(self.__class__, "binary_prediction"):
+            raise ValueError(
+                f"Subclass {self.__class__.__name__} must define class attribute 'binary_prediction' (True/False)."
+            )
+        if not isinstance(self.__class__.binary_prediction, bool):
+            raise TypeError(
+                f"'binary_prediction' in {self.__class__.__name__} must be of type bool."
+            )
+
+        self.binary_prediction = self.__class__.binary_prediction
+        self.params = {}
+        self.configure(config_file=config_file, **params)
+
+
+    def configure(self, config_file=None, **params):
+        """
+        Load and validate metric parameters from a YAML configuration file
+        and/or from explicit keyword arguments.
+
+        Parameters:
+            config_file (str, optional):
+                Path to the configuration file. If provided, it will load parameters
+                under the section with the metric's name.
+            \*\*params:
+                Parameters passed directly to the metric instance.
+
+        Raises:
+            ValueError:
+                If a required parameter is not specified or the configuration file is missing.
+            TypeError:
+                If a parameter value does not match the expected type.
+        """
+        if config_file:
+            try:
+                with open(config_file, 'r') as f:
+                    config = yaml.safe_load(f)
+                    file_params = config.get(self.name.lower(), {})
+                    self.params.update(file_params)
+            except FileNotFoundError:
+                raise ValueError(f"Configuration file '{config_file}' not found.")
+
+        self.params.update(params)
+
+        schema = getattr(self.__class__, 'param_schema', {})
+        for key, rules in schema.items():
+            if key not in self.params:
+                if 'default' in rules:
+                    self.params[key] = rules['default']
+                else:
+                    raise ValueError(f"Required parameter '{key}' not specified.")
+
+            if 'type' in rules and key in self.params:
+                expected_type = rules['type']
+                if expected_type is float:
+                    if not isinstance(self.params[key], (float, int)):
+                        raise TypeError(f"Parameter '{key}' must be of type float, got {type(self.params[key]).__name__} instead.")
+                else:
+                    if not isinstance(self.params[key], expected_type):
+                        raise TypeError(f"Parameter '{key}' must be of type {expected_type.__name__}, got {type(self.params[key]).__name__} instead.")
+
+    def _validate_inputs(self, y_true, y_pred):
+        """
+        Validate that y_true and y_pred are valid sequences of the same length.
+
+        If binary_prediction = True:
+            Both y_true and y_pred must be binary (0 or 1).
+        If binary_prediction = False:
+            y_true must be binary (0 or 1), y_pred can be continuous values.
+
+        Raises:
+            ValueError: If lengths differ or values are not valid.
+            TypeError: If inputs are not array-like.
+        """
+        y_true = np.asarray(y_true)
+        y_pred = np.asarray(y_pred)
+
+        if y_true.shape != y_pred.shape:
+            raise ValueError(
+                f"Shape mismatch: y_true has shape {y_true.shape}, y_pred has shape {y_pred.shape}."
+            )
+
+        if y_true.ndim != 1 or y_pred.ndim != 1:
+            raise ValueError("y_true and y_pred must be 1D arrays.")
+
+        if not np.isin(y_true, [0, 1]).all():
+            raise ValueError("y_true must contain only 0 or 1.")
+
+        if self.binary_prediction:
+            if not np.isin(y_pred, [0, 1]).all():
+                raise ValueError("y_pred must contain only 0 or 1 (binary_prediction=True).")
+
+        return y_true, y_pred
+    
+    def _compute(self, y_true, y_pred):
+        """
+        Compute the value of the metric (core implementation).
+
+        This method contains the actual logic of the metric and must be 
+        implemented by subclasses. It is automatically called by 
+        `compute()` after input validation.
+
+        Parameters:
+            y_true (array-like):
+                Ground truth binary labels.
+            y_pred (array-like):
+                Predicted binary labels.
+
+        Returns:
+            float: The value of the metric.
+
+        Raises:
+            NotImplementedError: If the method is not overridden by a subclass.
+        """
+        raise NotImplementedError("Subclasses must implement _compute().")
+
+
+
+    def compute(self, y_true, y_pred):
+        """
+        Compute the value of the metric (wrapper method).
+
+        This method performs input validation and then calls the internal
+        `_compute()` method, which contains the actual metric logic. 
+
+        **Important:** Subclasses **should not override** this method. 
+        Instead, implement `_compute()` to define the behavior of the metric.
+
+        Parameters
+        ----------
+        y_true : array-like
+            Ground truth binary labels.
+        y_pred : array-like
+            Predicted binary labels.
+
+        Returns
+        -------
+        float
+            The value of the metric.
+
+        Raises
+        ------
+        NotImplementedError
+            If `_compute()` is not implemented by the subclass.
+        """
+        y_true, y_pred = self._validate_inputs(y_true, y_pred)
+        return self._compute(y_true, y_pred)
+

tsadmetrics/evaluation/Report.py

@@ -0,0 +1,25 @@
+import pandas as pd
+
+class Report:
+    def __init__(self):
+        pass
+    def generate_report(self, results, output_file):
+        """
+        Generate a report from the evaluation results.
+
+        Parameters:
+            results (dict):
+                Dictionary containing evaluation results.
+            output_file (str):
+                Path to the output file where the report will be saved.
+        """
+        
+        if type(results) is dict:
+            df = pd.DataFrame.from_dict(results, orient='index')
+
+            df.index.name = 'dataset'
+            df.reset_index(inplace=True)
+
+            df.to_csv(output_file, index=False, sep=';')
+        else:
+            results.to_csv(output_file, sep=';')