tsadmetrics 0.1.17__py3-none-any.whl → 1.0.0__py3-none-any.whl

tsadmetrics/_tsadeval/affiliation/generics.py

@@ -1,135 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-from itertools import groupby
-from operator import itemgetter
-import math
-import gzip
-import glob
-import os
-
-def convert_vector_to_events(vector = [0, 1, 1, 0, 0, 1, 0]):
-    """
-    Convert a binary vector (indicating 1 for the anomalous instances)
-    to a list of events. The events are considered as durations,
-    i.e. setting 1 at index i corresponds to an anomalous interval [i, i+1).
-    
-    :param vector: a list of elements belonging to {0, 1}
-    :return: a list of couples, each couple representing the start and stop of
-    each event
-    """
-    positive_indexes = [idx for idx, val in enumerate(vector) if val > 0]
-    events = []
-    for k, g in groupby(enumerate(positive_indexes), lambda ix : ix[0] - ix[1]):
-        cur_cut = list(map(itemgetter(1), g))
-        events.append((cur_cut[0], cur_cut[-1]))
-    
-    # Consistent conversion in case of range anomalies (for indexes):
-    # A positive index i is considered as the interval [i, i+1),
-    # so the last index should be moved by 1
-    events = [(x, y+1) for (x,y) in events]
-        
-    return(events)
-
-def infer_Trange(events_pred, events_gt):
-    """
-    Given the list of events events_pred and events_gt, get the
-    smallest possible Trange corresponding to the start and stop indexes 
-    of the whole series.
-    Trange will not influence the measure of distances, but will impact the
-    measures of probabilities.
-    
-    :param events_pred: a list of couples corresponding to predicted events
-    :param events_gt: a list of couples corresponding to ground truth events
-    :return: a couple corresponding to the smallest range containing the events
-    """
-    if len(events_gt) == 0:
-        raise ValueError('The gt events should contain at least one event')
-    if len(events_pred) == 0:
-        # empty prediction, base Trange only on events_gt (which is non empty)
-        return(infer_Trange(events_gt, events_gt))
-        
-    min_pred = min([x[0] for x in events_pred])
-    min_gt = min([x[0] for x in events_gt])
-    max_pred = max([x[1] for x in events_pred])
-    max_gt = max([x[1] for x in events_gt])
-    Trange = (min(min_pred, min_gt), max(max_pred, max_gt))
-    return(Trange)
-
-def has_point_anomalies(events):
-    """
-    Checking whether events contain point anomalies, i.e.
-    events starting and stopping at the same time.
-    
-    :param events: a list of couples corresponding to predicted events
-    :return: True is the events have any point anomalies, False otherwise
-    """
-    if len(events) == 0:
-        return(False)
-    return(min([x[1] - x[0] for x in events]) == 0)
-
-def _sum_wo_nan(vec):
-    """
-    Sum of elements, ignoring math.isnan ones
-    
-    :param vec: vector of floating numbers
-    :return: sum of the elements, ignoring math.isnan ones
-    """
-    vec_wo_nan = [e for e in vec if not math.isnan(e)]
-    return(sum(vec_wo_nan))
-    
-def _len_wo_nan(vec):
-    """
-    Count of elements, ignoring math.isnan ones
-    
-    :param vec: vector of floating numbers
-    :return: count of the elements, ignoring math.isnan ones
-    """
-    vec_wo_nan = [e for e in vec if not math.isnan(e)]
-    return(len(vec_wo_nan))
-
-def read_gz_data(filename = 'data/machinetemp_groundtruth.gz'):
-    """
-    Load a file compressed with gz, such that each line of the
-    file is either 0 (representing a normal instance) or 1 (representing)
-    an anomalous instance.
-    :param filename: file path to the gz compressed file
-    :return: list of integers with either 0 or 1
-    """
-    with gzip.open(filename, 'rb') as f:
-        content = f.read().splitlines()
-    content = [int(x) for x in content]
-    return(content)
-
-def read_all_as_events():
-    """
-    Load the files contained in the folder `data/` and convert
-    to events. The length of the series is kept.
-    The convention for the file name is: `dataset_algorithm.gz`
-    :return: two dictionaries:
-        - the first containing the list of events for each dataset and algorithm,
-        - the second containing the range of the series for each dataset
-    """
-    filepaths = glob.glob('data/*.gz')
-    datasets = dict()
-    Tranges = dict()
-    for filepath in filepaths:
-        vector = read_gz_data(filepath)
-        events = convert_vector_to_events(vector)
-        # ad hoc cut for those files
-        cut_filepath = (os.path.split(filepath)[1]).split('_')
-        data_name = cut_filepath[0]
-        algo_name = (cut_filepath[1]).split('.')[0]
-        if not data_name in datasets:
-            datasets[data_name] = dict()
-            Tranges[data_name] = (0, len(vector))
-        datasets[data_name][algo_name] = events
-    return(datasets, Tranges)
-
-def f1_func(p, r):
-    """
-    Compute the f1 function
-    :param p: precision numeric value
-    :param r: recall numeric value
-    :return: f1 numeric value
-    """
-    return(2*p*r/(p+r))

tsadmetrics/_tsadeval/affiliation/metrics.py

@@ -1,114 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-from .generics import (
-        infer_Trange,
-        has_point_anomalies, 
-        _len_wo_nan, 
-        _sum_wo_nan,
-        read_all_as_events)
-from ._affiliation_zone import (
-        get_all_E_gt_func, 
-        affiliation_partition)
-from ._single_ground_truth_event import (
-        affiliation_precision_distance,
-        affiliation_recall_distance,
-        affiliation_precision_proba,
-        affiliation_recall_proba)
-
-def test_events(events):
-    """
-    Verify the validity of the input events
-    :param events: list of events, each represented by a couple (start, stop)
-    :return: None. Raise an error for incorrect formed or non ordered events
-    """
-    if type(events) is not list:
-        raise TypeError('Input `events` should be a list of couples')
-    if not all([type(x) is tuple for x in events]):
-        raise TypeError('Input `events` should be a list of tuples')
-    if not all([len(x) == 2 for x in events]):
-        raise ValueError('Input `events` should be a list of couples (start, stop)')
-    if not all([x[0] <= x[1] for x in events]):
-        raise ValueError('Input `events` should be a list of couples (start, stop) with start <= stop')
-    if not all([events[i][1] < events[i+1][0] for i in range(len(events) - 1)]):
-        raise ValueError('Couples of input `events` should be disjoint and ordered')
-
-def pr_from_events(events_pred, events_gt, Trange):
-    """
-    Compute the affiliation metrics including the precision/recall in [0,1],
-    along with the individual precision/recall distances and probabilities
-    
-    :param events_pred: list of predicted events, each represented by a couple
-    indicating the start and the stop of the event
-    :param events_gt: list of ground truth events, each represented by a couple
-    indicating the start and the stop of the event
-    :param Trange: range of the series where events_pred and events_gt are included,
-    represented as a couple (start, stop)
-    :return: dictionary with precision, recall, and the individual metrics
-    """
-    # testing the inputs
-    # test_events(events_pred)
-    # test_events(events_gt)
-    
-    # other tests
-    minimal_Trange = infer_Trange(events_pred, events_gt)
-    if not Trange[0] <= minimal_Trange[0]:
-        raise ValueError('`Trange` should include all the events')
-    if not minimal_Trange[1] <= Trange[1]:
-        raise ValueError('`Trange` should include all the events')
-    
-    if len(events_gt) == 0:
-        raise ValueError('Input `events_gt` should have at least one event')
-
-    if has_point_anomalies(events_pred) or has_point_anomalies(events_gt):
-        raise ValueError('Cannot manage point anomalies currently')
-
-    if Trange is None:
-        # Set as default, but Trange should be indicated if probabilities are used
-        raise ValueError('Trange should be indicated (or inferred with the `infer_Trange` function')
-
-    E_gt = get_all_E_gt_func(events_gt, Trange)
-    aff_partition = affiliation_partition(events_pred, E_gt)
-
-    # # Computing precision distance
-    # d_precision = [affiliation_precision_distance(Is, J) for Is, J in zip(aff_partition, events_gt)]
-    
-    # # Computing recall distance
-    # d_recall = [affiliation_recall_distance(Is, J) for Is, J in zip(aff_partition, events_gt)]
-
-    # Computing precision
-    p_precision = [affiliation_precision_proba(Is, J, E) for Is, J, E in zip(aff_partition, events_gt, E_gt)]
-
-    # Computing recall
-    p_recall = [affiliation_recall_proba(Is, J, E) for Is, J, E in zip(aff_partition, events_gt, E_gt)]
-
-    if _len_wo_nan(p_precision) > 0:
-        p_precision_average = _sum_wo_nan(p_precision) / _len_wo_nan(p_precision)
-    else:
-        p_precision_average = p_precision[0] # math.nan
-    p_recall_average = sum(p_recall) / len(p_recall)
-
-    dict_out = dict({'precision': p_precision_average,
-                     'recall': p_recall_average,
-                     'individual_precision_probabilities': p_precision,
-                     'individual_recall_probabilities': p_recall})
-    return(dict_out)
-
-def produce_all_results():
-    """
-    Produce the affiliation precision/recall for all files
-    contained in the `data` repository
-    :return: a dictionary indexed by data names, each containing a dictionary
-    indexed by algorithm names, each containing the results of the affiliation
-    metrics (precision, recall, individual probabilities and distances)
-    """
-    datasets, Tranges = read_all_as_events() # read all the events in folder `data`
-    results = dict()
-    for data_name in datasets.keys():
-        results_data = dict()
-        for algo_name in datasets[data_name].keys():
-            if algo_name != 'groundtruth':
-                results_data[algo_name] = pr_from_events(datasets[data_name][algo_name],
-                                                         datasets[data_name]['groundtruth'],
-                                                         Tranges[data_name])
-        results[data_name] = results_data
-    return(results)

tsadmetrics/_tsadeval/auc_roc_pr_plot.py

@@ -1,295 +0,0 @@
-# import numpy as np
-# from scipy.stats import norm
-# from matplotlib import pyplot as plt
-
-# from metrics import f1_from_pr
-
-
-# class Two_1d_normal_distributions:
-#     def __init__(self, P_ampl, N_ampl, P_mu, N_mu, P_std, N_std, color="b", betas=None):
-#         self.P_ampl = P_ampl
-#         self.N_ampl = N_ampl
-#         self.P_mu = P_mu
-#         self.N_mu = N_mu
-#         self.P_std = P_std
-#         self.N_std = N_std
-#         self.betas = (1 / 8, 1 / 4, 1 / 2, 1, 2, 4, 8, 16, 32) if betas == None else betas
-
-#         self.color = color
-#         self.N_color = "k"
-#         self.P_color = "r"
-
-#     def make(self, delta=0.05, steps=10001, start=-8, stop=8):
-#         index = 0
-
-#         # For plotting grahps
-#         self.fpr = []
-#         self.precision = []
-#         self.recall = []
-
-#         # For plotting x´s and o´s on the graphs
-#         self.x_fpr = []
-#         self.x_precision = []
-#         self.x_recall = []
-#         self.xs = 0
-#         self.x_threshold = []
-#         self.o_fpr = []
-#         self.o_precision = []
-#         self.o_recall = []
-#         self.os = 0
-#         self.o_threshold = []
-
-#         # Track maximum f scores for various beta values
-#         self.max_f = {i: 0 for i in self.betas}
-#         self.max_f_fpr = {i: 0 for i in self.betas}
-#         self.max_f_precision = {i: 0 for i in self.betas}
-#         self.max_f_recall = {i: 0 for i in self.betas}
-#         self.max_f_thresholds = {i: 0 for i in self.betas}
-
-#         for threshold in np.linspace(start, stop, steps):
-#             TN = self.N_ampl * norm.cdf(threshold, loc=self.N_mu, scale=self.N_std)
-#             FP = self.N_ampl - TN
-#             FN = self.P_ampl * norm.cdf(threshold, loc=self.P_mu, scale=self.P_std)
-#             TP = self.P_ampl - FN
-#             self.fpr.append(FP / (FP + TN))
-#             self.precision.append(TP / (TP + FP))
-#             self.recall.append(TP / (TP + FN))
-
-#             for beta in self.betas:
-#                 if f1_from_pr(p=self.precision[-1], r=self.recall[-1], beta=beta) > self.max_f[beta]:
-#                     self.max_f[beta] = f1_from_pr(p=self.precision[-1], r=self.recall[-1], beta=beta)
-#                     self.max_f_fpr[beta] = self.fpr[-1]
-#                     self.max_f_precision[beta] = self.precision[-1]
-#                     self.max_f_recall[beta] = self.recall[-1]
-#                     self.max_f_thresholds[beta] = threshold
-
-#             if (FN) / (self.P_ampl) >= self.xs * delta + delta * 0.5:
-#                 self.x_fpr.append(FP / (FP + TN))
-#                 self.x_precision.append(TP / (TP + FP))
-#                 self.x_recall.append(TP / (TP + FN))
-#                 self.xs += 1
-#                 # print(xs, TN+FN)
-#                 self.x_threshold.append(threshold)
-#             if (TN / self.N_ampl) >= self.os * delta + delta * 0.5:
-#                 self.o_fpr.append(FP / (FP + TN))
-#                 self.o_precision.append(TP / (TP + FP))
-#                 self.o_recall.append(TP / (TP + FN))
-#                 self.os += 1
-#                 self.o_threshold.append(threshold)
-#                 # print(os, TN+FN)
-
-#     def plot_roc_pr(self, roc_ax, pr_ax, plot_xs=True, plot_os=True, plot_fs=False):
-#         roc_ax.plot(self.fpr, self.recall, self.color, zorder=1)
-#         pr_ax.plot(self.precision, self.recall, self.color, zorder=1)
-#         if plot_xs:
-#             roc_ax.plot(self.x_fpr, self.x_recall, "x", color=self.color, zorder=1)
-#             pr_ax.plot(self.x_precision, self.x_recall, "x", color=self.color, zorder=1)
-#         if plot_os:
-#             roc_ax.plot(self.o_fpr, self.o_recall, "o", color=self.color, fillstyle="none", zorder=1)
-#             pr_ax.plot(self.o_precision, self.o_recall, "o", color=self.color, fillstyle="none", zorder=1)
-#         if plot_fs:
-#             roc_ax.plot(
-#                 list(self.max_f_fpr.values()),
-#                 list(self.max_f_recall.values()),
-#                 ".",
-#                 linestyle="None",
-#                 zorder=2,
-#                 color="k",
-#             )  # self.color)
-#             pr_ax.plot(
-#                 list(self.max_f_precision.values()), list(self.max_f_recall.values()), ".", zorder=2, color="k"
-#             )  # self.color)
-#             for beta in self.betas:
-#                 # roc_ax.plot([self.max_f_fpr[beta]],[self.max_f_recall[beta]], marker=f"$1/{int(1/beta)}$" if beta<1 else f"${beta}$", linestyle= "None", zorder=2, color="k")
-#                 # pr_ax.plot([self.max_f_precision[beta]], [self.max_f_recall[beta]], marker=f"$1/{int(1/beta)}$" if beta<1 else f"${beta}$", zorder=2, color="k")
-#                 if self.color == "forestgreen":  # need to place the numbers differently
-#                     roc_ax.text(
-#                         self.max_f_fpr[beta],
-#                         self.max_f_recall[beta],
-#                         f" $1/{int(1/beta)}$" if beta < 1 else f" ${beta}$",
-#                         horizontalalignment="left",
-#                         verticalalignment="top",
-#                         color=self.color,
-#                     )
-#                     pr_ax.text(
-#                         self.max_f_precision[beta],
-#                         self.max_f_recall[beta],
-#                         f"$1/{int(1/beta)}$" if beta < 1 else f"${beta}$",
-#                         horizontalalignment="left",
-#                         verticalalignment="bottom",
-#                         color=self.color,
-#                     )
-#                 else:
-#                     roc_ax.text(
-#                         self.max_f_fpr[beta],
-#                         self.max_f_recall[beta],
-#                         f"$1/{int(1/beta)}$" if beta < 1 else f"${beta}$",
-#                         horizontalalignment="right",
-#                         verticalalignment="bottom",
-#                         color=self.color,
-#                     )
-#                     pr_ax.text(
-#                         self.max_f_precision[beta],
-#                         self.max_f_recall[beta],
-#                         f"$1/{int(1/beta)}$ " if beta < 1 else f"${beta}$ ",
-#                         horizontalalignment="right",
-#                         verticalalignment="top",
-#                         color=self.color,
-#                     )
-
-#             # adjust axes to get the numbers within the figure
-#             xmin, xmax = roc_ax.get_xlim()
-#             xmin, xmax = pr_ax.get_xlim()
-#             roc_ax.set_xlim([xmin - 0.04, xmax])
-#             pr_ax.set_xlim([xmin - 0.01, xmax])
-#         pr_ax.set_xlabel("Precision")
-#         pr_ax.set_ylabel("Recall")
-#         roc_ax.set_xlabel("False positive rate")
-#         roc_ax.set_ylabel("Recall")
-
-#     def plot_roc_prec(self):
-#         plt.plot(self.fpr, self.recall, self.color, zorder=1)
-#         plt.plot(self.fpr, self.precision, self.color, zorder=1)
-#         plt.show()
-
-#     def plot_roc_pr_lines(self, ax):
-#         ax.plot(self.fpr, np.array(self.recall) + 1, self.color, zorder=1)
-#         ax.plot(np.array(self.precision) * (-1) + 1, self.recall, self.color, zorder=1)
-#         for i in range(0, len(self.recall), 5):
-#             ax.plot(
-#                 [self.fpr[i], 1 - self.precision[i]],
-#                 [self.recall[i] + 1, self.recall[i]],
-#                 marker="o",
-#                 color=self.N_color,
-#                 zorder=1,
-#                 alpha=0.3,
-#             )
-
-#     def plot_distributions(
-#         self, axes, start=-5, stop=7, steps=1001, normalize=True, plot_xs=True, plot_os=True, plot_fs=False, threshold=0
-#     ):
-#         grid = np.linspace(start, stop, steps)
-#         fill_alpha = 0.2
-
-#         y = lambda x: norm.pdf(x, loc=self.N_mu, scale=self.N_std) * (1 if normalize else self.N_ampl)
-#         axes[0].plot(
-#             grid,
-#             y(grid),
-#             color=self.N_color,
-#             label=f"pdf_N/{self.N_ampl}",
-#         )
-
-#         axes[0].fill_between(
-#             grid[grid <= threshold], 0, y(grid[grid <= threshold]), alpha=fill_alpha, lw=0, color="darkgreen"
-#         )
-#         axes[0].fill_between(
-#             grid[grid >= threshold], 0, y(grid[grid >= threshold]), alpha=fill_alpha, lw=0, color="orchid"
-#         )
-#         tn_x = min(self.N_mu, threshold - 0.75)
-#         fp_x = max(self.N_mu, threshold + 0.75)
-#         axes[0].text(tn_x, y(tn_x) / 2 - 0.005, "TN", horizontalalignment="center", verticalalignment="top")
-#         axes[0].text(fp_x, y(fp_x) / 2 - 0.005, "FP", horizontalalignment="center", verticalalignment="top")
-#         # add thresholdline, on the whole y-range
-#         ymin, ymax = axes[0].get_ylim()
-#         axes[0].plot([threshold, threshold], [ymin - 1, ymax + 1], "--", color="gray", lw=1)
-#         axes[0].set_ylim([ymin - 0.02, ymax])
-
-#         # same for anomal distributions
-#         y = lambda x: norm.pdf(x, loc=self.P_mu, scale=self.P_std) * (1 if normalize else self.P_ampl)
-#         axes[1].plot(
-#             grid,
-#             y(grid),
-#             color=self.P_color,
-#             label=f"pdf_P/{self.P_ampl}",
-#         )
-#         axes[1].fill_between(
-#             grid[grid <= threshold], 0, y(grid[grid <= threshold]), alpha=fill_alpha, lw=0, color="chocolate"
-#         )
-#         axes[1].fill_between(
-#             grid[grid >= threshold], 0, y(grid[grid >= threshold]), alpha=fill_alpha, lw=0, color="darkcyan"
-#         )
-#         fn_x = min(self.P_mu, threshold - 0.75)
-#         tp_x = max(self.P_mu, threshold + 0.75)
-#         axes[1].text(fn_x, y(fn_x) / 2, "FN", horizontalalignment="center", verticalalignment="top")
-#         axes[1].text(tp_x, y(tp_x) / 2, "TP", horizontalalignment="center", verticalalignment="top")
-
-#         # add thresholdline, on the whole y-range
-#         ymin, ymax = axes[1].get_ylim()
-#         axes[1].plot([threshold, threshold], [ymin - 1, ymax + 1], "--", color="gray", lw=1)
-#         axes[1].set_ylim([ymin, ymax])
-
-
-#     def plot_cdf(self, ax, start=-6, stop=8, steps=1001, normalize=True):
-#         grid = np.linspace(start, stop, steps)
-
-#         ax.plot(
-#             grid,
-#             norm.cdf(grid, loc=self.N_mu, scale=self.N_std) * (1 if normalize else self.N_ampl),
-#             color=self.N_color,
-#             label=f"pdf_N/{self.N_ampl}",
-#         )
-#         ax.plot(
-#             grid,
-#             norm.cdf(grid, loc=self.P_mu, scale=self.P_std) * (1 if normalize else self.P_ampl),
-#             color=self.P_color,
-#             label=f"pdf_P/{self.P_ampl}",
-#         )
-
-
-# if __name__ == "__main__":
-
-#     # Make detector distributions
-
-#     t1 = Two_1d_normal_distributions(
-#         1, 49, 1.8, -1, 2, 1, color="mediumblue", betas=(1 / 8, 1 / 4, 1 / 2, 1, 2, 4, 8, 16)
-#     )
-#     t2 = Two_1d_normal_distributions(
-#         1, 49, 1, -1, 1, 1, color="forestgreen", betas=(1 / 8, 1 / 4, 1 / 2, 1, 2, 4, 8, 16)
-#     )
-
-#     t1.make(steps=1001, delta=0.1)
-#     t2.make(steps=1001, delta=0.1)
-
-    
-#     #  Make roc and pr plots
-
-#     figsize = (4, 4)
-
-#     roc_fig, roc_ax = plt.subplots(figsize=figsize)
-#     pr_fig, pr_ax = plt.subplots(figsize=figsize)
-#     t1.plot_roc_pr(roc_ax, pr_ax, False, False, True)
-#     t2.plot_roc_pr(roc_ax, pr_ax, False, False, True)
-#     roc_fig.tight_layout()
-#     pr_fig.tight_layout()
-#     roc_fig.savefig("auc_roc_f.pdf")
-#     pr_fig.savefig("auc_pr_f.pdf")
-#     plt.show()
-#     plt.close("all")
-
-
-#     # Make distribution plots
-
-#     figsize = (5, 3)
-
-#     for beta in t1.betas:  # [16,4,1,1/4,1/8]:
-#         fig, axes = plt.subplots(2, 2, figsize=figsize, sharex=True, sharey=True)
-#         t1.plot_distributions(
-#             [axes[0][0], axes[1][0]], plot_xs=False, plot_os=False, plot_fs=True, threshold=t1.max_f_thresholds[beta]
-#         )
-#         t2.plot_distributions(
-#             [axes[0][1], axes[1][1]], plot_xs=False, plot_os=False, plot_fs=True, threshold=t2.max_f_thresholds[beta]
-#         )
-
-#         axes[0][0].set_title(f"Blue detector", color=t1.color)
-#         axes[0][1].set_title("Green detector", color=t2.color)
-#         axes[1][0].set_xlabel("Anomaly \n score", color=t1.color)
-#         axes[1][1].set_xlabel("Anomaly \n score", color=t2.color)
-#         shadowaxes = fig.add_subplot(111, xticks=[], yticks=[], frame_on=False)
-#         shadowaxes.set_ylabel("Probability density", labelpad=25)
-#         fig.tight_layout()
-#         axes[0][0].set_ylabel("Normal\nsamples", labelpad=25)
-#         axes[1][0].set_ylabel("Anomalous\nsamples", labelpad=25)
-
-#         plt.subplots_adjust(hspace=0.0)
-#         plt.savefig(f"auc_distributions_b{beta}.pdf")
-#         plt.show()

tsadmetrics/_tsadeval/discontinuity_graph.py

@@ -1,109 +0,0 @@
-# from maketable import *
-
-
-# class Score_graphs_table(Table):
-#     def __init__(self, metric_names, results, marks=[]):
-#         self.metric_names = metric_names
-#         self.results = results
-#         self.marks = marks
-#         super().__init__(Table_content([], [], []), scale=2)
-#         self.x_factor = 1 / 20 * self.scale
-#         self.y_factor = 1 / 2
-#         self.y_shift = -0.2
-
-#         self.row_length = 2
-#         self.n_rows = len(metric_names)
-
-#     def add_top_row(self):
-#         self.string += "Metric"
-#         self.string += "&"
-#         self.string += "Score"
-#         self.end_row()
-
-#     def add_next_row(self):
-#         self.string += self.metric_names[self.rows_added]
-#         self.string += "&"
-#         self.add_graph(self.rows_added + 1)
-#         self.end_row()
-#         self.rows_added += 1
-
-#     def add_graph(self, number):
-#         self.add_line(f"\\begin{{tikzpicture}}[baseline=-\\the\\dimexpr\\fontdimen22\\textfont2\\relax]")
-#         for x in self.marks:
-#             self.add_line(
-#                 f"\draw[-, gray] ({x*self.x_factor},{self.y_shift}) -- ({x*self.x_factor},{0.2*self.y_factor + self.y_shift});"
-#             )
-#         self.add_line(
-#             f"\draw[-, gray] (0,{self.y_shift}) -- ({self.x_factor*(len(self.results[self.metric_names[number-1]])-1)},{self.y_shift});"
-#         )
-#         self.add_line("\\foreach \\i/\\a in")
-#         self.add_line(
-#             str(
-#                 [
-#                     (round(i * self.x_factor, 3), round(a * self.y_factor + self.y_shift, 3))
-#                     for i, a in enumerate(self.results[self.metric_names[number - 1]])
-#                 ]
-#             )
-#             .replace(",", "/")
-#             .replace(")/", ",")
-#             .replace("(", "")
-#             .replace("[", "{")
-#             .replace("]", "}")
-#             .replace(")}", "}{")
-#         )
-#         self.add_line("\\coordinate (now) at (\\i,\\a) {};")
-#         self.add_line("  \\ifthenelse{\\equal{\\i}{0.0}}{}{")
-#         self.add_line("  \\draw[-, teal, thick] (prev) -- (now);")
-#         self.add_line("  }")
-#         self.add_line("  \\coordinate (prev) at (\\i,\\a) {};")
-#         self.add_line("}")
-#         self.add_line("\\end{tikzpicture}")
-
-
-# def score_graphs():
-#     # define scenario
-#     ts_length = 100
-#     pred_length = 5
-#     gt_length = 20
-#     gt_start = 40
-#     marks = [35, 40, 55, 60]
-#     assert pred_length % 2 == 1
-
-#     # prepare metrics list
-#     All_metrics.remove(metrics.Range_PR)
-#     all_metrics_and_rffront = [*All_metrics, metrics.Range_PR, Range_PR_front]
-
-#     # make results and names
-#     metric_names = []
-#     result = {}
-#     for metric in all_metrics_and_rffront:
-#         # set names
-#         if metric == metrics.TaF:
-#             metric_names.append(metric(5, [3, 4], [3], delta=10).name)
-#         elif metric == metrics.Time_Tolerant:
-#             metric_names.append(metric(5, [3, 4], [3], d=10).name)
-#         else:
-#             metric_names.append(metric(5, [3, 4], [3]).name)
-
-#         # get results
-#         current_result = []
-#         for pred_mid in range(pred_length // 2, ts_length - pred_length // 2):
-#             gt = [[gt_start, gt_start + gt_length - 1]]
-#             pred = [[pred_mid - pred_length // 2, pred_mid + pred_length // 2]]
-#             if metric == metrics.TaF:
-#                 current_result.append(metric(ts_length, gt, pred, delta=10).get_score())
-#             elif metric == metrics.Time_Tolerant:
-#                 current_result.append(metric(ts_length, gt, pred, d=10).get_score())
-#             else:
-#                 current_result.append(metric(ts_length, gt, pred).get_score())
-#         current_result = np.array(current_result)
-#         current_result = (current_result - min(current_result)) / (max(current_result) - min(current_result))
-#         result[metric_names[-1]] = current_result
-
-#     # make the table
-#     table = Score_graphs_table(metric_names, result, marks)
-#     table.write()
-#     print(table)
-
-
-# score_graphs()