tsfel 0.1.7__py3-none-any.whl

tests/benchmark.py

@@ -0,0 +1,160 @@
+import time
+
+import matplotlib.pyplot as plt
+import pandas as pd
+import seaborn as sns
+from datasets.empirical1000_dataset import Empirical1000Dataset
+
+import tsfel
+
+DOMAINS = [
+    "statistical",
+    "temporal",
+    "spectral",
+]  # It specifies the domains included in the analysis.
+
+emp_dataset = Empirical1000Dataset()
+
+#########################################################################################################
+#                  Experiment 1 - Overall executing time benchmarking per domain                        #
+#########################################################################################################
+execution_times_global = pd.DataFrame(
+    {
+        "Domain": pd.Series(dtype="str"),
+        "Execution Time (s)": pd.Series(dtype="int"),
+        "Normalized Execution Time (s/feature)": pd.Series(dtype="float32"),
+    },
+)
+
+for domain in DOMAINS:
+    n_features_domain = len(tsfel.get_features_by_domain(domain)[domain])
+
+    init_time = time.perf_counter()
+    for _id, ts in emp_dataset.raw.items():
+        _ = tsfel.time_series_features_extractor(
+            tsfel.get_features_by_domain(domain),
+            ts.sig,
+            fs=100,
+            verbose=False,
+        )
+    end_time = time.perf_counter()
+    total_time = end_time - init_time
+
+    execution_times_global = pd.concat(
+        [
+            execution_times_global,
+            pd.DataFrame(
+                {
+                    "Domain": [domain],
+                    "Execution Time (s)": [int(total_time)],
+                    "Normalized Execution Time (s/feature)": [
+                        total_time / n_features_domain,
+                    ],
+                },
+            ),
+        ],
+        ignore_index=True,
+    )
+
+#########################################################################################################
+# Experiment 2 - Detailed time benchmarking as function of time series length grouped by feature domain #
+#########################################################################################################
+execution_times_individual = pd.DataFrame(
+    {
+        "Domain": pd.Series(dtype="str"),
+        "ID": pd.Series(dtype="int"),
+        "Execution Time (s)": pd.Series(dtype="float32"),
+        "Length": pd.Series(dtype="int"),
+    },
+)
+
+for domain in ["statistical", "temporal", "spectral"]:
+    for id, ts in emp_dataset.raw.items():
+        init_time = time.perf_counter()
+        _ = tsfel.time_series_features_extractor(
+            tsfel.get_features_by_domain(domain),
+            ts.sig,
+            fs=100,
+            verbose=False,
+        )
+        end_time = time.perf_counter()
+        total_time = end_time - init_time
+
+        execution_times_individual = pd.concat(
+            [
+                execution_times_individual,
+                pd.DataFrame(
+                    {
+                        "Domain": [domain],
+                        "ID": [id],
+                        "Execution Time (s)": [total_time],
+                        "Length": [ts.len],
+                    },
+                ),
+            ],
+            ignore_index=True,
+        )
+
+# Some lengths have a low number of time series for statistical analysis (< 25 samples). We only consider lengths that
+# have at least 25 samples for the analysis. For each length, we calculate the average and standard deviation and plot
+# them on normal and log-log scales.
+length_counts = execution_times_individual["Length"].value_counts()
+valid_lengths = length_counts[length_counts >= 25].index
+filtered_df = execution_times_individual[execution_times_individual["Length"].isin(valid_lengths)]
+grouped_stats = filtered_df.groupby(["Length", "Domain"])["Execution Time (s)"].agg(["mean", "std"]).reset_index()
+
+
+fig, (ax0, ax1) = plt.subplots(ncols=2)
+for domain, marker in zip(DOMAINS, ["o", "v", "^"]):
+    _data_slice = grouped_stats[grouped_stats["Domain"] == domain]
+    ax0.errorbar(
+        _data_slice["Length"],
+        _data_slice["mean"],
+        yerr=_data_slice["std"],
+        label=domain,
+        marker=marker,
+        capsize=4,
+    )
+    ax1.errorbar(
+        _data_slice["Length"],
+        _data_slice["mean"],
+        yerr=_data_slice["std"],
+        label=domain,
+        marker=marker,
+        capsize=4,
+    )
+
+[ax.set(xlabel="Length (#)", ylabel="Execution Time (s)") for ax in [ax0, ax1]]
+[ax.spines[loc].set_visible(False) for ax in [ax0, ax1] for loc in ["top", "right"]]
+[ax.legend() for ax in [ax0, ax1]]
+ax1.set_xscale("log")
+ax1.set_yscale("log")
+
+#########################################################################################################
+#                    Experiment 4 - Measure the overall execution time per feature                      #
+#########################################################################################################
+execution_times_feature = pd.DataFrame(columns=["Domain", "Feature_Name", "Execution_Time"])
+cfg = tsfel.get_features_by_domain()
+for domain in DOMAINS:
+    for feature in cfg[domain]:
+        print(domain, feature)
+        init_time = time.perf_counter()
+        for _, ts in emp_dataset.raw.items():
+            _ = tsfel.time_series_features_extractor(
+                {domain: {feature: cfg[domain][feature]}},
+                ts.sig,
+                fs=100,
+                verbose=False,
+            )
+        execution_time = time.perf_counter() - init_time
+
+        execution_times_feature.loc[len(execution_times_feature)] = {
+            "Domain": domain,
+            "Feature_Name": feature,
+            "Execution_Time": execution_time,
+        }
+
+fig, ax = plt.subplots(1, 1)
+sns.barplot(execution_times_feature, x="Feature_Name", y="Execution_Time", hue="Domain", ax=ax)
+ax.tick_params(axis="x", rotation=90)
+plt.show(block=False)

tests/complexity_measures_intuition.py

@@ -0,0 +1,93 @@
+"""This module offers helper functions designed to facilitate testing and
+enhance understanding of features that quantify repetitiveness in dynamic
+systems."""
+
+import matplotlib.pyplot as plt
+import neurokit2 as nk
+import numpy as np
+from statsmodels.graphics.tsaplots import plot_acf
+from statsmodels.tsa.stattools import acf
+
+from tests.tests_tools.test_data.complexity_datasets import load_complexities_datasets
+
+
+def plot_dfa(info, scale, fluctuations, ax):
+    """Plot log-log visualization of the detrended fluctuation analysis
+    (DFA)."""
+
+    polyfit = np.polyfit(np.log2(scale), np.log2(fluctuations), 1)
+    fluctfit = 2 ** np.polyval(polyfit, np.log2(scale))
+    ax.loglog(scale, fluctuations, "o", c="#90A4AE")
+    ax.loglog(
+        scale,
+        fluctfit,
+        c="#E91E63",
+        label=r"$\alpha$ = {:.3f}".format(info["Alpha"]),
+    )
+
+    return ax
+
+
+def plot_multiscale_entropy(info, ax):
+    """Plots the entropy values for each scale factor."""
+
+    ax.plot(
+        info["Scale"][np.isfinite(info["Value"])],
+        info["Value"][np.isfinite(info["Value"])],
+        color="#FF9800",
+    )
+
+    return ax
+
+
+def get_first_acf_time_constant(signal):
+    """Captures the approximate scale of the autocorrelation function by
+    measuring the first time lag at which the autocorrelation function (acf)
+    drops below 1/e = 0.3679."""
+    n = signal.size
+    maxlag = int(n / 4)
+    threshold = 0.36787944117144233  # 1 / np.exp(1)
+
+    acf_vec = acf(signal, adjusted=True, fft=n > 400, nlags=maxlag)[
+        1:
+    ]  # n > 400 empirically selected based on performance tests
+    idxs = np.where(acf_vec < threshold)[0]
+    first_lag = idxs[0] + 1 if idxs.size > 0 else maxlag
+
+    return first_lag
+
+
+dataset = load_complexities_datasets()
+
+f, ax = plt.subplots(4, len(dataset), figsize=(15, 5))
+for i, (k, _v) in enumerate(dataset.items()):
+    signal = dataset[k]
+
+    # First row (raw data)
+    ax[0, i].plot(signal, "k")
+    ax[0, i].set_title(k)
+    ax[0, i].spines[["left", "top", "right"]].set_visible(False)
+    ax[0, i].tick_params(left=False, right=False, labelleft=False)
+
+    # Second row (first_acf_time_constant)
+    plot_acf(signal, ax[1, i], adjusted=True, fft=len(signal) > 1250, title="")
+    ax[1, i].axhline(y=(1 / np.e), color="r", linestyle="--")
+    ax[1, i].spines[["top", "right"]].set_visible(False)
+    ax[1, i].set_title("$\\tau = %d$" % get_first_acf_time_constant(signal))
+
+    # Third row (dfa)
+    # TODO: Change to our own implementation instead of relying on neurokit2
+    _, info = nk.fractal_dfa(signal)
+    plot_dfa(info, info["scale"], info["Fluctuations"], ax[2, i])
+    ax[2, i].set_title("$\\alpha = %0.2f$" % info["Alpha"])
+    ax[2, i].set_axis_off()
+
+    # Forth row (mse)
+    mse, info = nk.entropy_multiscale(signal)
+    plot_multiscale_entropy(info, ax[3, i])
+    ax[3, i].set_title("mse = %0.2f" % mse)
+    ax[3, i].set_ylim([0, 2.5])
+    ax[3, i].spines[["top", "right"]].set_visible(False)
+
+f.tight_layout()
+plt.show(block=False)

tests/datasets/__init__.py

@@ -0,0 +1,4 @@
+"""A utility module providing simplified access to time series datasets used
+for benchmarking and testing."""
+
+from .empirical1000_dataset import Empirical1000Dataset

tests/datasets/empirical1000_dataset.py

@@ -0,0 +1,188 @@
+import os
+import shutil
+import zipfile
+from pathlib import Path
+from urllib.parse import urlparse
+
+import numpy as np
+import pandas as pd
+import requests
+import scipy.io as sio
+from tqdm import tqdm
+
+module_path = os.path.dirname(__file__)
+data_path = os.path.join(module_path, "cached_datasets")
+
+
+class Empirical1000Dataset:
+    """A convenience class to access the Empirical1000 dataset.
+
+    When using one this dataset, please cite [1]_.
+
+    Parameters
+    ----------
+    use_cache : bool (default: True)
+        Whether a cached version of the dataset should be used, if one is found.
+        The dataset is always cached upon loading, and this parameter only determines
+        whether the cached version shall be refreshed upon loading.
+
+    extract_features : bool (default: False)
+        Whether a feature extraction routine should be called after loading the data files.
+        TODO: Implement this functionality.
+
+    Notes
+    -----
+        To optimize running times it is recommended to use `use_cache=True` (default) in order
+        to only experience a time once downloading and work on a cached version of the dataset afterward.
+
+
+    References
+    ----------
+    .. [1] Fulcher, B. D., Lubba, C. H., Sethi, S. S., & Jones, N. S. (2020).
+    A self-organizing, living library of time-series data. Scientific data, 7(1), 213.
+    """
+
+    def __init__(self, use_cache=True, extract_features=False):
+        self.use_cache = use_cache
+
+        self.raw = {}
+        self.features = {}  # An add-on to this class will contain a featurized data representation.
+
+        self.__dataset_url = "https://figshare.com/ndownloader/articles/5436136/versions/10"
+        self.cache_folder_path = data_path
+        self.__empirical1000_folder_path = os.path.join(
+            self.cache_folder_path,
+            "Empirical1000",
+        )
+
+        if not os.path.exists(self.cache_folder_path) or not os.listdir(self.cache_folder_path) or not use_cache:
+            print("Cache folder is empty. Downloading the Empirical1000 dataset...")
+            Path(os.path.join(self.cache_folder_path, "Empirical1000")).mkdir(
+                parents=True,
+                exist_ok=True,
+            )
+            self.__download_dataset()
+
+        self.__data_files = sio.loadmat(
+            os.path.join(self.__empirical1000_folder_path, "INP_1000ts.mat"),
+        )
+        self.metadata = pd.read_csv(
+            os.path.join(self.__empirical1000_folder_path, "hctsa_timeseries-info.csv"),
+        )
+
+        for i, name in enumerate(self.metadata["Name"]):
+            self.raw[i] = EmpiricalTimeSeries(
+                name,
+                self.__data_files["timeSeriesData"][0][i].flatten(),
+            )
+            if extract_features:
+                tqdm_iterator = tqdm(
+                    total=len(self.metadata["Name"]),
+                    desc="Extracting features.",
+                )
+                tqdm_iterator.update(1)
+
+    def __download_dataset(self):
+        try:
+            # Send a request to the dataset URL with allow_redirects=False to handle redirection
+            response = requests.get(
+                self.__dataset_url,
+                stream=True,
+                allow_redirects=False,
+            )
+
+            # Check if the request was successful (status code 200)
+            if response.status_code == 200:
+                total_size = int(response.headers.get("content-length", 0)) or None
+                filename = self.__extract_filename(response)
+
+                with tqdm(
+                    total=total_size,
+                    unit="B",
+                    unit_scale=True,
+                    desc="Downloading Empirical1000 dataset",
+                    dynamic_ncols=True,
+                ) as pbar:
+                    with open(
+                        os.path.join(self.cache_folder_path, filename),
+                        "wb",
+                    ) as out_file:
+                        shutil.copyfileobj(response.raw, out_file)
+                        pbar.update(os.path.getsize(out_file.name))
+
+                zip_file_path = os.path.join(self.cache_folder_path, filename)
+                with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+                    zip_ref.extractall(self.__empirical1000_folder_path)
+
+                # A sanitizing routine that deletes non-relevant files.
+                os.remove(zip_file_path)
+                extracted_files = os.listdir(self.__empirical1000_folder_path)
+                timeseries_files = [
+                    file for file in extracted_files if "timeseries-info" in file.lower() or "inp" in file.lower()
+                ]
+
+                for file in extracted_files:
+                    if file not in timeseries_files:
+                        os.remove(os.path.join(self.__empirical1000_folder_path, file))
+
+                print(f"Dataset downloaded and saved to: {self.cache_folder_path}")
+            else:
+                print(
+                    f"Failed to download dataset. Status code: {response.status_code}",
+                )
+
+        except Exception as e:
+            print(f"An error occurred: {e}")
+
+    @staticmethod
+    def __extract_filename(response):
+        # Try to extract the filename from the Content-Disposition header.
+        # If the header is not present, extract the filename from the URL
+        content_disposition = response.headers.get("Content-Disposition")
+        if content_disposition and "filename=" in content_disposition:
+            filename = content_disposition.split("filename=")[1].strip("\";'")
+        else:
+            filename = os.path.basename(urlparse(response.url).path)
+
+        return filename
+
+    def get_by_len(self, l):
+        query = np.nonzero(self.metadata["Length"] == l)[0]
+        filtered_data = {key: self.raw[key] for key in query if key in self.raw}
+
+        return filtered_data
+
+
+class EmpiricalTimeSeries:
+    """The object representing a time series from the Empirical1000 dataset.
+
+    Attributes:
+        name (str): The name of the time series.
+        sig (array-like): The time series data.
+        len (int): The length of the time series.
+
+    Methods:
+        __init__(self, name, sig):
+            Initializes a new EmpiricalTimeSeries object.
+    """
+
+    def __init__(self, name, sig):
+        """Initializes a new EmpiricalTimeSeries object.
+
+        Parameters:
+            name (str): The name or identifier of the time series.
+            sig (list or array-like): The time series data.
+        """
+        self.name = name
+        self.sig = sig
+        self.len = len(sig)
+
+
+class FeaturizedEmpiricalTimeSeries(EmpiricalTimeSeries):
+    def __init__(self, empirical_time_series):
+        super().__init__(empirical_time_series.name, empirical_time_series.sig)
+        self.example_feature = self.get_feature()
+
+    @staticmethod
+    def get_feature(self):
+        return -1

tests/performance_tests_acf.py

@@ -0,0 +1,46 @@
+import timeit
+
+import matplotlib.pyplot as plt
+import numpy as np
+from mpl_toolkits.axes_grid1.inset_locator import inset_axes
+from statsmodels.tsa.stattools import acf
+
+use_fft = [False, True]
+signal_lengths = np.hstack((np.arange(15, 515, 15), np.arange(500, 5500, 500)))
+results = [[], []]
+
+for i, use_fft in enumerate([False, True]):
+    for length in signal_lengths:
+        signal = np.random.randn(length)
+
+        def measure_acf(signal, use_fft):
+            return acf(signal, adjusted=True, fft=use_fft, nlags=int(len(signal) / 4))[1:]
+
+        cpu_time = timeit.timeit("measure_acf()", globals=globals(), number=10)
+        mean_cpu_time = cpu_time / 10  # Average time per execution
+
+        results[i].append(mean_cpu_time * 1000)
+results = np.array(results)
+
+fig, ax = plt.subplots(1, 1)
+ax.plot(signal_lengths, results[0], "o-", label="full convolution")
+ax.plot(signal_lengths, results[1], "o-", label="fft")
+ax.set_ylabel("CPU time / ms")
+ax.set_xlabel("Signal size / #")
+ax.spines[["top", "right"]].set_visible(False)
+ax.legend()
+
+ax_inset = inset_axes(ax, width="30%", height="30%", loc=1)  # loc=2 is for upper left
+ax_inset.plot(signal_lengths, results[0], "o-", label="full convolution")
+ax_inset.plot(signal_lengths, results[1], "o-", label="FFT")
+ax_inset.set_xlim(0, 500)
+ax_inset.set_ylim(0, 0.10)
+ax_inset.yaxis.set_visible(False)
+ax_inset.yaxis.set_ticks([])
+ax_inset.axvline(
+    x=signal_lengths[np.where(results[0] > results[1])[0][0]],
+    color="r",
+    linestyle="--",
+)
+
+plt.show(block=False)

tests/test_calc_features.py

@@ -0,0 +1,151 @@
+import glob
+import json
+import os
+from pathlib import Path
+
+# import tsfel
+import pandas as pd
+
+from tsfel.feature_extraction.calc_features import dataset_features_extractor, time_series_features_extractor
+from tsfel.feature_extraction.features_settings import get_features_by_domain, get_features_by_tag
+from tsfel.utils.add_personal_features import add_feature_json
+from tsfel.utils.signal_processing import merge_time_series, signal_window_splitter
+
+
+# Example of user preprocess sensor data
+def pre_process(sensor_data):
+    if "Accelerometer" in sensor_data:
+        sensor_data["Accelerometer"].iloc[:, 1] = sensor_data["Accelerometer"].iloc[:, 1] * 10
+    return sensor_data
+
+
+# DATASET DIR
+main_directory = os.path.join("tests", "tests_tools", "test_dataset", "")
+
+# JSON DIR
+# tsfel_path_json = tsfel.__path__[0] + os.sep + "feature_extraction" + os.sep + "features.json"
+personal_path_json = os.path.join("tests", "tests_tools", "test_features.json")
+personal_features_path = os.path.join(
+    "tests",
+    "tests_tools",
+    "test_personal_features.py",
+)
+
+# DEFAULT PARAM for testing
+time_unit = 1e9  # seconds
+resample_rate = 30  # resample sampling frequency
+window_size = 100  # number of points
+overlap = 0  # varies between 0 and 1
+search_criteria = ["Accelerometer.txt", "Gyroscope.txt"]
+output_directory = str(Path.home()) + os.sep + "Documents" + os.sep + "tsfel_output" + os.sep
+sensor_data = {}
+key = "Accelerometer"
+
+folders = list(glob.glob(main_directory + "**/", recursive=True))
+sensor_data[key] = pd.read_csv(folders[-1] + key + ".txt", header=None)
+
+
+# Features Dictionary
+# settings0 = json.load(open(tsfel_path_json))
+settings2 = get_features_by_domain("statistical")
+settings3 = get_features_by_domain("temporal")
+settings4 = get_features_by_domain("spectral")
+settings5 = get_features_by_domain("fractal")
+settings6 = get_features_by_domain()
+settings8 = get_features_by_tag("inertial")
+settings10 = get_features_by_tag()
+
+# Signal processing
+data_new = merge_time_series(sensor_data, resample_rate, time_unit)
+data_new = data_new[data_new.columns[:-1]]
+windows = signal_window_splitter(data_new, window_size, overlap)
+
+n_jobs = 1
+# time_series_features_extractor
+
+# multi windows and multi axis
+# input: list
+features0 = time_series_features_extractor(
+    settings6,
+    windows,
+    fs=resample_rate,
+    n_jobs=n_jobs,
+)
+
+# multiple windows and single axis
+# input: np.array
+features1 = time_series_features_extractor(
+    settings6,
+    data_new.values[:, 0],
+    fs=resample_rate,
+    n_jobs=n_jobs,
+    window_size=window_size,
+    overlap=overlap,
+)
+# input: pd.series
+features2 = time_series_features_extractor(
+    settings6,
+    data_new.iloc[:, 0],
+    fs=resample_rate,
+    n_jobs=n_jobs,
+    window_size=window_size,
+    overlap=overlap,
+)
+
+# single window and multi axis
+# input: pd.DataFrame
+features3 = time_series_features_extractor(
+    settings6,
+    data_new,
+    fs=resample_rate,
+    n_jobs=n_jobs,
+)
+# input: np.array
+features4 = time_series_features_extractor(
+    settings4,
+    data_new.values,
+    fs=resample_rate,
+    n_jobs=n_jobs,
+)
+
+# single window and single axis
+# input: pd.Series
+features5 = time_series_features_extractor(
+    settings2,
+    data_new.iloc[:, 0],
+    fs=resample_rate,
+    n_jobs=n_jobs,
+)
+# input: np.array
+features6 = time_series_features_extractor(
+    settings5,
+    data_new.values[:, 0],
+    fs=resample_rate,
+    n_jobs=n_jobs,
+)
+
+# personal features
+settings1 = json.load(open(personal_path_json))
+add_feature_json(personal_features_path, personal_path_json)
+features7 = time_series_features_extractor(
+    settings1,
+    data_new.values[:, 0],
+    fs=resample_rate,
+    n_jobs=n_jobs,
+    features_path=personal_features_path,
+)
+
+
+# Dataset features extractor
+data = dataset_features_extractor(
+    main_directory,
+    settings4,
+    search_criteria=search_criteria,
+    time_unit=time_unit,
+    resample_rate=resample_rate,
+    window_size=window_size,
+    overlap=overlap,
+    pre_process=pre_process,
+    output_directory=output_directory,
+)
+print("-----------------------------------OK-----------------------------------")