braindecode 0.8.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

braindecode/datasets/xy.py

@@ -3,19 +3,29 @@
 #
 # License: BSD (3-clause)
 
-import numpy as np
-import pandas as pd
+from __future__ import annotations
+
 import logging
+
 import mne
+import numpy as np
+import pandas as pd
+from numpy.typing import ArrayLike, NDArray
 
-from .base import BaseDataset, BaseConcatDataset
+from .base import BaseConcatDataset, BaseDataset
 
 log = logging.getLogger(__name__)
 
 
 def create_from_X_y(
-        X, y, drop_last_window, sfreq, ch_names=None, window_size_samples=None,
-        window_stride_samples=None):
+    X: NDArray,
+    y: ArrayLike,
+    drop_last_window: bool,
+    sfreq: float,
+    ch_names: ArrayLike = None,
+    window_size_samples: int | None = None,
+    window_stride_samples: int | None = None,
+) -> BaseConcatDataset:
     """Create a BaseConcatDataset of WindowsDatasets from X and y to be used for
     decoding with skorch and braindecode, where X is a list of pre-cut trials
     and y are corresponding targets.
@@ -46,7 +56,9 @@ def create_from_X_y(
     """
     # Prevent circular import
     from ..preprocessing.windowers import (
-        create_fixed_length_windows, )
+        create_fixed_length_windows,
+    )
+
     n_samples_per_x = []
     base_datasets = []
     if ch_names is None:
@@ -57,16 +69,19 @@ def create_from_X_y(
         n_samples_per_x.append(x.shape[1])
         info = mne.create_info(ch_names=ch_names, sfreq=sfreq)
         raw = mne.io.RawArray(x, info)
-        base_dataset = BaseDataset(raw, pd.Series({"target": target}),
-                                   target_name="target")
+        base_dataset = BaseDataset(
+            raw, pd.Series({"target": target}), target_name="target"
+        )
         base_datasets.append(base_dataset)
     base_datasets = BaseConcatDataset(base_datasets)
 
     if window_size_samples is None and window_stride_samples is None:
         if not len(np.unique(n_samples_per_x)) == 1:
-            raise ValueError("if 'window_size_samples' and "
-                             "'window_stride_samples' are None, "
-                             "all trials have to have the same length")
+            raise ValueError(
+                "if 'window_size_samples' and "
+                "'window_stride_samples' are None, "
+                "all trials have to have the same length"
+            )
         window_size_samples = n_samples_per_x[0]
         window_stride_samples = n_samples_per_x[0]
     windows_datasets = create_fixed_length_windows(
@@ -75,6 +90,6 @@ def create_from_X_y(
         stop_offset_samples=None,
         window_size_samples=window_size_samples,
         window_stride_samples=window_stride_samples,
-        drop_last_window=drop_last_window
+        drop_last_window=drop_last_window,
     )
     return windows_datasets

braindecode/datautil/__init__.py

@@ -2,32 +2,51 @@
 Utilities for data manipulation.
 """
 
-
 from .serialization import (
-    save_concat_dataset, load_concat_dataset, _check_save_dir_empty)
+    _check_save_dir_empty,
+    load_concat_dataset,
+    save_concat_dataset,
+)
 
 
 def __getattr__(name):
     # ideas from https://stackoverflow.com/a/57110249/1469195
-    from warnings import warn
     import importlib
-    if name == 'create_from_X_y':
-        warn('create_from_X_y has been moved to datasets, please use from braindecode.datasets import create_from_X_y')
-        xy = importlib.import_module('..datasets.xy', __package__)
+    from warnings import warn
+
+    if name == "create_from_X_y":
+        warn(
+            "create_from_X_y has been moved to datasets, please use from braindecode.datasets import create_from_X_y"
+        )
+        xy = importlib.import_module("..datasets.xy", __package__)
         return xy.create_from_X_y
-    if name in ['create_from_mne_raw', 'create_from_mne_epochs']:
-        warn(f'{name} has been moved to datasets, please use from braindecode.datasets import {name}')
-        mne = importlib.import_module('..datasets.mne', __package__)
+    if name in ["create_from_mne_raw", "create_from_mne_epochs"]:
+        warn(
+            f"{name} has been moved to datasets, please use from braindecode.datasets import {name}"
+        )
+        mne = importlib.import_module("..datasets.mne", __package__)
         return mne.__dict__[name]
-    if name in ['zscore', 'scale', 'exponential_moving_demean',
-                             'exponential_moving_standardize', 'filterbank',
-                             'preprocess', 'Preprocessor']:
-        warn(f'{name} has been moved to preprocessing, please use from braindecode.preprocessing import {name}')
-        preprocess = importlib.import_module('..preprocessing.preprocess', __package__)
+    if name in [
+        "scale",
+        "exponential_moving_demean",
+        "exponential_moving_standardize",
+        "filterbank",
+        "preprocess",
+        "Preprocessor",
+    ]:
+        warn(
+            f"{name} has been moved to preprocessing, please use from braindecode.preprocessing import {name}"
+        )
+        preprocess = importlib.import_module("..preprocessing.preprocess", __package__)
         return preprocess.__dict__[name]
-    if name in ['create_windows_from_events', 'create_fixed_length_windows']:
-        warn(f'{name} has been moved to preprocessing, please use from braindecode.preprocessing import {name}')
-        windowers = importlib.import_module('..preprocessing.windowers', __package__)
+    if name in ["create_windows_from_events", "create_fixed_length_windows"]:
+        warn(
+            f"{name} has been moved to preprocessing, please use from braindecode.preprocessing import {name}"
+        )
+        windowers = importlib.import_module("..preprocessing.windowers", __package__)
         return windowers.__dict__[name]
 
-    raise AttributeError('No possible import named ' + name)
+    raise AttributeError("No possible import named " + name)
+
+
+__all__ = ["load_concat_dataset", "save_concat_dataset", "_check_save_dir_empty"]

braindecode/datautil/serialization.py

@@ -6,8 +6,8 @@ Convenience functions for storing and loading of windows datasets.
 #
 # License: BSD (3-clause)
 
-import os
 import json
+import os
 import pickle
 import warnings
 from glob import glob
@@ -17,17 +17,24 @@ import mne
 import pandas as pd
 from joblib import Parallel, delayed
 
-from ..datasets.base import BaseDataset, BaseConcatDataset, WindowsDataset, EEGWindowsDataset
+from ..datasets.base import (
+    BaseConcatDataset,
+    BaseDataset,
+    EEGWindowsDataset,
+    WindowsDataset,
+)
 
 
 def save_concat_dataset(path, concat_dataset, overwrite=False):
-    warnings.warn('"save_concat_dataset()" is deprecated and will be removed in'
-                  ' the future. Use dataset.save() instead.')
+    warnings.warn(
+        '"save_concat_dataset()" is deprecated and will be removed in'
+        " the future. Use dataset.save() instead.",
+        UserWarning,
+    )
     concat_dataset.save(path=path, overwrite=overwrite)
 
 
-def _outdated_load_concat_dataset(path, preload, ids_to_load=None,
-                                  target_name=None):
+def _outdated_load_concat_dataset(path, preload, ids_to_load=None, target_name=None):
     """Load a stored BaseConcatDataset of BaseDatasets or WindowsDatasets from
     files.
 
@@ -48,15 +55,16 @@ def _outdated_load_concat_dataset(path, preload, ids_to_load=None,
     concat_dataset: BaseConcatDataset of BaseDatasets or WindowsDatasets
     """
     # assume we have a single concat dataset to load
-    is_raw = (path / '0-raw.fif').is_file()
+    is_raw = (path / "0-raw.fif").is_file()
     assert not (not is_raw and target_name is not None), (
-        'Setting a new target is only supported for raws.')
-    is_epochs = (path / '0-epo.fif').is_file()
+        "Setting a new target is only supported for raws."
+    )
+    is_epochs = (path / "0-epo.fif").is_file()
     paths = [path]
     # assume we have multiple concat datasets to load
     if not (is_raw or is_epochs):
-        is_raw = (path / '0' / '0-raw.fif').is_file()
-        is_epochs = (path / '0' / '0-epo.fif').is_file()
+        is_raw = (path / "0" / "0-raw.fif").is_file()
+        is_epochs = (path / "0" / "0-epo.fif").is_file()
         paths = path.glob("*/")
         paths = sorted(paths, key=lambda p: int(p.name))
         if ids_to_load is not None:
@@ -64,33 +72,32 @@ def _outdated_load_concat_dataset(path, preload, ids_to_load=None,
         ids_to_load = None
     # if we have neither a single nor multiple datasets, something went wrong
     assert is_raw or is_epochs, (
-        f'Expect either raw or epo to exist in {path} or in '
-        f'{path / "0"}')
+        f"Expect either raw or epo to exist in {path} or in {path / '0'}"
+    )
 
     datasets = []
     for path in paths:
         if is_raw and target_name is None:
-            target_file_name = path / 'target_name.json'
-            target_name = json.load(open(target_file_name, "r"))['target_name']
+            target_file_name = path / "target_name.json"
+            target_name = json.load(open(target_file_name, "r"))["target_name"]
 
         all_signals, description = _load_signals_and_description(
-            path=path, preload=preload, is_raw=is_raw,
-            ids_to_load=ids_to_load
+            path=path, preload=preload, is_raw=is_raw, ids_to_load=ids_to_load
         )
         for i_signal, signal in enumerate(all_signals):
             if is_raw:
                 datasets.append(
-                    BaseDataset(signal, description.iloc[i_signal],
-                                target_name=target_name))
-            else:
-                datasets.append(
-                    WindowsDataset(signal, description.iloc[i_signal])
+                    BaseDataset(
+                        signal, description.iloc[i_signal], target_name=target_name
+                    )
                 )
+            else:
+                datasets.append(WindowsDataset(signal, description.iloc[i_signal]))
     concat_ds = BaseConcatDataset(datasets)
-    for kwarg_name in ['raw_preproc_kwargs', 'window_kwargs', 'window_preproc_kwargs']:
-        kwarg_path = path / '.'.join([kwarg_name, 'json'])
+    for kwarg_name in ["raw_preproc_kwargs", "window_kwargs", "window_preproc_kwargs"]:
+        kwarg_path = path / ".".join([kwarg_name, "json"])
         if kwarg_path.exists():
-            with open(kwarg_path, 'r') as f:
+            with open(kwarg_path, "r") as f:
                 kwargs = json.load(f)
             kwargs = [tuple(kwarg) for kwarg in kwargs]
             setattr(concat_ds, kwarg_name, kwargs)
@@ -100,14 +107,22 @@ def _outdated_load_concat_dataset(path, preload, ids_to_load=None,
 def _load_signals_and_description(path, preload, is_raw, ids_to_load=None):
     all_signals = []
     file_name = "{}-raw.fif" if is_raw else "{}-epo.fif"
-    description_df = pd.read_json(path / "description.json")
+    description_df = pd.read_json(
+        path / "description.json", typ="series", convert_dates=False
+    )
+
+    if "timestamp" in description_df.index:
+        timestamp_numeric = pd.to_numeric(description_df["timestamp"])
+        description_df["timestamp"] = pd.to_datetime(timestamp_numeric)
+
     if ids_to_load is None:
         file_names = path.glob(f"*{file_name.lstrip('{}')}")
         # Extract ids, e.g.,
         # '/home/schirrmr/data/preproced-tuh/all-sensors/11-raw.fif' ->
         # '11-raw.fif' -> 11
         ids_to_load = sorted(
-            [int(os.path.split(f)[-1].split('-')[0]) for f in file_names])
+            [int(os.path.split(f)[-1].split("-")[0]) for f in file_names]
+        )
     for i in ids_to_load:
         fif_file = path / file_name.format(i)
         all_signals.append(_load_signals(fif_file, preload, is_raw))
@@ -133,10 +148,10 @@ def _load_signals(fif_file, preload, is_raw):
     # If pickle didn't exist read via mne (likely slower) and save pkl after
     if is_raw:
         signals = mne.io.read_raw_fif(fif_file, preload=preload)
-    elif fif_file.name.endswith('-epo.fif'):
+    elif fif_file.name.endswith("-epo.fif"):
         signals = mne.read_epochs(fif_file, preload=preload)
     else:
-        raise ValueError('fif_file must end with raw.fif or epo.fif.')
+        raise ValueError("fif_file must end with raw.fif or epo.fif.")
 
     # Only do this for raw objects. Epoch objects are not picklable as they
     # hold references to open files in `signals._raw[0].fid`.
@@ -154,8 +169,7 @@ def _load_signals(fif_file, preload, is_raw):
     return signals
 
 
-def load_concat_dataset(path, preload, ids_to_load=None, target_name=None,
-                        n_jobs=1):
+def load_concat_dataset(path, preload, ids_to_load=None, target_name=None, n_jobs=1):
     """Load a stored BaseConcatDataset of BaseDatasets or WindowsDatasets from
     files.
 
@@ -183,11 +197,14 @@ def load_concat_dataset(path, preload, ids_to_load=None, target_name=None,
     # if we encounter a dataset that was saved in 'the old way', call the
     # corresponding 'old' loading function
     if _is_outdated_saved(path):
-        warnings.warn("The way your dataset was saved is deprecated by now. "
-                      "Please save it again using dataset.save().", UserWarning)
+        warnings.warn(
+            "The way your dataset was saved is deprecated by now. "
+            "Please save it again using dataset.save().",
+            UserWarning,
+        )
         return _outdated_load_concat_dataset(
-            path=path, preload=preload, ids_to_load=ids_to_load,
-            target_name=target_name)
+            path=path, preload=preload, ids_to_load=ids_to_load, target_name=target_name
+        )
 
     # else we have a dataset saved in the new way with subdirectories in path
     # for every dataset with description.json and -epo.fif or -raw.fif,
@@ -197,9 +214,9 @@ def load_concat_dataset(path, preload, ids_to_load=None, target_name=None,
         ids_to_load = [p.name for p in path.iterdir()]
         ids_to_load = sorted(ids_to_load, key=lambda i: int(i))
     ids_to_load = [str(i) for i in ids_to_load]
-    first_raw_fif_path = path / ids_to_load[0] / f'{ids_to_load[0]}-raw.fif'
+    first_raw_fif_path = path / ids_to_load[0] / f"{ids_to_load[0]}-raw.fif"
     is_raw = first_raw_fif_path.exists()
-    metadata_path = path / ids_to_load[0] / 'metadata_df.pkl'
+    metadata_path = path / ids_to_load[0] / "metadata_df.pkl"
     has_stored_windows = metadata_path.exists()
 
     # Parallelization of mne.read_epochs with preload=False fails with
@@ -207,8 +224,10 @@ def load_concat_dataset(path, preload, ids_to_load=None, target_name=None,
     # So ignore n_jobs in that case and load with a single job.
     if not is_raw and n_jobs != 1:
         warnings.warn(
-            'Parallelized reading with `preload=False` is not supported for '
-            'windowed data. Will use `n_jobs=1`.', UserWarning)
+            "Parallelized reading with `preload=False` is not supported for "
+            "windowed data. Will use `n_jobs=1`.",
+            UserWarning,
+        )
         n_jobs = 1
     datasets = Parallel(n_jobs)(
         delayed(_load_parallel)(path, i, preload, is_raw, has_stored_windows)
@@ -219,9 +238,9 @@ def load_concat_dataset(path, preload, ids_to_load=None, target_name=None,
 
 def _load_parallel(path, i, preload, is_raw, has_stored_windows):
     sub_dir = path / i
-    file_name_patterns = ['{}-raw.fif', '{}-epo.fif']
+    file_name_patterns = ["{}-raw.fif", "{}-epo.fif"]
     if all([(sub_dir / p.format(i)).exists() for p in file_name_patterns]):
-        raise FileExistsError('Found -raw.fif and -epo.fif in directory.')
+        raise FileExistsError("Found -raw.fif and -epo.fif in directory.")
 
     fif_name_pattern = file_name_patterns[0] if is_raw else file_name_patterns[1]
     fif_file_name = fif_name_pattern.format(i)
@@ -229,48 +248,55 @@ def _load_parallel(path, i, preload, is_raw, has_stored_windows):
 
     signals = _load_signals(fif_file_path, preload, is_raw)
 
-    description_file_path = sub_dir / 'description.json'
-    description = pd.read_json(description_file_path, typ='series')
+    description_file_path = sub_dir / "description.json"
+    description = pd.read_json(description_file_path, typ="series", convert_dates=False)
+
+    # if 'timestamp' in description.index:
+    #     timestamp_numeric = pd.to_numeric(description['timestamp'])
+    #     description['timestamp'] = pd.to_datetime(timestamp_numeric, unit='s')
 
-    target_file_path = sub_dir / 'target_name.json'
+    target_file_path = sub_dir / "target_name.json"
     target_name = None
     if target_file_path.exists():
-        target_name = json.load(open(target_file_path, "r"))['target_name']
+        target_name = json.load(open(target_file_path, "r"))["target_name"]
 
     if is_raw and (not has_stored_windows):
         dataset = BaseDataset(signals, description, target_name)
     else:
-        window_kwargs = _load_kwargs_json('window_kwargs', sub_dir)
-        windows_ds_kwargs = [kwargs[1] for kwargs in window_kwargs if kwargs[0] == 'WindowsDataset']
+        window_kwargs = _load_kwargs_json("window_kwargs", sub_dir)
+        windows_ds_kwargs = [
+            kwargs[1] for kwargs in window_kwargs if kwargs[0] == "WindowsDataset"
+        ]
         windows_ds_kwargs = windows_ds_kwargs[0] if len(windows_ds_kwargs) == 1 else {}
         if is_raw:
-            metadata = pd.read_pickle(path / i / 'metadata_df.pkl')
+            metadata = pd.read_pickle(path / i / "metadata_df.pkl")
             dataset = EEGWindowsDataset(
                 signals,
                 metadata=metadata,
                 description=description,
-                targets_from=windows_ds_kwargs.get('targets_from', 'metadata'),
-                last_target_only=windows_ds_kwargs.get('last_target_only', True),
+                targets_from=windows_ds_kwargs.get("targets_from", "metadata"),
+                last_target_only=windows_ds_kwargs.get("last_target_only", True),
             )
         else:
             # MNE epochs dataset
             dataset = WindowsDataset(
-                signals, description,
-                targets_from=windows_ds_kwargs.get('targets_from', 'metadata'),
-                last_target_only=windows_ds_kwargs.get('last_target_only', True)
+                signals,
+                description,
+                targets_from=windows_ds_kwargs.get("targets_from", "metadata"),
+                last_target_only=windows_ds_kwargs.get("last_target_only", True),
             )
-        setattr(dataset, 'window_kwargs', window_kwargs)
-    for kwargs_name in ['raw_preproc_kwargs', 'window_preproc_kwargs']:
+        setattr(dataset, "window_kwargs", window_kwargs)
+    for kwargs_name in ["raw_preproc_kwargs", "window_preproc_kwargs"]:
         kwargs = _load_kwargs_json(kwargs_name, sub_dir)
         setattr(dataset, kwargs_name, kwargs)
     return dataset
 
 
 def _load_kwargs_json(kwargs_name, sub_dir):
-    kwargs_file_name = '.'.join([kwargs_name, 'json'])
+    kwargs_file_name = ".".join([kwargs_name, "json"])
     kwargs_file_path = os.path.join(sub_dir, kwargs_file_name)
     if os.path.exists(kwargs_file_path):
-        kwargs = json.load(open(kwargs_file_path, 'r'))
+        kwargs = json.load(open(kwargs_file_path, "r"))
         kwargs = [tuple(kwarg) for kwarg in kwargs]
         return kwargs
 
@@ -279,18 +305,28 @@ def _is_outdated_saved(path):
     """Data was saved in the old way if there are 'description.json', '-raw.fif'
     or '-epo.fif' in path (no subdirectories) OR if there are more 'fif' files
     than 'description.json' files."""
-    description_files = glob(os.path.join(path, '**/description.json'))
-    fif_files = glob(os.path.join(path, '**/*-raw.fif')) + glob(os.path.join(path, '**/*-epo.fif'))
+    description_files = glob(os.path.join(path, "**/description.json"))
+    fif_files = glob(os.path.join(path, "**/*-raw.fif")) + glob(
+        os.path.join(path, "**/*-epo.fif")
+    )
     multiple = len(description_files) != len(fif_files)
     kwargs_in_path = any(
-        [os.path.exists(os.path.join(path, kwarg_name))
-         for kwarg_name in ['raw_preproc_kwargs', 'window_kwargs',
-                            'window_preproc_kwargs']])
-    return (os.path.exists(os.path.join(path, 'description.json')) or
-            os.path.exists(os.path.join(path, '0-raw.fif')) or
-            os.path.exists(os.path.join(path, '0-epo.fif')) or
-            multiple or
-            kwargs_in_path)
+        [
+            os.path.exists(os.path.join(path, kwarg_name))
+            for kwarg_name in [
+                "raw_preproc_kwargs",
+                "window_kwargs",
+                "window_preproc_kwargs",
+            ]
+        ]
+    )
+    return (
+        os.path.exists(os.path.join(path, "description.json"))
+        or os.path.exists(os.path.join(path, "0-raw.fif"))
+        or os.path.exists(os.path.join(path, "0-epo.fif"))
+        or multiple
+        or kwargs_in_path
+    )
 
 
 def _check_save_dir_empty(save_dir):
@@ -306,10 +342,12 @@ def _check_save_dir_empty(save_dir):
     FileExistsError
         If ``save_dir`` is not a valid directory for saving.
     """
-    sub_dirs = [os.path.basename(s).isdigit()
-                for s in glob(os.path.join(save_dir, '*'))]
+    sub_dirs = [
+        os.path.basename(s).isdigit() for s in glob(os.path.join(save_dir, "*"))
+    ]
     if any(sub_dirs):
         raise FileExistsError(
-            f'Directory {save_dir} already contains subdirectories. Please '
-            'select a different directory, set overwrite=True, or resolve '
-            'manually.')
+            f"Directory {save_dir} already contains subdirectories. Please "
+            "select a different directory, set overwrite=True, or resolve "
+            "manually."
+        )