py-neuromodulation 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

py_neuromodulation/{nm_stream.py → stream/stream.py}

@@ -1,28 +1,146 @@
-"""Module for offline data streams."""
+"""Module for generic and offline data streams."""
 
 from typing import TYPE_CHECKING
+from collections.abc import Iterator
 import numpy as np
-import pandas as pd
 from pathlib import Path
+
+import py_neuromodulation as nm
 from contextlib import suppress
 
-from py_neuromodulation.nm_stream_abc import NMStream
-from py_neuromodulation.nm_types import _PathLike
-from py_neuromodulation import logger
+from py_neuromodulation.stream.data_processor import DataProcessor
+from py_neuromodulation.utils.types import _PathLike, FeatureName
+from py_neuromodulation.stream.settings import NMSettings
 
 if TYPE_CHECKING:
-    from py_neuromodulation.nm_settings import NMSettings
+    import pandas as pd
 
 
-class _GenericStream(NMStream):
+class Stream:
     """_GenericStream base class.
     This class can be inherited for different types of offline streams
 
     Parameters
     ----------
-    nm_stream_abc : nm_stream_abc.NMStream
+    nm_stream_abc : stream_abc.NMStream
     """
 
+    def __init__(
+        self,
+        sfreq: float,
+        channels: "pd.DataFrame | _PathLike | None" = None,
+        data: "np.ndarray | pd.DataFrame | None" = None,
+        settings: NMSettings | _PathLike | None = None,
+        line_noise: float | None = 50,
+        sampling_rate_features_hz: float | None = None,
+        path_grids: _PathLike | None = None,
+        coord_names: list | None = None,
+        stream_name: str
+        | None = "example_stream",  # Timon: do we need those in the nmstream_abc?
+        is_stream_lsl: bool = False,
+        coord_list: list | None = None,
+        verbose: bool = True,
+    ) -> None:
+        """Stream initialization
+
+        Parameters
+        ----------
+        sfreq : float
+            sampling frequency of data in Hertz
+        channels : pd.DataFrame | _PathLike
+            parametrization of channels (see define_channels.py for initialization)
+        data : np.ndarray | pd.DataFrame | None, optional
+            data to be streamed with shape (n_channels, n_time), by default None
+        settings : NMSettings | _PathLike | None, optional
+            Initialized settings.NMSettings object, by default the py_neuromodulation/settings.yaml are read
+            and passed into a settings object
+        line_noise : float | None, optional
+            line noise, by default 50
+        sampling_rate_features_hz : float | None, optional
+            feature sampling rate, by default None
+        path_grids : _PathLike | None, optional
+            path to grid_cortex.tsv and/or gird_subcortex.tsv, by default Non
+        coord_names : list | None, optional
+            coordinate name in the form [coord_1_name, coord_2_name, etc], by default None
+        coord_list : list | None, optional
+            coordinates in the form [[coord_1_x, coord_1_y, coord_1_z], [coord_2_x, coord_2_y, coord_2_z],], by default None
+        verbose : bool, optional
+            print out stream computation time information, by default True
+        """
+        self.settings: NMSettings = NMSettings.load(settings)
+
+        if channels is None and data is not None:
+            channels = nm.utils.channels.get_default_channels_from_data(data)
+
+        if channels is not None:
+            self.channels = nm.io.load_channels(channels)
+
+        if self.channels.query("used == 1 and target == 0").shape[0] == 0:
+            raise ValueError(
+                "No channels selected for analysis that have column 'used' = 1 and 'target' = 0. Please check your channels"
+            )
+
+        if channels is None and data is None:
+            raise ValueError("Either `channels` or `data` must be passed to `Stream`.")
+
+        # If features that use frequency ranges are on, test them against nyquist frequency
+        use_freq_ranges: list[FeatureName] = [
+            "bandpass_filter",
+            "stft",
+            "fft",
+            "welch",
+            "bursts",
+            "coherence",
+            "nolds",
+            "bispectrum",
+        ]
+
+        need_nyquist_check = any(
+            (f in use_freq_ranges for f in self.settings.features.get_enabled())
+        )
+
+        if need_nyquist_check:
+            assert all(
+                fb.frequency_high_hz < sfreq / 2
+                for fb in self.settings.frequency_ranges_hz.values()
+            ), (
+                "If a feature that uses frequency ranges is selected, "
+                "the frequency band ranges need to be smaller than the nyquist frequency.\n"
+                f"Got sfreq = {sfreq} and fband ranges:\n {self.settings.frequency_ranges_hz}"
+            )
+
+        if sampling_rate_features_hz is not None:
+            self.settings.sampling_rate_features_hz = sampling_rate_features_hz
+
+        if path_grids is None:
+            path_grids = nm.PYNM_DIR
+
+        self.path_grids = path_grids
+        self.verbose = verbose
+        self.sfreq = sfreq
+        self.line_noise = line_noise
+        self.coord_names = coord_names
+        self.coord_list = coord_list
+        self.sess_right = None
+        self.projection = None
+        self.model = None
+
+        # TODO(toni): is it necessary to initialize the DataProcessor on stream init?
+        self.data_processor = DataProcessor(
+            sfreq=self.sfreq,
+            settings=self.settings,
+            channels=self.channels,
+            path_grids=self.path_grids,
+            coord_names=coord_names,
+            coord_list=coord_list,
+            line_noise=line_noise,
+            verbose=self.verbose,
+        )
+
+        self.data = data
+
+        self.target_idx_initialized: bool = False
+
     def _add_target(self, feature_dict: dict, data: np.ndarray) -> None:
         """Add target channels to feature series.
 
@@ -39,12 +157,10 @@ class _GenericStream(NMStream):
             feature dict with target channels added
         """
 
-        if self.nm_channels["target"].sum() > 0:
+        if self.channels["target"].sum() > 0:
             if not self.target_idx_initialized:
-                self.target_indexes = self.nm_channels[
-                    self.nm_channels["target"] == 1
-                ].index
-                self.target_names = self.nm_channels.loc[
+                self.target_indexes = self.channels[self.channels["target"] == 1].index
+                self.target_names = self.channels.loc[
                     self.target_indexes, "name"
                 ].to_list()
                 self.target_idx_initialized = True
@@ -52,16 +168,16 @@ class _GenericStream(NMStream):
             for target_idx, target_name in zip(self.target_indexes, self.target_names):
                 feature_dict[target_name] = data[target_idx, -1]
 
-    def _handle_data(self, data: np.ndarray | pd.DataFrame) -> np.ndarray:
-        names_expected = self.nm_channels["name"].to_list()
+    def _handle_data(self, data: "np.ndarray | pd.DataFrame") -> np.ndarray:
+        names_expected = self.channels["name"].to_list()
 
         if isinstance(data, np.ndarray):
             if not len(names_expected) == data.shape[0]:
                 raise ValueError(
                     "If data is passed as an array, the first dimension must"
-                    " match the number of channel names in `nm_channels`.\n"
+                    " match the number of channel names in `channels`.\n"
                     f" Number of data channels (data.shape[0]): {data.shape[0]}\n"
-                    f' Length of nm_channels["name"]: {len(names_expected)}.'
+                    f' Length of channels["name"]: {len(names_expected)}.'
                 )
             return data
 
@@ -72,51 +188,79 @@ class _GenericStream(NMStream):
         ):
             raise ValueError(
                 "If data is passed as a DataFrame, the"
-                "column names must match the channel names in `nm_channels`.\n"
+                "column names must match the channel names in `channels`.\n"
                 f"Input dataframe column names: {names_data}\n"
-                f'Expected (from nm_channels["name"]): : {names_expected}.'
+                f'Expected (from channels["name"]): : {names_expected}.'
             )
         return data.to_numpy().transpose()
 
-    def _run(
+    def run(
         self,
-        data: np.ndarray | pd.DataFrame | None = None,
-        out_path_root: _PathLike = "",
-        folder_name: str = "sub",
-        is_stream_lsl: bool = True,
-        stream_lsl_name: str = None,
+        data: "np.ndarray | pd.DataFrame | None" = None,
+        out_dir: _PathLike = "",
+        experiment_name: str = "sub",
+        is_stream_lsl: bool = False,
+        stream_lsl_name: str | None = None,
         plot_lsl: bool = False,
         save_csv: bool = False,
         save_interval: int = 10,
         return_df: bool = True,
-    ) -> pd.DataFrame:
-        from py_neuromodulation.nm_database import NMDatabase
+    ) -> "pd.DataFrame":
+        self.is_stream_lsl = is_stream_lsl
+        self.stream_lsl_name = stream_lsl_name
 
-        out_path_root = Path.cwd() if not out_path_root else Path(out_path_root)
+        # Validate input data
+        if data is not None:
+            data = self._handle_data(data)
+        elif self.data is not None:
+            data = self._handle_data(self.data)
+        elif self.data is None and data is None and self.is_stream_lsl is False:
+            raise ValueError("No data passed to run function.")
 
-        self.PATH_OUT = out_path_root
-        self.PATH_OUT_folder_name = folder_name
+        # Generate output dirs
+        self.out_dir_root = Path.cwd() if not out_dir else Path(out_dir)
+        self.out_dir = self.out_dir_root / experiment_name
+        # TONI: Need better default experiment name
+        self.experiment_name = experiment_name if experiment_name else "sub"
 
-        out_dir = out_path_root / folder_name
-        out_dir.mkdir(parents=True, exist_ok=True)
+        self.out_dir.mkdir(parents=True, exist_ok=True)
 
-        # TONI: Need better default experiment name
-        experiment_name = folder_name if folder_name else "sub"
+        # Open database connection
+        # TONI: we should give the user control over the save format
+        from py_neuromodulation.utils.database import NMDatabase
 
         db = NMDatabase(experiment_name, out_dir)  # Create output database
 
         self.batch_count: int = 0  # Keep track of the number of batches processed
 
+        # Reinitialize the data processor in case the nm_channels or nm_settings changed between runs of the same Stream
+        # TONI: then I think we can just not initialize the data processor in the init function
+        self.data_processor = DataProcessor(
+            sfreq=self.sfreq,
+            settings=self.settings,
+            channels=self.channels,
+            path_grids=self.path_grids,
+            coord_names=self.coord_names,
+            coord_list=self.coord_list,
+            line_noise=self.line_noise,
+            verbose=self.verbose,
+        )
+
+        nm.logger.log_to_file(out_dir)
+
+        # Initialize generator
+        self.generator: Iterator
         if not is_stream_lsl:
-            from py_neuromodulation.nm_generator import raw_data_generator
+            from py_neuromodulation.stream.generator import RawDataGenerator
 
-            generator = raw_data_generator(
-                data=data,
-                settings=self.settings,
-                sfreq=self.sfreq,
+            self.generator = RawDataGenerator(
+                data,
+                self.sfreq,
+                self.settings.sampling_rate_features_hz,
+                self.settings.segment_length_features_ms,
             )
         else:
-            from py_neuromodulation.nm_mnelsl_stream import LSLStream
+            from py_neuromodulation.stream.mnelsl_stream import LSLStream
 
             self.lsl_stream = LSLStream(
                 settings=self.settings, stream_name=stream_lsl_name
@@ -134,21 +278,13 @@ class _GenericStream(NMStream):
                     f"does not match the settings ({self.sfreq})."
                     "The sampling frequency read from the stream will be used"
                 )
-                logger.warning(error_msg)
+                nm.logger.warning(error_msg)
                 self.sfreq = self.lsl_stream.stream.sinfo.sfreq
 
-            generator = self.lsl_stream.get_next_batch()
+            self.generator = self.lsl_stream.get_next_batch()
 
         prev_batch_end = 0
-
-        while True:
-            next_item = next(generator, None)
-
-            if next_item is not None:
-                timestamps, data_batch = next_item
-            else:
-                break
-
+        for timestamps, data_batch in self.generator:
             if data_batch is None:
                 break
 
@@ -156,7 +292,7 @@ class _GenericStream(NMStream):
 
             this_batch_end = timestamps[-1]
             batch_length = this_batch_end - prev_batch_end
-            logger.debug(
+            nm.logger.debug(
                 f"{batch_length:.3f} seconds of new data processed",
             )
 
@@ -167,7 +303,7 @@ class _GenericStream(NMStream):
             prev_batch_end = this_batch_end
 
             if self.verbose:
-                logger.info("Time: %.2f", feature_dict["time"] / 1000)
+                nm.logger.info("Time: %.2f", feature_dict["time"] / 1000)
 
             self._add_target(feature_dict, data_batch)
 
@@ -185,15 +321,13 @@ class _GenericStream(NMStream):
         db.commit()  # Save last batches
 
         # If save_csv is False, still save the first row to get the column names
-        feature_df: pd.DataFrame = (
+        feature_df: "pd.DataFrame" = (
             db.fetch_all() if (save_csv or return_df) else db.head()
         )
 
         db.close()  # Close the database connection
 
-        self.save_after_stream(
-            out_dir=out_dir, prefix=experiment_name, feature_arr=feature_df
-        )
+        self._save_after_stream(feature_arr=feature_df)
 
         return feature_df  # TONI: Not sure if this makes sense anymore
 
@@ -240,9 +374,9 @@ class _GenericStream(NMStream):
         if sfreq is None:
             sfreq = self.sfreq
 
-        if self.nm_channels is not None:
-            ch_names = self.nm_channels["name"].to_list()
-            ch_types = self.nm_channels["type"].to_list()
+        if self.channels is not None:
+            ch_names = self.channels["name"].to_list()
+            ch_types = self.channels["type"].to_list()
         else:
             ch_names = [f"ch_{i}" for i in range(data.shape[0])]
             ch_types = ["ecog" for i in range(data.shape[0])]
@@ -261,138 +395,33 @@ class _GenericStream(NMStream):
         if plot_psd:
             raw.compute_psd().plot()
 
-
-class Stream(_GenericStream):
-    def __init__(
+    def _save_after_stream(
         self,
-        sfreq: float,
-        data: np.ndarray | pd.DataFrame | None = None,
-        nm_channels: pd.DataFrame | _PathLike | None = None,
-        settings: "NMSettings | _PathLike | None" = None,
-        sampling_rate_features_hz: float | None = None,
-        line_noise: float | None = 50,
-        path_grids: _PathLike | None = None,
-        coord_names: list | None = None,
-        coord_list: list | None = None,
-        verbose: bool = True,
+        feature_arr: "pd.DataFrame | None" = None,
     ) -> None:
-        """Stream initialization
-
-        Parameters
-        ----------
-        sfreq : float
-            sampling frequency of data in Hertz
-        data : np.ndarray | pd.DataFrame | None, optional
-            data to be streamed with shape (n_channels, n_time), by default None
-        nm_channels : pd.DataFrame | _PathLike
-            parametrization of channels (see nm_define_channels.py for initialization)
-        settings : NMSettings | _PathLike | None, optional
-            Initialized nm_settings.NMSettings object, by default the py_neuromodulation/nm_settings.yaml are read
-            and passed into a settings object
-        line_noise : float | None, optional
-            line noise, by default 50
-        sampling_rate_features_hz : float | None, optional
-            feature sampling rate, by default None
-        path_grids : _PathLike | None, optional
-            path to grid_cortex.tsv and/or gird_subcortex.tsv, by default Non
-        coord_names : list | None, optional
-            coordinate name in the form [coord_1_name, coord_2_name, etc], by default None
-        coord_list : list | None, optional
-            coordinates in the form [[coord_1_x, coord_1_y, coord_1_z], [coord_2_x, coord_2_y, coord_2_z],], by default None
-        verbose : bool, optional
-            log stream computation time information, by default True
-        """
-
-        if nm_channels is None and data is not None:
-            from py_neuromodulation.nm_define_nmchannels import (
-                get_default_channels_from_data,
-            )
-
-            nm_channels = get_default_channels_from_data(data)
-
-        if nm_channels is None and data is None:
-            raise ValueError(
-                "Either `nm_channels` or `data` must be passed to `Stream`."
-            )
-
-        super().__init__(
-            sfreq=sfreq,
-            nm_channels=nm_channels,
-            settings=settings,
-            line_noise=line_noise,
-            sampling_rate_features_hz=sampling_rate_features_hz,
-            path_grids=path_grids,
-            coord_names=coord_names,
-            coord_list=coord_list,
-            verbose=verbose,
-        )
-
-        self.data = data
-
-        self.target_idx_initialized: bool = False
-
-    def run(
+        """Save features, settings, nm_channels and sidecar after run"""
+        self._save_sidecar()
+        if feature_arr is not None:
+            self._save_features(feature_arr)
+        self._save_settings()
+        self._save_channels()
+
+    def _save_features(
         self,
-        data: np.ndarray | pd.DataFrame | None = None,
-        out_path_root: _PathLike = Path.cwd(),
-        folder_name: str = "sub",
-        stream_lsl: bool = False,
-        stream_lsl_name: str = None,
-        save_csv: bool = False,
-        plot_lsl: bool = False,
-        save_interval: float = 10,
-    ) -> pd.DataFrame:
-        """Call run function for offline stream.
-
-        Parameters
-        ----------
-        data : np.ndarray | pd.DataFrame
-            shape (n_channels, n_time)
-        out_path_root : _PathLike | None, optional
-            Full path to store estimated features, by default None
-            If None, data is simply returned and not saved
-        folder_name : str, optional
-             folder output name, commonly subject or run name, by default "sub"
-        stream_lsl : bool, optional
-            stream data from LSL, by default False
-        stream_lsl_name : str, optional
-            stream name, by default None
-        plot_lsl : bool, optional
-            plot data with mne_lsl stream_viewer
-        save_csv : bool, optional
-            save csv file, by default False
-        save_interval : int, optional
-            save interval in number of samples, by default 10
-
-        Returns
-        -------
-        pd.DataFrame
-            feature DataFrame
-        """
-
-        super().run()  # reinitialize the stream
+        feature_arr: "pd.DataFrame",
+    ) -> None:
+        nm.io.save_features(feature_arr, self.out_dir, self.experiment_name)
 
-        self.stream_lsl = stream_lsl
-        self.stream_lsl_name = stream_lsl_name
+    def _save_channels(self) -> None:
+        self.data_processor.save_channels(self.out_dir, self.experiment_name)
 
-        if data is not None:
-            data = self._handle_data(data)
-        elif self.data is not None:
-            data = self._handle_data(self.data)
-        elif self.data is None and data is None and self.stream_lsl is False:
-            raise ValueError("No data passed to run function.")
+    def _save_settings(self) -> None:
+        self.data_processor.save_settings(self.out_dir, self.experiment_name)
 
-        out_path = Path(out_path_root, folder_name)
-        out_path.mkdir(parents=True, exist_ok=True)
-        logger.log_to_file(out_path)
-
-        return self._run(
-            data,
-            out_path_root,
-            folder_name,
-            is_stream_lsl=stream_lsl,
-            stream_lsl_name=stream_lsl_name,
-            save_csv=save_csv,
-            plot_lsl=plot_lsl,
-            save_interval=save_interval,
+    def _save_sidecar(self) -> None:
+        """Save sidecar incduing fs, coords, sess_right to
+        out_path_root and subfolder 'folder_name'"""
+        additional_args = {"sess_right": self.sess_right}
+        self.data_processor.save_sidecar(
+            self.out_dir, self.experiment_name, additional_args
         )

py_neuromodulation/utils/__init__.py

@@ -0,0 +1,2 @@
+from .channels import *
+from . import types

py_neuromodulation/{nm_define_nmchannels.py → utils/channels.py}

@@ -1,9 +1,11 @@
-"""Module for handling nm_channels."""
+"""Module for handling channels."""
 
 from collections.abc import Iterable
-import pandas as pd
+from typing import TYPE_CHECKING
 import numpy as np
 
+if TYPE_CHECKING:
+    import pandas as pd
 
 _LFP_TYPES = ["seeg", "dbs", "lfp"]  # must be lower-case
 
@@ -17,10 +19,10 @@ def set_channels(
     ecog_only: bool = False,
     used_types: Iterable[str] | None = ("ecog", "dbs", "seeg"),
     target_keywords: Iterable[str] | None = ("mov", "squared", "label"),
-) -> pd.DataFrame:
-    """Return dataframe with channel-specific settings in nm_channels format.
+) -> "pd.DataFrame":
+    """Return dataframe with channel-specific settings in channels format.
 
-    Return an nm_channels dataframe with the columns: "name", "rereference",
+    Return an channels dataframe with the columns: "name", "rereference",
     "used", "target", "type", "status", "new_name"]. "name" is set to ch_names,
     "rereference" can be specified individually. "used" is set to 1 for all
     channel types specified in `used_types`, else to 0. "target" is set to 1
@@ -63,8 +65,10 @@ def set_channels(
 
     Returns
     -------
-        df: DataFrame in nm_channels format
+        df: DataFrame in channels format
     """
+    import pandas as pd
+
     if not (len(ch_names) == len(ch_types)):
         raise ValueError(
             "Number of `ch_names` and `ch_types` must match."
@@ -196,8 +200,8 @@ def set_channels(
 
 
 def _get_default_references(
-    df: pd.DataFrame, ch_names: list[str], ch_types: list[str]
-) -> pd.DataFrame:
+    df: "pd.DataFrame", ch_names: list[str], ch_types: list[str]
+) -> "pd.DataFrame":
     """Add references with default settings (ECOG CAR, LFP bipolar)."""
     ecog_chs = []
     lfp_chs = []
@@ -250,7 +254,7 @@ def get_default_channels_from_data(
     data: np.ndarray,
     car_rereferencing: bool = True,
 ):
-    """Return default nm_channels dataframe with
+    """Return default channels dataframe with
     ecog datatype, no bad channels, no targets, common average rereferencing
 
     Parameters
@@ -272,6 +276,7 @@ def get_default_channels_from_data(
          - status
          - new_name
     """
+    import pandas as pd
 
     ch_name = [f"ch{idx}" for idx in range(data.shape[0])]
     status = ["good" for _ in range(data.shape[0])]

py_neuromodulation/{nm_database.py → utils/database.py}

@@ -1,8 +1,8 @@
 import sqlite3
 from pathlib import Path
 import pandas as pd
-from py_neuromodulation.nm_types import _PathLike
-from py_neuromodulation.nm_IO import generate_unique_filename
+from py_neuromodulation.utils.types import _PathLike
+from py_neuromodulation.utils.io import generate_unique_filename
 
 
 class NMDatabase: