py-neuromodulation 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

py_neuromodulation/{nm_rereference.py → processing/rereference.py}

@@ -1,95 +1,97 @@
-"""Re-referencing Module."""
-import numpy as np
-import pandas as pd
-
-
-class ReReferencer:
-    ref_matrix: np.ndarray
-
-    def __init__(
-        self,
-        sfreq: int | float,
-        nm_channels: pd.DataFrame,
-    ) -> None:
-        """Initialize real-time rereference information.
-
-        Parameters
-        ----------
-        sfreq : int | float
-            Sampling frequency. Is not used, only kept for compatibility.
-        nm_channels : Pandas DataFrame
-            Dataframe containing information about rereferencing, as
-            specified in nm_channels.csv.
-
-
-        Raises:
-            ValueError: rereferencing using undefined channel
-            ValueError: rereferencing to same channel
-        """
-        nm_channels = nm_channels[nm_channels["used"] == 1].reset_index(
-            drop=True
-        )
-        # (channels_used,) = np.where((nm_channels.used == 1))
-
-        ch_names = nm_channels["name"].tolist()
-
-        # no re-referencing is being performed when there is a single channel present only
-        if nm_channels.shape[0] in (0, 1):
-            self.ref_matrix = None
-            return
-
-        ch_types = nm_channels["type"]
-        refs = nm_channels["rereference"]
-
-        type_map = {}
-        for ch_type in ch_types.unique():
-            type_map[ch_type] = np.where(
-                (ch_types == ch_type) & (nm_channels["status"] == "good")
-            )[0]
-
-        ref_matrix = np.zeros((len(nm_channels), len(nm_channels)))
-        for ind in range(len(nm_channels)):
-            ref_matrix[ind, ind] = 1
-            # if ind not in channels_used:
-            #    continue
-            ref = refs[ind]
-            if ref.lower() == "none" or pd.isnull(ref):
-                ref_idx = None
-                continue
-            if ref.lower() == "average":
-                ch_type = ch_types[ind]
-                ref_idx = type_map[ch_type][type_map[ch_type] != ind]
-            else:
-                ref_idx = []
-                ref_channels = ref.split("&")
-                for ref_chan in ref_channels:
-                    if ref_chan not in ch_names:
-                        raise ValueError(
-                            "One or more of the reference channels are not"
-                            " part of the recording channels. First missing"
-                            f" channel: {ref_chan}."
-                        )
-                    if ref_chan == ch_names[ind]:
-                        raise ValueError(
-                            "You cannot rereference to the same channel."
-                            f" Channel: {ref_chan}."
-                        )
-                    ref_idx.append(ch_names.index(ref_chan))
-            ref_matrix[ind, ref_idx] = -1 / len(ref_idx)
-        self.ref_matrix = ref_matrix
-
-    def process(self, data: np.ndarray) -> np.ndarray:
-        """Rereference data according to the initialized ReReferencer class.
-
-        Args:
-            data (numpy ndarray) :
-                shape(n_channels, n_samples) - data to be rereferenced.
-
-        Returns:
-            reref_data (numpy ndarray):
-            shape(n_channels, n_samples) - rereferenced data
-        """
-        if self.ref_matrix is not None:
-            return self.ref_matrix @ data
-        else:
-            return data
+"""Re-referencing Module."""
+
+import numpy as np
+import pandas as pd
+
+from py_neuromodulation.utils.types import NMPreprocessor
+
+
+class ReReferencer(NMPreprocessor):
+    def __init__(
+        self,
+        sfreq: float,
+        channels: pd.DataFrame,
+    ) -> None:
+        """Initialize real-time rereference information.
+
+        Parameters
+        ----------
+        sfreq : float
+            Sampling frequency. Is not used, only kept for compatibility.
+        channels : Pandas DataFrame
+            Dataframe containing information about rereferencing, as
+            specified in channels.csv.
+
+
+        Raises:
+            ValueError: rereferencing using undefined channel
+            ValueError: rereferencing to same channel
+        """
+
+        self.ref_matrix: np.ndarray | None
+
+        channels = channels[channels["used"] == 1].reset_index(drop=True)
+        # (channels_used,) = np.where((channels.used == 1))
+
+        ch_names = channels["name"].tolist()
+
+        # no re-referencing is being performed when there is a single channel present only
+        if channels.shape[0] in (0, 1):
+            self.ref_matrix = None
+            return
+
+        ch_types = channels["type"]
+        refs = channels["rereference"]
+
+        type_map = {}
+        for ch_type in ch_types.unique():
+            type_map[ch_type] = np.where(
+                (ch_types == ch_type) & (channels["status"] == "good")
+            )[0]
+
+        ref_matrix = np.zeros((len(channels), len(channels)))
+        for ind in range(len(channels)):
+            ref_matrix[ind, ind] = 1
+            # if ind not in channels_used:
+            #    continue
+            ref = refs[ind]
+            if ref.lower() == "none" or pd.isnull(ref):
+                ref_idx = None
+                continue
+            if ref.lower() == "average":
+                ch_type = ch_types[ind]
+                ref_idx = type_map[ch_type][type_map[ch_type] != ind]
+            else:
+                ref_idx = []
+                ref_channels = ref.split("&")
+                for ref_chan in ref_channels:
+                    if ref_chan not in ch_names:
+                        raise ValueError(
+                            "One or more of the reference channels are not"
+                            " part of the recording channels. First missing"
+                            f" channel: {ref_chan}."
+                        )
+                    if ref_chan == ch_names[ind]:
+                        raise ValueError(
+                            "You cannot rereference to the same channel."
+                            f" Channel: {ref_chan}."
+                        )
+                    ref_idx.append(ch_names.index(ref_chan))
+            ref_matrix[ind, ref_idx] = -1 / len(ref_idx)
+        self.ref_matrix = ref_matrix
+
+    def process(self, data: np.ndarray) -> np.ndarray:
+        """Rereference data according to the initialized ReReferencer class.
+
+        Args:
+            data (numpy ndarray) :
+                shape(n_channels, n_samples) - data to be rereferenced.
+
+        Returns:
+            reref_data (numpy ndarray):
+                shape(n_channels, n_samples) - rereferenced data
+        """
+        if self.ref_matrix is not None:
+            return self.ref_matrix @ data
+        else:
+            return data

py_neuromodulation/{nm_resample.py → processing/resample.py}

@@ -1,50 +1,56 @@
-"""Module for resampling."""
-import mne
-import numpy as np
-
-
-class Resampler:
-    """Resample data.
-
-    Parameters
-    ----------
-    sfreq : int | float
-        Original sampling frequency.
-
-    Attributes
-    ----------
-    up: float
-        Factor to upsample by.
-    """
-
-    def __init__(
-        self,
-        sfreq: int | float,
-        resample_freq_hz: int | float,
-    ) -> None:
-
-        assert isinstance(resample_freq_hz, (float, int))
-
-        self.sfreq_new: int | float = resample_freq_hz
-        ratio = float(self.sfreq_new / sfreq)
-        if ratio == 1.0:
-            self.up = 0.0
-        else:
-            self.up = ratio
-
-    def process(self, data: np.ndarray) -> np.ndarray:
-        """Resample raw data using mne.filter.resample.
-
-        Parameters
-        ----------
-        data : np.ndarray
-            Data to resample
-
-        Returns
-        -------
-        np.ndarray
-            Resampled data
-        """
-        if not self.up:
-            return data
-        return mne.filter.resample(data.astype(np.float64), up=self.up, down=1.0)
+"""Module for resampling."""
+
+import numpy as np
+from py_neuromodulation.utils.types import NMBaseModel, Field, NMPreprocessor
+
+
+class ResamplerSettings(NMBaseModel):
+    resample_freq_hz: float = Field(default=1000, gt=0)
+
+
+class Resampler(NMPreprocessor):
+    """Resample data.
+
+    Parameters
+    ----------
+    sfreq : float
+        Original sampling frequency.
+
+    Attributes
+    ----------
+    up: float
+        Factor to upsample by.
+    """
+
+    def __init__(
+        self,
+        sfreq: float,
+        resample_freq_hz: float,
+    ) -> None:
+        self.settings = ResamplerSettings(resample_freq_hz=resample_freq_hz)
+
+        ratio = float(resample_freq_hz / sfreq)
+        if ratio == 1.0:
+            self.up = 0.0
+        else:
+            self.up = ratio
+
+    def process(self, data: np.ndarray) -> np.ndarray:
+        """Resample raw data using mne.filter.resample.
+
+        Parameters
+        ----------
+        data : np.ndarray
+            Data to resample
+
+        Returns
+        -------
+        np.ndarray
+            Resampled data
+        """
+        if not self.up:
+            return data
+
+        from mne.filter import resample
+
+        return resample(data.astype(np.float64), up=self.up, down=1.0)

py_neuromodulation/stream/__init__.py

@@ -0,0 +1,3 @@
+from .generator import RawDataGenerator
+from .mnelsl_player import LSLOfflinePlayer
+from .mnelsl_stream import LSLStream

py_neuromodulation/stream/data_processor.py

@@ -0,0 +1,325 @@
+"""This module contains the class to process a given batch of data."""
+
+from time import time
+from typing import TYPE_CHECKING
+import numpy as np
+
+from py_neuromodulation import logger
+from py_neuromodulation.utils.types import _PathLike
+from py_neuromodulation.features import FeatureProcessors
+from py_neuromodulation.utils import io
+from py_neuromodulation.stream.settings import NMSettings
+
+if TYPE_CHECKING:
+    from py_neuromodulation.processing.projection import Projection
+    import pandas as pd
+
+
+class DataProcessor:
+    def __init__(
+        self,
+        sfreq: float,
+        settings: NMSettings | _PathLike,
+        channels: "pd.DataFrame | _PathLike",
+        coord_names: list | None = None,
+        coord_list: list | None = None,
+        line_noise: float | None = None,
+        path_grids: _PathLike | None = None,
+        verbose: bool = True,
+    ) -> None:
+        from py_neuromodulation.processing import DataPreprocessor
+
+        """Initialize run class.
+
+        Parameters
+        ----------
+        settings : settings.NMSettings object
+        channels : pd.DataFrame | _PathLike
+            Initialized pd.DataFrame with channel specific information.
+            The path to a channels.csv can be also passed.
+        coord_names : list | None
+            list of coordinate names
+        coord_list : list | None
+            list of list of 3D coordinates
+        path_grids : _PathLike | None
+            List to grid_cortex.tsv and grid_subcortex.tsv for grid point projection
+        verbose : boolean
+            if True, log signal processed and computation time
+        """
+
+        self.settings = NMSettings.load(settings)
+        self.channels = io.load_channels(channels)
+
+        self.sfreq_features: float = self.settings.sampling_rate_features_hz
+        self._sfreq_raw_orig: float = sfreq
+        self.sfreq_raw: float = sfreq // 1
+        self.line_noise: float | None = line_noise
+        self.path_grids: _PathLike | None = path_grids
+        self.verbose: bool = verbose
+
+        self.features_previous = None
+
+        (self.ch_names_used, _, self.feature_idx, _) = self._get_ch_info()
+
+        self.preprocessors = DataPreprocessor(
+            settings=self.settings,
+            channels=self.channels,
+            sfreq=self.sfreq_raw,
+            line_noise=self.line_noise,
+        )
+
+        if self.settings.postprocessing.feature_normalization:
+            from py_neuromodulation.processing.normalization import FeatureNormalizer
+
+            self.feature_normalizer = FeatureNormalizer(self.settings)
+
+        self.features = FeatureProcessors(
+            settings=self.settings,
+            ch_names=self.ch_names_used,
+            sfreq=self.sfreq_raw,
+        )
+
+        if coord_list is not None and coord_names is not None:
+            self.coords = self._set_coords(
+                coord_names=coord_names, coord_list=coord_list
+            )
+
+        self.projection = self._get_projection(self.settings, self.channels)
+
+        self.cnt_samples = 0
+
+    @staticmethod
+    def _add_coordinates(coord_names: list[str], coord_list: list) -> dict:
+        """Write cortical and subcortical coordinate information in joint dictionary
+
+        Parameters
+        ----------
+        coord_names : list[str]
+            list of coordinate names
+        coord_list : list
+            list of list of 3D coordinates
+
+        Returns
+        -------
+        dict with (sub)cortex_left and (sub)cortex_right ch_names and positions
+        """
+
+        def is_left_coord(val: float, coord_region: str) -> bool:
+            if coord_region.split("_")[1] == "left":
+                return val < 0
+            return val > 0
+
+        coords: dict[str, dict[str, list | np.ndarray]] = {}
+
+        for coord_region in [
+            coord_loc + "_" + lat
+            for coord_loc in ["cortex", "subcortex"]
+            for lat in ["left", "right"]
+        ]:
+            coords[coord_region] = {}
+
+            ch_type = "ECOG" if "cortex" == coord_region.split("_")[0] else "LFP"
+
+            coords[coord_region]["ch_names"] = [
+                coord_name
+                for coord_name, ch in zip(coord_names, coord_list)
+                if is_left_coord(ch[0], coord_region) and (ch_type in coord_name)
+            ]
+
+            # multiply by 1000 to get m instead of mm
+            positions = []
+            for coord, coord_name in zip(coord_list, coord_names):
+                if is_left_coord(coord[0], coord_region) and (ch_type in coord_name):
+                    positions.append(coord)
+            coords[coord_region]["positions"] = (
+                np.array(positions, dtype=np.float64) * 1000
+            )
+
+        return coords
+
+    def _get_ch_info(
+        self,
+    ) -> tuple[list[str], list[str], list[int], np.ndarray]:
+        """Get used feature and label info from channels"""
+        channels = self.channels
+        ch_names_used = channels[channels["used"] == 1]["new_name"].tolist()
+        ch_types_used = channels[channels["used"] == 1]["type"].tolist()
+
+        # used channels for feature estimation
+        feature_idx = np.where(channels["used"] & ~channels["target"])[0].tolist()
+
+        # If multiple targets exist, select only the first
+        label_idx = np.where(channels["target"] == 1)[0]
+
+        return ch_names_used, ch_types_used, feature_idx, label_idx
+
+    @staticmethod
+    def _get_grids(
+        settings: "NMSettings",
+        path_grids: _PathLike | None,
+    ) -> "tuple[pd.DataFrame | None, pd.DataFrame | None]":
+        """Read settings specified grids
+
+        Parameters
+        ----------
+        settings : settings.NMSettings object
+        path_grids : _PathLike | str
+
+        Returns
+        -------
+        Tuple
+            grid_cortex, grid_subcortex,
+            might be None if not specified in settings
+        """
+        if settings.postprocessing.project_cortex:
+            grid_cortex = io.read_grid(path_grids, "cortex")
+        else:
+            grid_cortex = None
+        if settings.postprocessing.project_subcortex:
+            grid_subcortex = io.read_grid(path_grids, "subcortex")
+        else:
+            grid_subcortex = None
+        return grid_cortex, grid_subcortex
+
+    def _get_projection(
+        self, settings: "NMSettings", channels: "pd.DataFrame"
+    ) -> "Projection | None":
+        from py_neuromodulation.processing.projection import Projection
+
+        """Return projection of used coordinated and grids"""
+
+        if not any(
+            (
+                settings.postprocessing.project_cortex,
+                settings.postprocessing.project_subcortex,
+            )
+        ):
+            return None
+
+        grid_cortex, grid_subcortex = self._get_grids(self.settings, self.path_grids)
+        projection = Projection(
+            settings=settings,
+            grid_cortex=grid_cortex,
+            grid_subcortex=grid_subcortex,
+            coords=self.coords,
+            channels=channels,
+            plot_projection=False,
+        )
+        return projection
+
+    def _set_coords(
+        self, coord_names: list[str] | None, coord_list: list | None
+    ) -> dict:
+        if not any(
+            (
+                self.settings.postprocessing.project_cortex,
+                self.settings.postprocessing.project_subcortex,
+            )
+        ):
+            return {}
+
+        if any((coord_list is None, coord_names is None)):
+            raise ValueError(
+                "No coordinates could be loaded. Please provide coord_list and"
+                f" coord_names. Got: {coord_list=}, {coord_names=}."
+            )
+
+        return self._add_coordinates(
+            coord_names=coord_names,
+            coord_list=coord_list,  # type: ignore # None case handled above
+        )
+
+    def process(self, data: np.ndarray) -> dict[str, float]:
+        """Given a new data batch, calculate and return features.
+
+        Parameters
+        ----------
+        data : np.ndarray
+            Current batch of raw data
+
+        Returns
+        -------
+        pandas Series
+            Features calculated from current data
+        """
+        start_time = time()
+
+        nan_channels = np.isnan(data).any(axis=1)
+
+        data = np.nan_to_num(data)[self.feature_idx, :]
+
+        data = self.preprocessors.process_data(data)
+
+        # calculate features
+        features_dict = self.features.estimate_features(data)
+
+        # normalize features
+        if self.settings.postprocessing.feature_normalization:
+            normed_features = self.feature_normalizer.process(
+                np.fromiter(features_dict.values(), dtype=np.float64)
+            )
+            features_dict = {
+                key: normed_features[idx]
+                for idx, key in enumerate(features_dict.keys())
+            }
+
+        # project features to grid
+        if self.projection:
+            self.projection.project_features(features_dict)
+
+        # check for all features, where the channel had a NaN, that the feature is also put to NaN
+        if nan_channels.sum() > 0:
+            # TONI: no need to do this if we store both old and new names for the channels
+            new_nan_channels = []
+            for ch in list(np.array(self.ch_names_used)[nan_channels]):
+                for key in features_dict.keys():
+                    if ch in key:
+                        new_nan_channels.append(key)
+
+            for ch in new_nan_channels:
+                features_dict[ch] = np.nan
+
+        if self.verbose:
+            logger.info("Last batch took: %.3f seconds to process", time() - start_time)
+
+        return features_dict
+
+    def save_sidecar(
+        self,
+        out_dir: _PathLike,
+        prefix: str = "",
+        additional_args: dict | None = None,
+    ) -> None:
+        """Save sidecar incuding fs, coords, sess_right to out_dir."""
+
+        sidecar: dict = {
+            "original_fs": self._sfreq_raw_orig,
+            "final_fs": self.sfreq_raw,
+            "sfreq": self.sfreq_features,
+        }
+        if self.projection:
+            sidecar["coords"] = self.projection.coords
+            if self.settings.postprocessing.project_cortex:
+                sidecar["grid_cortex"] = self.projection.grid_cortex
+                sidecar["proj_matrix_cortex"] = self.projection.proj_matrix_cortex
+            if self.settings.postprocessing.project_subcortex:
+                sidecar["grid_subcortex"] = self.projection.grid_subcortex
+                sidecar["proj_matrix_subcortex"] = self.projection.proj_matrix_subcortex
+        if additional_args is not None:
+            sidecar = sidecar | additional_args
+
+        io.save_sidecar(sidecar, out_dir, prefix)
+
+    def save_settings(self, out_dir: _PathLike, prefix: str = "") -> None:
+        self.settings.save(out_dir, prefix)
+
+    def save_channels(self, out_dir: _PathLike, prefix: str) -> None:
+        io.save_channels(self.channels, out_dir, prefix)
+
+    def save_features(
+        self,
+        feature_arr: "pd.DataFrame",
+        out_dir: _PathLike = "",
+        prefix: str = "",
+    ) -> None:
+        io.save_features(feature_arr, out_dir, prefix)

py_neuromodulation/stream/generator.py

@@ -0,0 +1,53 @@
+import numpy as np
+
+
+class RawDataGenerator:
+    """
+    This generator function mimics online data acquisition.
+    The data are iteratively sampled with settings.sampling_rate_features_hz
+    """
+
+    def __init__(
+        self,
+        data: np.ndarray,
+        sfreq: float,
+        sampling_rate_features_hz: float,
+        segment_length_features_ms: float,
+    ) -> None:
+        """
+        Arguments
+        ---------
+            data (np array): shape (channels, time)
+            settings (settings.NMSettings): settings object
+            sfreq (float): sampling frequency of the data
+
+        Returns
+        -------
+            np.array: 1D array of time stamps
+            np.array: new batch for run function of full segment length shape
+        """
+        self.batch_counter: int = 0  # counter for the batches
+
+        self.data = data
+        self.sfreq = sfreq
+        # Width, in data points, of the moving window used to calculate features
+        self.segment_length = segment_length_features_ms / 1000 * sfreq
+        # Ratio of the sampling frequency of the input data to the sampling frequency
+        self.stride = sfreq / sampling_rate_features_hz
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        start = self.stride * self.batch_counter
+        end = start + self.segment_length
+
+        self.batch_counter += 1
+
+        start_idx = int(start)
+        end_idx = int(end)
+
+        if end_idx > self.data.shape[1]:
+            raise StopIteration
+
+        return np.arange(start, end) / self.sfreq, self.data[:, start_idx:end_idx]