py-neuromodulation 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

py_neuromodulation/{helper.py → _write_example_dataset_helper.py}

@@ -1,6 +1,6 @@
 import mne
 import mne_bids
-import pybv
+import pybv  # pip install pybv
 import os
 
 

py_neuromodulation/nm_EpochStream.py

@@ -4,11 +4,11 @@ import pickle
 import numpy as np
 import pandas as pd
 
-import py_neuromodulation as py_nm
+import py_neuromodulation as nm
 from py_neuromodulation import nm_generator
 
 
-class EpochStream(py_nm.nm_stream.PNStream):
+class EpochStream(nm.nm_stream.PNStream):
     def __init__(self) -> None:
         super().__init__()
 
@@ -43,7 +43,6 @@ class EpochStream(py_nm.nm_stream.PNStream):
     def run(
         self,
     ):
-
         self._set_run()
         # shape is n, channels=7, 800 Hz
 

py_neuromodulation/nm_IO.py

@@ -2,6 +2,10 @@ import json
 import os
 import sys
 from pathlib import Path
+import logging
+
+logger = logging.getLogger("PynmLogger")
+
 
 import mne
 import mne_bids
@@ -9,6 +13,9 @@ import numpy as np
 import pandas as pd
 from scipy import io
 
+import pyarrow
+from pyarrow import csv
+
 import py_neuromodulation
 
 _PathLike = str | os.PathLike
@@ -27,7 +34,6 @@ def load_nm_channels(
     reference Union[list, str]
     """
 
-    
     if isinstance(nm_channels, pd.DataFrame):
         nm_ch_return = nm_channels
     elif nm_channels:
@@ -39,6 +45,7 @@ def load_nm_channels(
 
     return nm_ch_return
 
+
 def read_BIDS_data(
     PATH_RUN: _PathLike | mne_bids.BIDSPath,
     BIDS_PATH: _PathLike | None = None,
@@ -70,10 +77,8 @@ def read_BIDS_data(
     if raw_arr.info["line_freq"] is not None:
         line_noise = int(raw_arr.info["line_freq"])
     else:
-        print(
-            "Line noise is not available in the data, using value of {} Hz.".format(
-                line_noise
-            )
+        logger.info(
+            f"Line noise is not available in the data, using value of {line_noise} Hz."
         )
     return (
         raw_arr,
@@ -120,7 +125,6 @@ def read_grid(PATH_GRIDS: _PathLike | None, grid_str: str) -> pd.DataFrame:
 def get_annotations(
     PATH_ANNOTATIONS: str, PATH_RUN: str, raw_arr: mne.io.RawArray
 ):
-
     try:
         annot = mne.read_annotations(
             Path(PATH_ANNOTATIONS) / (os.path.basename(PATH_RUN)[:-5] + ".txt")
@@ -130,18 +134,19 @@ def get_annotations(
         # annotations starting with "BAD" are omitted with reject_by_annotations 'omit' param
         annot_data = raw_arr.get_data(reject_by_annotation="omit")
     except FileNotFoundError:
-        print("Annotations file could not be found")
-        print(
-            "expected location: "
+        logger.critical(
+            "Annotations file could not be found"
+            + "expected location: "
             + str(
-                Path(PATH_ANNOTATIONS) / (os.path.basename(PATH_RUN)[:-5] + ".txt")
+                Path(PATH_ANNOTATIONS)
+                / (os.path.basename(PATH_RUN)[:-5] + ".txt")
             )
         )
     return annot, annot_data, raw_arr
 
 
 def read_plot_modules(
-    PATH_PLOT: _PathLike = Path(__file__).absolute().parent / "plots"
+    PATH_PLOT: _PathLike = Path(__file__).absolute().parent / "plots",
 ):
     """Read required .mat files for plotting
 
@@ -180,55 +185,6 @@ def read_plot_modules(
     )
 
 
-def add_labels(
-    features: pd.DataFrame,
-    settings: dict,
-    nm_channels: pd.DataFrame,
-    raw_arr_data: np.ndarray,
-    fs: int | float,
-) -> pd.DataFrame | None:
-    """Given a constructed feature data frame, resample the target labels and add to dataframe
-
-    Parameters
-    ----------
-    features : pd.DataFrame
-        computed feature dataframe
-    settings_wrapper : settings.py
-        initialized settings used for feature estimation
-    raw_arr_data : np.ndarray
-        raw data including target
-
-    Returns
-    -------
-    pd.DataFrame | None
-        computed feature dataframe including resampled features
-    """
-    # resample_label
-    ind_label = np.where(nm_channels.target == 1)[0]
-    if ind_label.shape[0] == 0:
-        print("no target specified")
-        return None
-
-    offset_time = settings["segment_length_features_ms"]
-
-    offset_start = np.ceil(offset_time / 1000 * fs).astype(int)
-    data = raw_arr_data[ind_label, offset_start:]
-    if data.ndim == 1:
-        data = np.expand_dims(data, axis=0)
-    label_downsampled = data[
-        :,
-        :: int(np.ceil(fs / settings["sampling_rate_features_hz"])),
-    ]
-
-    # and add to df
-    if features.shape[0] == label_downsampled.shape[1]:
-        for idx, label_ch in enumerate(nm_channels.name[ind_label]):
-            features[label_ch] = label_downsampled[idx, :]
-    else:
-        print("label dimensions don't match, saving downsampled label extra")
-    return features
-
-
 def save_features_and_settings(
     df_features,
     run_analysis,
@@ -256,7 +212,7 @@ def save_features_and_settings(
 
     # create out folder if doesn't exist
     if not os.path.exists(os.path.join(out_path, folder_name)):
-        print("Creating output folder: " + str(folder_name))
+        logger.Info(f"Creating output folder: {folder_name}")
         os.makedirs(os.path.join(out_path, folder_name))
 
     dict_sidecar = {"fs": fs, "coords": coords, "line_noise": line_noise}
@@ -267,6 +223,16 @@ def save_features_and_settings(
     save_nm_channels(nm_channels, out_path, folder_name)
 
 
+def write_csv(df, path_out):
+    """
+    Function to save Pandas dataframes to disk as CSV using
+    PyArrow (almost 10x faster than Pandas)
+    Difference with pandas.df.to_csv() is that it does not
+    write an index column by default
+    """
+    csv.write_csv(pyarrow.Table.from_pandas(df), path_out)
+
+
 def save_settings(
     settings: dict, path_out: _PathLike, folder_name: str | None = None
 ) -> None:
@@ -278,7 +244,7 @@ def save_settings(
 
     with open(path_out, "w") as f:
         json.dump(settings, f, indent=4)
-    print("settings.json saved to " + path_out)
+    logger.info(f"settings.json saved to {path_out}")
 
 
 def save_nm_channels(
@@ -291,8 +257,8 @@ def save_nm_channels(
         path_out = os.path.join(
             path_out, folder_name, folder_name + "_nm_channels.csv"
         )
-    nmchannels.to_csv(path_out)
-    print("nm_channels.csv saved to " + path_out)
+    write_csv(nmchannels, path_out)
+    logger.info(f"nm_channels.csv saved to {path_out}")
 
 
 def save_features(
@@ -305,8 +271,8 @@ def save_features(
         path_out = os.path.join(
             path_out, folder_name, folder_name + "_FEATURES.csv"
         )
-    df_features.to_csv(path_out)
-    print("FEATURES.csv saved to " + str(path_out))
+    write_csv(df_features, path_out)
+    logger.info(f"FEATURES.csv saved to {str(path_out)}")
 
 
 def save_sidecar(
@@ -333,7 +299,7 @@ def save_general_dict(
             indent=4,
             separators=(",", ": "),
         )
-    print(f"{str_add} saved to " + str(path_out))
+    logger.info(f"{str_add} saved to " + str(path_out))
 
 
 def default_json_convert(obj) -> list | int | float:
@@ -359,11 +325,11 @@ def read_settings(PATH: str) -> dict:
 
 
 def read_features(PATH: str) -> pd.DataFrame:
-    return pd.read_csv(PATH + "_FEATURES.csv", index_col=0)
+    return pd.read_csv(PATH + "_FEATURES.csv", engine="pyarrow")
 
 
 def read_nm_channels(PATH: str) -> pd.DataFrame:
-    return pd.read_csv(PATH + "_nm_channels.csv", index_col=0)
+    return pd.read_csv(PATH + "_nm_channels.csv")
 
 
 def get_run_list_indir(PATH: str) -> list:
@@ -405,7 +371,14 @@ def get_paths_example_data():
 
     PATH_BIDS = Path(SCRIPT_DIR) / "data"
 
-    PATH_RUN = Path(SCRIPT_DIR) / "data" / f"sub-{sub}" / f"ses-{ses}" / datatype / RUN_NAME
+    PATH_RUN = (
+        Path(SCRIPT_DIR)
+        / "data"
+        / f"sub-{sub}"
+        / f"ses-{ses}"
+        / datatype
+        / RUN_NAME
+    )
 
     # Provide a path for the output data.
     PATH_OUT = PATH_BIDS / "derivatives"

py_neuromodulation/nm_RMAP.py

@@ -1,14 +1,304 @@
-import enum
-import nibabel as nib
 import numpy as np
 import os
-#from numba import jit
+import wget
+
+
+# from numba import jit
 from scipy import stats
+import scipy.io as sio
 import pandas as pd
+from typing import Union, Tuple, List
+import nibabel as nib
+from matplotlib import pyplot as plt
+
+import py_neuromodulation
 
 from py_neuromodulation import nm_plots
 
-class RMAPChannelSelector:
+LIST_STRUC_UNCONNECTED_GRIDPOINTS_HULL = [256, 385, 417, 447, 819, 914]
+LIST_STRUC_UNCONNECTED_GRIDPOINTS_WHOLEBRAIN = [
+    1,
+    8,
+    16,
+    33,
+    34,
+    35,
+    36,
+    37,
+    51,
+    75,
+    77,
+    78,
+    99,
+    109,
+    115,
+    136,
+    155,
+    170,
+    210,
+    215,
+    243,
+    352,
+    359,
+    361,
+    415,
+    416,
+    422,
+    529,
+    567,
+    569,
+    622,
+    623,
+    625,
+    627,
+    632,
+    633,
+    634,
+    635,
+    639,
+    640,
+    641,
+    643,
+    644,
+    650,
+    661,
+    663,
+    667,
+    683,
+    684,
+    685,
+    704,
+    708,
+    722,
+    839,
+    840,
+    905,
+    993,
+    1011,
+]
+
+
+class ConnectivityChannelSelector:
+
+    def __init__(
+        self,
+        whole_brain_connectome: bool = True,
+        func_connectivity: bool = True,
+    ) -> None:
+        """ConnectivityChannelSelector
+
+        Parameters
+        ----------
+        whole_brain_connectome : bool, optional
+            if True a 1236 whole-brain point grid is chosen,
+            if False, a 1025 point grid of the cortical hull is loaded,
+            by default True
+        func_connectivity : bool, optional
+            if true, functional connectivity fMRI is loaded,
+            if false structural dMRIby, default True
+        """
+
+        self.connectome_name = self._get_connectome_name(
+            whole_brain_connectome, func_connectivity
+        )
+
+        self.whole_brain_connectome = whole_brain_connectome
+        self.func_connectivity = func_connectivity
+
+        self.PATH_CONN_DECODING = os.path.join(
+            py_neuromodulation.__path__[0],
+            "ConnectivityDecoding",
+        )
+
+        if whole_brain_connectome:
+            self.PATH_GRID = os.path.join(
+                self.PATH_CONN_DECODING,
+                "mni_coords_whole_brain.mat",
+            )
+            self.grid = sio.loadmat(self.PATH_GRID)["downsample_ctx"]
+            if func_connectivity is False:
+                # reduce the grid to only valid points that are not in LIST_STRUC_UNCONNECTED_GRIDPOINTS_WHOLEBRAIN
+                self.grid = np.delete(
+                    self.grid,
+                    LIST_STRUC_UNCONNECTED_GRIDPOINTS_WHOLEBRAIN,
+                    axis=0,
+                )
+        else:
+            self.PATH_GRID = os.path.join(
+                self.PATH_CONN_DECODING,
+                "mni_coords_cortical_surface.mat",
+            )
+            self.grid = sio.loadmat(self.PATH_GRID)["downsample_ctx"]
+            if func_connectivity is False:
+                # reduce the grid to only valid points that are not in LIST_STRUC_UNCONNECTED_GRIDPOINTS_HULL
+                self.grid = np.delete(
+                    self.grid, LIST_STRUC_UNCONNECTED_GRIDPOINTS_HULL, axis=0
+                )
+
+        if func_connectivity:
+            self.RMAP_arr = nib.load(
+                os.path.join(self.PATH_CONN_DECODING, "RMAP_func_all.nii")
+            ).get_fdata()
+        else:
+            self.RMAP_arr = nib.load(
+                os.path.join(self.PATH_CONN_DECODING, "RMAP_struc.nii")
+            ).get_fdata()
+
+    def _get_connectome_name(
+        self, whole_brain_connectome: str, func_connectivity: str
+    ):
+
+        connectome_name = "connectome_"
+        if whole_brain_connectome:
+            connectome_name += "whole_brain_"
+        else:
+            connectome_name += "hull_"
+        if func_connectivity:
+            connectome_name += "func"
+        else:
+            connectome_name += "struc"
+        return connectome_name
+
+    def get_available_connectomes(self) -> list:
+        """Return list of saved connectomes in the
+        package folder/ConnectivityDecoding/connectome_folder/ folder.
+
+        Returns
+        -------
+        list_connectomes: list
+        """
+        return os.listdir(
+            os.path.join(
+                self.PATH_CONN_DECODING,
+                "connectome_folder",
+            )
+        )
+
+    def plot_grid(self) -> None:
+        """Plot the loaded template grid that passed coordinates are matched to."""
+
+        fig = plt.figure()
+        ax = fig.add_subplot(111, projection="3d")
+        ax.scatter(
+            self.grid[:, 0], self.grid[:, 1], self.grid[:, 2], s=50, alpha=0.2
+        )
+        plt.show()
+
+    def get_closest_node(
+        self, coord: Union[List, np.array]
+    ) -> Tuple[List, List]:
+        """Given a list or np.array of coordinates, return the closest nodes in the
+        grid and their indices.
+
+        Parameters
+        ----------
+        coord : np.array
+            MNI coordinates with shape (num_channels, 3)
+
+        Returns
+        -------
+        Tuple[List, List]
+            Grid coordinates, grid indices
+        """
+
+        idx_ = []
+        for c in coord:
+            dist = np.linalg.norm(self.grid - c, axis=1)
+            idx_.append(np.argmin(dist))
+
+        return [self.grid[idx] for idx in idx_], idx_
+
+    def get_rmap_correlations(
+        self, fps: Union[list, np.array], RMAP_use: np.array = None
+    ) -> List:
+        """Calculate correlations of passed fingerprints with the RMAP
+
+        Parameters
+        ----------
+        fps : Union[list, np.array]
+            List of fingerprints
+        RMAP_use : np.array, optional
+            Passed RMAP, by default None
+
+        Returns
+        -------
+        List
+            correlation values
+        """
+
+        RMAP_ = self.RMAP_arr if RMAP_use is None else RMAP_use
+        RMAP_ = RMAP_.flatten()
+        corrs = []
+        for fp in fps:
+            corrs.append(np.corrcoef(RMAP_, fp.flatten())[0][1])
+        return corrs
+
+    def load_connectome(
+        self,
+        whole_brain_connectome: bool = None,
+        func_connectivity: bool = None,
+    ) -> None:
+        """Load connectome, if not available download connectome from
+        Zenodo.
+
+        Parameters
+        ----------
+        whole_brain_connectome : bool, optional
+            if true whole brain connectome
+            if false cortical hull grid connectome, by default None
+        func_connectivity : bool, optional
+            if true fMRI if false dMRI, by default None
+        """
+
+        if whole_brain_connectome is not None:
+            self.whole_brain_connectome = whole_brain_connectome
+        if func_connectivity is not None:
+            self.func_connectivity = func_connectivity
+
+        self.connectome_name = self._get_connectome_name(
+            self.whole_brain_connectome, self.func_connectivity
+        )
+
+        PATH_CONNECTOME = os.path.join(
+            self.PATH_CONN_DECODING,
+            "connectome_folder",
+            self.connectome_name + ".mat",
+        )
+
+        if os.path.exists(PATH_CONNECTOME) is False:
+            user_input = input(
+                "Do you want to download the connectome? (yes/no): "
+            ).lower()
+            if user_input == "yes":
+                self._download_connectome()
+            elif user_input == "no":
+                print("Connectome missing, has to be downloaded")
+
+        self.connectome = sio.loadmat(PATH_CONNECTOME)
+
+    def get_grid_fingerprints(self, grid_idx: Union[list, np.array]) -> list:
+        return [self.connectome[str(grid_idx)] for grid_idx in grid_idx]
+
+    def download_connectome(
+        self,
+    ):
+        # download the connectome from the Zenodo API
+        print("Downloading the connectome...")
+
+        record_id = "10804702"
+        file_name = self.connectome_name
+
+        wget.download(
+            f"https://zenodo.org/api/records/{record_id}/files/{file_name}/content",
+            out=os.path.join(
+                self.PATH_CONN_DECODING,
+                "connectome_folder",
+                f"{self.connectome_name}.mat",
+            ),
+        )
+
+
+class RMAPCross_Val_ChannelSelector:
 
     def __init__(self) -> None:
         pass
@@ -90,7 +380,7 @@ class RMAPChannelSelector:
         return r
 
     @staticmethod
-    #@jit(nopython=True)
+    # @jit(nopython=True)
     def calculate_RMap_numba(fp, performances):
         # The RMap also needs performances; for every fingerprint / channel
         # Save the corresponding performance
@@ -112,7 +402,7 @@ class RMAPChannelSelector:
         return RMAP
 
     @staticmethod
-    #@jit(nopython=True)
+    # @jit(nopython=True)
     def get_corr_numba(fp, fp_test):
         val = np.corrcoef(fp_test, fp)[0][1]
         return val
@@ -126,7 +416,7 @@ class RMAPChannelSelector:
         per_predict = []
 
         for idx_left_out, f_left_out in enumerate(l_fps_names):
-            print(idx_left_out)
+            # print(idx_left_out)
             l_cv = l_fps_dat.copy()
             per_cv = l_per.copy()
 
@@ -160,7 +450,7 @@ class RMAPChannelSelector:
         per_left_out = []
 
         for subject_test in sub_list:
-            print(subject_test)
+            # print(subject_test)
             idx_test = [
                 idx for idx, f in enumerate(l_fps_names) if subject_test in f
             ]
@@ -224,11 +514,18 @@ class RMAPChannelSelector:
         idx_max = np.argmax(np.array(fp_pairs)[:, 3])
         return fp_pairs[idx_max][0:3]
 
-    def plot_performance_prediction_correlation(per_left_out, per_predict, out_path_save: str = None):
+    def plot_performance_prediction_correlation(
+        per_left_out, per_predict, out_path_save: str = None
+    ):
         df_plt_corr = pd.DataFrame()
         df_plt_corr["test_performance"] = per_left_out
-        df_plt_corr["struct_conn_predict"] = per_predict  # change "struct" with "funct" for functional connectivity
+        df_plt_corr["struct_conn_predict"] = (
+            per_predict  # change "struct" with "funct" for functional connectivity
+        )
 
         nm_plots.reg_plot(
-            x_col="test_performance", y_col="struct_conn_predict", data=df_plt_corr, out_path_save=out_path_save
+            x_col="test_performance",
+            y_col="struct_conn_predict",
+            data=df_plt_corr,
+            out_path_save=out_path_save,
         )

py_neuromodulation/nm_analysis.py

@@ -221,7 +221,7 @@ class Feature_Reader:
             pd.DataFrame: z-scored feature_arr
         """
         cols_norm = [c for c in self.feature_arr.columns if "time" not in c]
-        feature_arr_norm = stats.zscore(self.feature_arr[cols_norm])
+        feature_arr_norm = stats.zscore(self.feature_arr[cols_norm], nan_policy="omit")
         feature_arr_norm["time"] = self.feature_arr["time"]
         return feature_arr_norm
 

py_neuromodulation/nm_artifacts.py

@@ -0,0 +1,25 @@
+from pyparrm import PARRM
+
+
+class PARRMArtifactRejection:
+    def __init__(self, data, sampling_freq, artefact_freq, verbose=False):
+        self.data = data
+        self.sampling_freq = sampling_freq
+        self.artefact_freq = artefact_freq
+        self.verbose = verbose
+
+        self.parrm = PARRM(
+            data=data,
+            sampling_freq=sampling_freq,
+            artefact_freq=artefact_freq,
+            verbose=False,
+        )
+
+    def filter_data(self):
+        self.parrm.find_period()
+        self.parrm.create_filter(
+            filter_direction="both",
+        )
+        filtered_data = self.parrm.filter_data()
+
+        return filtered_data