eegdash 0.0.2__tar.gz → 0.0.3__tar.gz

{eegdash-0.0.2 → eegdash-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: eegdash
-Version: 0.0.2
+Version: 0.0.3
 Summary: EEG data for machine learning
 Author-email: Young Truong <dt.young112@gmail.com>, Arnaud Delorme <adelorme@gmail.com>
 License: GNU General Public License
@@ -108,16 +108,16 @@ Additionally, users can search for a specific dataset by specifying criteria.
 EEGDashInstance.find({'task': 'FaceRecognition'})
 ```
 
-After locating the desired dataset or data record, users can download it locally by executing the following command:
+After locating the desired dataset or data record, users can download it locally by executing the following command. This will return an xArray Python object.
 
 ```python
-EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
+XArrayData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
 ```
 
-Optionally, this is how you may access the raw data for the first record.
+Optionally, this is how you may access the raw data for the first record. This will return an numpy array.
 
 ```python
-EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
+npData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
 ```
 
 ## Example use

{eegdash-0.0.2 → eegdash-0.0.3}/README.md

@@ -72,16 +72,16 @@ Additionally, users can search for a specific dataset by specifying criteria.
 EEGDashInstance.find({'task': 'FaceRecognition'})
 ```
 
-After locating the desired dataset or data record, users can download it locally by executing the following command:
+After locating the desired dataset or data record, users can download it locally by executing the following command. This will return an xArray Python object.
 
 ```python
-EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
+XArrayData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
 ```
 
-Optionally, this is how you may access the raw data for the first record.
+Optionally, this is how you may access the raw data for the first record. This will return an numpy array.
 
 ```python
-EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
+npData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
 ```
 
 ## Example use

{eegdash-0.0.2 → eegdash-0.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "eegdash"
-version = "0.0.2"
+version = "0.0.3"
 authors = [
   { name="Young Truong", email="dt.young112@gmail.com" },
   { name="Arnaud Delorme", email="adelorme@gmail.com" },

{eegdash-0.0.2/eegdash → eegdash-0.0.3/src}/data_utils.py

@@ -3,11 +3,104 @@ import sys
 from joblib import Parallel, delayed
 import mne
 import numpy as np
+import pandas as pd
 from pathlib import Path
 import re
 import json
+from mne.io import BaseRaw
+from mne._fiff.utils import _find_channels, _read_segments_file
+import s3fs
+import tempfile
+from mne._fiff.utils import _read_segments_file
 
-verbose = False
+class RawEEGDash(BaseRaw):
+    r"""Raw object from EEG-Dash connection with Openneuro S3 file.
+
+    Parameters
+    ----------
+    input_fname : path-like
+        Path to the S3 file
+    eog : list | tuple | 'auto'
+        Names or indices of channels that should be designated EOG channels.
+        If 'auto', the channel names containing ``EOG`` or ``EYE`` are used.
+        Defaults to empty tuple.
+    %(preload)s
+        Note that preload=False will be effective only if the data is stored
+        in a separate binary file.
+    %(uint16_codec)s
+    %(montage_units)s
+    %(verbose)s
+
+    See Also
+    --------
+    mne.io.Raw : Documentation of attributes and methods.
+
+    Notes
+    -----
+    .. versionadded:: 0.11.0
+    """
+
+    def __init__(
+        self,
+        input_fname,
+        metadata,
+        eog=(),
+        preload=False,
+        *,
+        cache_dir='./.eegdash_cache',
+        uint16_codec=None,
+        montage_units="auto",
+        verbose=None,
+    ):
+        '''
+        Get to work with S3 endpoint first, no caching
+        '''
+        # Create a simple RawArray
+        sfreq = metadata['sfreq']  # Sampling frequency
+        n_times = metadata['n_times']
+        ch_names = metadata['ch_names']
+        ch_types = []
+        for ch in metadata['ch_types']:
+            chtype = ch.lower()
+            if chtype == 'heog' or chtype == 'veog':
+                chtype = 'eog'
+            ch_types.append(chtype)
+        info = mne.create_info(ch_names=ch_names, sfreq=sfreq, ch_types=ch_types)
+        self.s3file = input_fname
+        os.makedirs(cache_dir, exist_ok=True)
+        self.filecache = os.path.join(cache_dir, os.path.basename(self.s3file))
+
+        if preload and not os.path.exists(self.filecache):
+            self._download_s3()
+            preload = self.filecache
+
+        super().__init__(
+            info,
+            preload,
+            last_samps=[n_times-1],
+            orig_format="single",
+            verbose=verbose,
+        )
+
+    def _download_s3(self):
+        filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
+        print('s3file', self.s3file)
+        print('filecache', self.filecache)
+        filesystem.download(self.s3file, self.filecache)
+        self.filenames = [self.filecache]
+
+    def _read_segment(
+        self, start=0, stop=None, sel=None, data_buffer=None, *, verbose=None
+    ):
+        if not os.path.exists(self.filecache): # not preload
+            self._download_s3()
+        else: # not preload and file is not cached
+            self.filenames = [self.filecache]
+        return super()._read_segment(start, stop, sel, data_buffer, verbose=verbose)
+    
+    def _read_segment_file(self, data, idx, fi, start, stop, cals, mult):
+        """Read a chunk of data from the file."""
+        _read_segments_file(self, data, idx, fi, start, stop, cals, mult, dtype="<f4")
 
 
 class BIDSDataset():
@@ -20,7 +113,7 @@ class BIDSDataset():
     }
     METADATA_FILE_EXTENSIONS = ['eeg.json', 'channels.tsv', 'electrodes.tsv', 'events.tsv', 'events.json']
     def __init__(self,
-            data_dir=None,                            # location of asr cleaned data 
+            data_dir=None,                            # location of bids dataset 
             dataset='',                               # dataset name
             raw_format='eeglab',                      # format of raw data
         ):                            
@@ -51,6 +144,18 @@ class BIDSDataset():
             lookup = re.search(rf'{property}-(.*?)[_\/]', filename)
         return lookup.group(1) if lookup else ''
 
+    def merge_json_inheritance(self, json_files):
+        '''
+        Merge list of json files found by get_bids_file_inheritance,
+        expecting the order (from left to right) is from lowest level to highest level,
+        and return a merged dictionary
+        '''
+        json_files.reverse()
+        json_dict = {}
+        for f in json_files:
+            json_dict.update(json.load(open(f)))
+        return json_dict
+
     def get_bids_file_inheritance(self, path, basename, extension):
         '''
         Get all files with given extension that applies to the basename file 
@@ -72,7 +177,7 @@ class BIDSDataset():
         for file in os.listdir(path):
             # target_file = path / f"{cur_file_basename}_{extension}"
             if os.path.isfile(path/file):
-                cur_file_basename = file[:file.rfind('_')]
+                cur_file_basename = file[:file.rfind('_')] # TODO: change to just search for any file with extension
                 if file.endswith(extension) and cur_file_basename in basename:
                     filepath = path / file
                     bids_files.append(filepath)
@@ -214,4 +319,21 @@ class BIDSDataset():
         return self.get_property_from_filename('run', data_filepath)
 
     def subject(self, data_filepath):
-        return self.get_property_from_filename('sub', data_filepath)
+        return self.get_property_from_filename('sub', data_filepath)
+
+    def num_channels(self, data_filepath):
+        channels_tsv = pd.read_csv(self.get_bids_metadata_files(data_filepath, 'channels.tsv')[0], sep='\t')
+        return len(channels_tsv)
+
+    def channel_labels(self, data_filepath):
+        channels_tsv = pd.read_csv(self.get_bids_metadata_files(data_filepath, 'channels.tsv')[0], sep='\t')
+        return channels_tsv['name'].tolist()
+    
+    def channel_types(self, data_filepath):
+        channels_tsv = pd.read_csv(self.get_bids_metadata_files(data_filepath, 'channels.tsv')[0], sep='\t')
+        return channels_tsv['type'].tolist()
+            
+    def num_times(self, data_filepath):
+        eeg_jsons = self.get_bids_metadata_files(data_filepath, 'eeg.json')
+        eeg_json_dict = self.merge_json_inheritance(eeg_jsons)
+        return int(eeg_json_dict['SamplingFrequency'] * eeg_json_dict['RecordingDuration'])

{eegdash-0.0.2 → eegdash-0.0.3/src}/eegdash.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: eegdash
-Version: 0.0.2
+Version: 0.0.3
 Summary: EEG data for machine learning
 Author-email: Young Truong <dt.young112@gmail.com>, Arnaud Delorme <adelorme@gmail.com>
 License: GNU General Public License
@@ -108,16 +108,16 @@ Additionally, users can search for a specific dataset by specifying criteria.
 EEGDashInstance.find({'task': 'FaceRecognition'})
 ```
 
-After locating the desired dataset or data record, users can download it locally by executing the following command:
+After locating the desired dataset or data record, users can download it locally by executing the following command. This will return an xArray Python object.
 
 ```python
-EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
+XArrayData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
 ```
 
-Optionally, this is how you may access the raw data for the first record.
+Optionally, this is how you may access the raw data for the first record. This will return an numpy array.
 
 ```python
-EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
+npData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
 ```
 
 ## Example use

eegdash-0.0.3/src/eegdash.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+LICENSE
+README.md
+pyproject.toml
+src/__init__.py
+src/data_utils.py
+src/main.py
+src/eegdash.egg-info/PKG-INFO
+src/eegdash.egg-info/SOURCES.txt
+src/eegdash.egg-info/dependency_links.txt
+src/eegdash.egg-info/requires.txt
+src/eegdash.egg-info/top_level.txt

eegdash-0.0.3/src/eegdash.egg-info/top_level.txt

@@ -0,0 +1,4 @@
+DomainModels
+__init__
+data_utils
+main

eegdash-0.0.3/src/main.py

@@ -0,0 +1,199 @@
+import pymongo
+from dotenv import load_dotenv
+import os
+import s3fs
+from joblib import Parallel, delayed
+import tempfile
+import mne
+import numpy as np
+import xarray as xr
+from .data_utils import BIDSDataset
+class EEGDash:
+    AWS_BUCKET = 's3://openneuro.org'
+    def __init__(self, 
+                 is_public=True):
+        if is_public:
+            DB_CONNECTION_STRING="mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0"
+        else:
+            load_dotenv()
+            DB_CONNECTION_STRING = os.getenv('DB_CONNECTION_STRING')
+
+        self.__client = pymongo.MongoClient(DB_CONNECTION_STRING)
+        self.__db = self.__client['eegdash']
+        self.__collection = self.__db['records']   
+
+        self.is_public = is_public
+        self.filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
+    
+    def find(self, *args):
+        results = self.__collection.find(*args)
+        
+        # convert to list using get_item on each element
+        return [result for result in results]
+
+    def exist(self, schema_ref='eeg_signal', data_name=''):
+        query = {
+            "schema_ref": schema_ref,
+            "data_name": data_name
+        }
+        sessions = self.find(query)
+        return len(sessions) > 0
+
+    def add(self, record:dict):
+        input_record = self._validate_input(record)
+        print(input_record)
+        self.__collection.insert_one(input_record)
+
+    def _validate_input(self, record:dict):
+        input_types = {
+            'schema_ref': str,
+            'data_name': str,
+            'dataset': str,
+            'bidspath': str,
+            'subject': str,
+            'task': str,
+            'session': str,
+            'run': str,
+            'sampling_frequency': float,
+            'modality': str,
+            'nchans': int,
+            'ntimes': int,
+            'channel_types': list,
+            'channel_names': list,
+        }
+        record['schema_ref'] = 'eeg_signal'
+        if 'data_name' not in record:
+            raise ValueError("Missing key: data_name")
+        # check if args are in the keys and has correct type
+        for key,value in record.items():
+            if key not in input_types:
+                raise ValueError(f"Invalid input: {key}")
+            if not isinstance(value, input_types[key]):
+                raise ValueError(f"Invalid input: {key}")
+
+        return record
+
+    def load_eeg_data_from_s3(self, s3path):
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.set') as tmp:
+            with self.filesystem.open(s3path) as s3_file:
+                tmp.write(s3_file.read())
+            tmp_path = tmp.name
+            eeg_data = self.load_eeg_data_from_bids_file(tmp_path)
+            os.unlink(tmp_path)
+            return eeg_data
+
+    def load_eeg_data_from_bids_file(self,  bids_file, eeg_attrs=None):
+        '''
+        bids_file must be a file of the bids_dataset
+        '''
+        EEG = mne.io.read_raw_eeglab(bids_file)
+        eeg_data = EEG.get_data()
+    
+        fs = EEG.info['sfreq']
+        max_time = eeg_data.shape[1] / fs
+        time_steps = np.linspace(0, max_time, eeg_data.shape[1]).squeeze() # in seconds
+
+        channel_names = EEG.ch_names
+
+        eeg_xarray = xr.DataArray(
+            data=eeg_data,
+            dims=['channel','time'],
+            coords={
+                'time': time_steps,
+                'channel': channel_names
+            },
+            # attrs=attrs
+        )
+        return eeg_xarray
+
+    def load_eeg_attrs_from_bids_file(self, bids_dataset: BIDSDataset, bids_file):
+        '''
+        bids_file must be a file of the bids_dataset
+        '''
+        if bids_file not in bids_dataset.files:
+            raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
+        f = os.path.basename(bids_file)
+        dsnumber = bids_dataset.dataset
+        # extract openneuro path by finding the first occurrence of the dataset name in the filename and remove the path before that
+        openneuro_path = dsnumber + bids_file.split(dsnumber)[1]
+
+        attrs = {
+            'schema_ref': 'eeg_signal',
+            'data_name': f'{bids_dataset.dataset}_{f}',
+            'dataset': bids_dataset.dataset,
+            'bidspath': openneuro_path,
+            'subject': bids_dataset.subject(bids_file),
+            'nchans': bids_dataset.num_channels(bids_file),
+            'ntimes': bids_dataset.num_times(bids_file),
+            'channel_types': bids_dataset.channel_types(bids_file),
+            'channel_names': bids_dataset.channel_labels(bids_file),
+            'task': bids_dataset.task(bids_file),
+            'session': bids_dataset.session(bids_file),
+            'run': bids_dataset.run(bids_file),
+            'sampling_frequency': bids_dataset.sfreq(bids_file), 
+            'modality': 'EEG',
+        }
+
+        return attrs
+
+    def add_bids_dataset(self, dataset, data_dir, raw_format='eeglab', overwrite=True):
+        '''
+        Create new records for the dataset in the MongoDB database if not found
+        '''
+        if self.is_public:
+            raise ValueError('This operation is not allowed for public users')
+
+        bids_dataset = BIDSDataset(
+            data_dir=data_dir,
+            dataset=dataset,
+            raw_format=raw_format,
+        )
+        for bids_file in bids_dataset.get_files():
+            print('bids raw file', bids_file)
+
+            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
+
+            if self.exist(data_name=signalstore_data_id):
+                if overwrite:
+                    eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                    print('updating record', eeg_attrs['data_name'])
+                    self.update(eeg_attrs)
+                else:
+                    print('data already exist and not overwriting. skipped')
+                    continue
+            else:
+                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                # Assume raw data already exists on Openneuro, recreating record only
+                print('adding record', eeg_attrs['data_name'])
+                self.add(eeg_attrs)
+
+    def get_s3path(self, record):
+        return f"{self.AWS_BUCKET}/{record['bidspath']}"
+
+    def get(self, query:dict):
+        '''
+        query: {
+            'dataset': 'dsxxxx',
+
+        }'''
+        sessions = self.find(query)
+        results = []
+        if sessions:
+            print(f'Found {len(sessions)} records')
+            results = Parallel(n_jobs=-1 if len(sessions) > 1 else 1, prefer="threads", verbose=1)(
+                delayed(self.load_eeg_data_from_s3)(self.get_s3path(session)) for session in sessions
+            )
+        return results
+
+    def update(self, record:dict):
+        record['schema_ref'] = 'eeg_signal'
+        self.__collection.update_one({'schema_ref': record['schema_ref'], 'data_name': record['data_name']}, 
+                    {'$set': record}
+                )
+def main():
+    eegdash = EEGDash()
+    record = eegdash.find({'dataset': 'ds005511', 'subject': 'NDARUF236HM7'})
+    print(record)
+
+if __name__ == '__main__':
+    main()

eegdash-0.0.2/eegdash/main.py

@@ -1,17 +0,0 @@
-from eegdash.signalstore_data_utils import SignalstoreBIDS
-
-class EEGDash:
-    def __init__(self):
-        self.sstore = SignalstoreBIDS(
-            # dbconnectionstring='mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.3.1',
-            dbconnectionstring='mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0',
-            is_public=True,
-            local_filesystem=False,
-            project_name='eegdash'
-        )
-    
-    def find(self, *args):
-        return self.sstore.find(*args)
-    
-    def get(self, *args):
-        return self.sstore.get(*args)

eegdash-0.0.2/eegdash/script.py

@@ -1,25 +0,0 @@
-import argparse
-from signalstore_data_utils import SignalstoreOpenneuro
-
-def main():
-    # Create the parser
-    parser = argparse.ArgumentParser(description="A simple command line argument parser")
-
-    # Add arguments
-    parser.add_argument('--data', type=str, default="/mnt/nemar/openneuro/ds004186", help="Path to data directory (Default: /mnt/nemar/openneuro/ds004186)")
-    parser.add_argument('--dataset', type=str, default="ds004186", help="Dataset name (Default: ds004186)")
-
-    # Parse the arguments
-    args = parser.parse_args()
-    print('Arguments:', args)
-
-    signalstore = SignalstoreOpenneuro(
-        is_public=False,
-        local_filesystem=False,
-    )
-    hbn_datasets = ['ds005505', 'ds005510', 'ds005514','ds005512','ds005511','ds005509','ds005508','ds005507','ds005506']
-    for ds in hbn_datasets:
-        signalstore.add_bids_dataset(dataset=ds, data_dir=f'/mnt/nemar/openneuro/{ds}', raw_format='eeglab')
-
-if __name__ == "__main__":
-    main()

eegdash-0.0.2/eegdash/signalstore_data_utils.py

@@ -1,630 +0,0 @@
-from pathlib import Path
-from dotenv import load_dotenv
-import re
-import numpy as np
-import xarray as xr
-import os
-from signalstore.store import UnitOfWorkProvider
-# from mongomock import MongoClient
-from pymongo.mongo_client import MongoClient
-from pymongo.server_api import ServerApi
-from fsspec.implementations.local import LocalFileSystem
-from fsspec.implementations.dirfs import DirFileSystem
-import pandas as pd
-import json
-import s3fs
-from signalstore.store.data_access_objects import FileSystemDAO
-from .data_utils import BIDSDataset
-import tempfile
-import mne
-from joblib import Parallel, delayed
-
-class SignalstoreOpenneuro():
-    AWS_BUCKET = 'openneuro.org'
-    PROJECT_NAME = 'eegdash'
-    def __init__(self, 
-                 dbconnectionstring="mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.3.1",
-                 is_public=False,
-                 local_filesystem=True,
-                 ):
-        self.is_public = is_public
-        self.project_name = self.PROJECT_NAME
-        if is_public:
-            dbconnectionstring='mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0',
-        else:
-            load_dotenv()
-            dbconnectionstring = os.getenv('DB_CONNECTION_STRING')
-
-        # Create a new client and connect to the server
-        client = MongoClient(dbconnectionstring, server_api=ServerApi('1'))
-        # Send a ping to confirm a successful connection
-        try:
-            client.admin.command('ping')
-            print("Pinged your deployment. You successfully connected to MongoDB!")
-        except Exception as e:
-            print(e)
-
-        memory_store = {}
-        self.filesystem = self.set_up_filesystem(is_local=local_filesystem)
-        self.uow_provider = UnitOfWorkProvider(
-            mongo_client=client,
-            filesystem=self.filesystem,
-            memory_store=memory_store,
-            default_filetype='zarr'
-        )
-
-        self.uow = self.uow_provider(self.PROJECT_NAME)
-        self.load_domain_models()
-
-    def set_up_filesystem(self, is_local=True):
-        if is_local:
-            cache_path='/mnt/nemar/dtyoung/eeg-dash-data'                  # path where signalstore netCDF files are stored
-            # Create a directory for the dataset
-            store_path = Path(cache_path)
-            if not os.path.exists(store_path):
-                os.makedirs(store_path)
-
-            filesystem = LocalFileSystem()
-            tmp_dir_fs = DirFileSystem(
-                store_path,
-                filesystem=filesystem
-            )
-            return tmp_dir_fs
-        else:
-            s3 = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
-            return s3
-
-    def load_domain_models(self):
-        dir_path = os.path.dirname(os.path.realpath(__file__))
-        cwd = Path(dir_path)
-        domain_models_path = cwd / f"DomainModels/{self.project_name}/data_models.json"
-        metamodel_path = cwd / f"DomainModels/{self.project_name}/metamodels.json"
-        property_path = cwd / f"DomainModels/{self.project_name}/property_models.json"
-        with open(metamodel_path) as f:
-            metamodels = json.load(f)
-
-        with open(property_path) as f:
-            property_models = json.load(f)
-            
-        # load domain models json file
-        with open(domain_models_path) as f:
-            domain_models = json.load(f)
-            
-        with self.uow as uow:
-            for property_model in property_models:
-                if not uow.domain_models.exists(property_model['schema_name']):
-                    uow.domain_models.add(property_model)
-                    model = uow.domain_models.get(property_model['schema_name'])
-                    print('property model: ', model['schema_name'])
-            for metamodel in metamodels:
-                if not uow.domain_models.exists(metamodel['schema_name']):
-                    uow.domain_models.add(metamodel)
-                    model = uow.domain_models.get(metamodel['schema_name'])
-                    print('meta model: ', model['schema_name'])
-            for domain_model in domain_models:
-                if not uow.domain_models.exists(domain_model['schema_name']):
-                    uow.domain_models.add(domain_model)
-                    model = uow.domain_models.get(domain_model['schema_name'])
-                    print('domain model: ', model['schema_name'])
-                uow.commit()
-
-    def extract_attribute(self, pattern, filename):
-        match = re.search(pattern, filename)
-        return match.group(1) if match else None
-
-    def load_eeg_attrs_from_bids_file(self, bids_dataset: BIDSDataset, bids_file):
-        '''
-        bids_file must be a file of the bids_dataset
-        '''
-        if bids_file not in bids_dataset.files:
-            raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
-        f = os.path.basename(bids_file)
-        dsnumber = bids_dataset.dataset
-        # extract openneuro path by finding the first occurrence of the dataset name in the filename and remove the path before that
-        openneuro_path = dsnumber + bids_file.split(dsnumber)[1]
-
-        attrs = {
-            'schema_ref': 'eeg_signal',
-            'data_name': f'{bids_dataset.dataset}_{f}',
-            'dataset': bids_dataset.dataset,
-            'bidspath': openneuro_path,
-            'subject': bids_dataset.subject(bids_file),
-            'task': bids_dataset.task(bids_file),
-            'session': bids_dataset.session(bids_file),
-            'run': bids_dataset.run(bids_file),
-            'sampling_frequency': bids_dataset.sfreq(bids_file), 
-            'modality': 'EEG',
-        }
-
-        return attrs
-
-    def load_eeg_data_from_s3(self, s3path):
-        # import boto3
-        # import scipy.io
-        # import io
-
-
-        # # Initialize the S3 client
-        # s3 = boto3.client('s3')
-
-
-        # # S3 bucket and object key
-        # bucket_name = 'your-bucket-name'
-        # object_key = 'path/to/your/file.mat'
-
-
-        # # Get the object from S3 and stream it into memory
-        # response = s3.get_object(Bucket=bucket_name, Key=object_key)
-
-
-        # # Read the content into a BytesIO buffer
-        # mat_file_stream = io.BytesIO(response['Body'].read())
-
-
-        # # Load the MAT file using scipy.io.loadmat
-        # data = scipy.io.loadmat(mat_file_stream)
-
-
-        # # Work with the data
-        # print(data)
-        with tempfile.NamedTemporaryFile(delete=False, suffix='.set') as tmp:
-            with self.filesystem.open(s3path) as s3_file:
-                tmp.write(s3_file.read())
-            tmp_path = tmp.name
-            eeg_data = self.load_eeg_data_from_bids_file(tmp_path)
-            os.unlink(tmp_path)
-            return eeg_data
-
-    def load_eeg_data_from_bids_file(self,  bids_file, eeg_attrs=None):
-        '''
-        bids_file must be a file of the bids_dataset
-        '''
-        EEG = mne.io.read_raw_eeglab(bids_file)
-        eeg_data = EEG.get_data()
-    
-        fs = EEG.info['sfreq']
-        max_time = eeg_data.shape[1] / fs
-        time_steps = np.linspace(0, max_time, eeg_data.shape[1]).squeeze() # in seconds
-
-        channel_names = EEG.ch_names
-
-        eeg_xarray = xr.DataArray(
-            data=eeg_data,
-            dims=['channel','time'],
-            coords={
-                'time': time_steps,
-                'channel': channel_names
-            },
-            # attrs=attrs
-        )
-        return eeg_xarray
-
-    def exist(self, schema_ref='eeg_signal', data_name=''):
-        with self.uow as uow:
-            query = {
-                "schema_ref": schema_ref,
-                "data_name": data_name
-            }
-            sessions = uow.data.find(query)
-            if len(sessions) > 0:
-                return True
-            else:
-                return False
-
-    def add_bids_dataset(self, dataset, data_dir, raw_format='eeglab', overwrite=False):
-        '''
-        Create new records for the dataset in the MongoDB database if not found
-        '''
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        bids_dataset = BIDSDataset(
-            data_dir=data_dir,
-            dataset=dataset,
-            raw_format=raw_format,
-        )
-        for bids_file in bids_dataset.get_files():
-            print('bids raw file', bids_file)
-
-            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
-            if overwrite:
-                self.remove(signalstore_data_id)
-
-            if self.exist(data_name=signalstore_data_id):
-                print('data already exist. skipped')
-                continue
-            else:
-                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                with self.uow as uow:
-                    # Assume raw data already exists on Openneuro, recreating record only
-                    eeg_attrs['has_file'] = True
-                    print('adding record', eeg_attrs['data_name'])
-                    uow.data.add(eeg_attrs)
-                    uow.commit()
-
-    def update_bids_dataset(self, dataset, data_dir, raw_format='eeglab'):
-        '''
-        Create new records for the dataset in the MongoDB database if not found
-        '''
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        bids_dataset = BIDSDataset(
-            data_dir=data_dir,
-            dataset=dataset,
-            raw_format=raw_format,
-        )
-        for bids_file in bids_dataset.get_files():
-            print('bids raw file', bids_file)
-
-            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
-
-            if not self.exist(data_name=signalstore_data_id):
-                raise ValueError('data not found')
-            else:
-                self.remove(data_name=signalstore_data_id)
-
-                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                with self.uow as uow:
-                    # Assume raw data already exists on Openneuro, recreating record only
-                    eeg_attrs['has_file'] = True
-                    print('adding record', eeg_attrs['data_name'])
-                    uow.data.add(eeg_attrs)
-                    uow.commit()
-
-    def remove(self, schema_ref='eeg_signal', data_name=''):
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        print('Removing record', data_name)
-        with self.uow as uow:
-            sessions = uow.data.find({'schema_ref': schema_ref, 'data_name': data_name})
-            if len(sessions) > 0:
-                for session in sessions:
-                    uow.data.remove(session['schema_ref'], session['data_name'])
-                    uow.commit()
-            uow.purge()
-            assert len(uow.data.find({'schema_ref': schema_ref, 'data_name': data_name})) == 0, 'Data still exists'
-
-    def remove_all(self):
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        with self.uow as uow:
-            sessions = uow.data.find({})
-            print(len(sessions))
-            for session in range(len(sessions)):
-                uow.data.remove(session['schema_ref'], session['data_name'])
-                uow.commit()
-
-            uow.purge()
-            
-            print('Verifying deletion job. Dataset length: ', len(uow.data.find({})))
-
-    def find(self, query:dict, validate=False):
-        '''
-        query: {
-            'dataset': 'dsxxxx',
-
-        }'''
-        with self.uow as uow:
-            sessions = uow.data.find(query, validate=validate)
-            if sessions:
-                print(f'Found {len(sessions)} records')
-                return sessions
-            else:
-                return []
-
-    def get(self, query:dict, validate=False):
-        '''
-        query: {
-            'dataset': 'dsxxxx',
-
-        }'''
-        with self.uow as uow:
-            sessions = uow.data.find(query, validate=validate)
-            results = []
-            if sessions:
-                print(f'Found {len(sessions)} records')
-                results = Parallel(n_jobs=-1, prefer="threads", verbose=1)(
-                    delayed(self.load_eeg_data_from_s3)(Path(self.AWS_BUCKET) / session['bidspath']) for session in sessions
-                )
-            return results
-
-class SignalstoreBIDS():
-    AWS_BUCKET = 'eegdash'
-    def __init__(self, 
-                 project_name=AWS_BUCKET,
-                 dbconnectionstring="mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.3.1",
-                 is_public=False,
-                 local_filesystem=True,
-                 ):
-        self.is_public = is_public
-        if is_public:
-            dbconnectionstring='mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0',
-        else:
-            load_dotenv()
-            dbconnectionstring = os.getenv('DB_CONNECTION_STRING')
-
-        # Create a new client and connect to the server
-        client = MongoClient(dbconnectionstring, server_api=ServerApi('1'))
-        # Send a ping to confirm a successful connection
-        try:
-            client.admin.command('ping')
-            print("Pinged your deployment. You successfully connected to MongoDB!")
-        except Exception as e:
-            print(e)
-
-        memory_store = {}
-        filesystem = self.set_up_filesystem(is_local=local_filesystem)
-        self.uow_provider = UnitOfWorkProvider(
-            mongo_client=client,
-            filesystem=filesystem,
-            memory_store=memory_store,
-            default_filetype='zarr'
-        )
-
-        self.project_name=project_name
-        self.uow = self.uow_provider(self.project_name)
-        # self.load_domain_models()
-
-    def set_up_filesystem(self, is_local=True):
-        if is_local:
-            cache_path='/mnt/nemar/dtyoung/eeg-ssl-data'                  # path where signalstore netCDF files are stored
-            # Create a directory for the dataset
-            store_path = Path(cache_path)
-            if not os.path.exists(store_path):
-                os.makedirs(store_path)
-
-            filesystem = LocalFileSystem()
-            tmp_dir_fs = DirFileSystem(
-                store_path,
-                filesystem=filesystem
-            )
-            return tmp_dir_fs
-        else:
-            if self.is_public:
-                s3 = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
-            else:
-                s3 = s3fs.S3FileSystem(client_kwargs={'region_name': 'us-east-2'})
-            return s3
-
-    def load_domain_models(self):
-        cwd = Path.cwd()
-        domain_models_path = cwd / f"DomainModels/{self.project_name}/data_models.json"
-        metamodel_path = cwd / f"DomainModels/{self.project_name}/metamodels.json"
-        property_path = cwd / f"DomainModels/{self.project_name}/property_models.json"
-        with open(metamodel_path) as f:
-            metamodels = json.load(f)
-
-        with open(property_path) as f:
-            property_models = json.load(f)
-            
-        # load domain models json file
-        with open(domain_models_path) as f:
-            domain_models = json.load(f)
-            
-        with self.uow as uow:
-            for property_model in property_models:
-                uow.domain_models.add(property_model)
-                model = uow.domain_models.get(property_model['schema_name'])
-                print('property model: ', model['schema_name'])
-            for metamodel in metamodels:
-                uow.domain_models.add(metamodel)
-                model = uow.domain_models.get(metamodel['schema_name'])
-                print('meta model: ', model['schema_name'])
-            for domain_model in domain_models:
-                uow.domain_models.add(domain_model)
-                model = uow.domain_models.get(domain_model['schema_name'])
-                print('domain model: ', model['schema_name'])
-                uow.commit()
-
-    def extract_attribute(self, pattern, filename):
-        match = re.search(pattern, filename)
-        return match.group(1) if match else None
-
-    def load_eeg_attrs_from_bids_file(self, bids_dataset: BIDSDataset, bids_file):
-        '''
-        bids_file must be a file of the bids_dataset
-        '''
-        if bids_file not in bids_dataset.files:
-            raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
-        f = os.path.basename(bids_file)
-        attrs = {
-            'schema_ref': 'eeg_signal',
-            'data_name': f'{bids_dataset.dataset}_{f}',
-            'dataset': bids_dataset.dataset,
-            'subject': bids_dataset.subject(bids_file),
-            'task': bids_dataset.task(bids_file),
-            'session': bids_dataset.session(bids_file),
-            'run': bids_dataset.run(bids_file),
-            'sampling_frequency': bids_dataset.sfreq(bids_file), 
-            'modality': 'EEG',
-        }
-
-        return attrs
-
-    def load_eeg_data_from_bids_file(self, bids_dataset: BIDSDataset, bids_file, eeg_attrs=None):
-        '''
-        bids_file must be a file of the bids_dataset
-        '''
-        if bids_file not in bids_dataset.files:
-            raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
-
-        attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file) if eeg_attrs is None else eeg_attrs
-
-        eeg_data = bids_dataset.load_and_preprocess_raw(bids_file)
-        print('data shape:', eeg_data.shape)
-    
-        fs = attrs['sampling_frequency']
-        max_time = eeg_data.shape[1] / fs
-        time_steps = np.linspace(0, max_time, eeg_data.shape[1]).squeeze() # in seconds
-        # print('time steps', len(time_steps))
-
-        # replace eeg.set with channels.tsv
-        # todo this is still a hacky way
-        channels_tsv = bids_dataset.get_bids_metadata_files(bids_file, 'channels.tsv')
-        channels_tsv = Path(channels_tsv[0]) 
-        if channels_tsv.exists():
-            channels = pd.read_csv(channels_tsv, sep='\t') 
-            # get channel names from channel_coords
-            channel_names = channels['name'].values
-
-        eeg_xarray = xr.DataArray(
-            data=eeg_data,
-            dims=['channel','time'],
-            coords={
-                'time': time_steps,
-                'channel': channel_names
-            },
-            attrs=attrs
-        )
-        return eeg_xarray
-
-    def exist(self, schema_ref='eeg_signal', data_name=''):
-        with self.uow as uow:
-            query = {
-                "schema_ref": schema_ref,
-                "data_name": data_name
-            }
-            sessions = uow.data.find(query)
-            if len(sessions) > 0:
-                return True
-            else:
-                return False
-
-    def add_bids_dataset(self, dataset, data_dir, raw_format='eeglab', overwrite=False, record_only=False):
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        bids_dataset = BIDSDataset(
-            data_dir=data_dir,
-            dataset=dataset,
-            raw_format=raw_format,
-        )
-        for bids_file in bids_dataset.get_files():
-            print('bids raw file', bids_file)
-
-            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
-            if overwrite:
-                self.remove(signalstore_data_id)
-
-            if self.exist(data_name=signalstore_data_id):
-                print('data already exist. skipped')
-                continue
-            else:
-                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                with self.uow as uow:
-                    # Assume raw data already exists, recreating record only
-                    eeg_attrs['has_file'] = True
-                    print('adding record', eeg_attrs['data_name'])
-                    uow.data.add(eeg_attrs)
-                    uow.commit()
-                if  not record_only:
-                    eeg_xarray = self.load_eeg_data_from_bids_file(bids_dataset, bids_file, eeg_attrs)
-                    with self.uow as uow:
-                        print('adding data', eeg_xarray.attrs['data_name'])
-                        uow.data.add(eeg_xarray)
-                        uow.commit()
-
-    def remove(self, schema_ref='eeg_signal', data_name=''):
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        with self.uow as uow:
-            sessions = uow.data.find({'schema_ref': schema_ref, 'data_name': data_name})
-            if len(session) > 0:
-                for session in range(len(sessions)):
-                    uow.data.remove(session['schema_ref'], session['data_name'])
-                    uow.commit()
-
-    def remove_all(self):
-        if self.is_public:
-            raise ValueError('This operation is not allowed for public users')
-
-        with self.uow as uow:
-            sessions = uow.data.find({})
-            print(len(sessions))
-            for session in range(len(sessions)):
-                uow.data.remove(session['schema_ref'], session['data_name'])
-                uow.commit()
-
-            uow.purge()
-            
-            print('Verifying deletion job. Dataset length: ', len(uow.data.find({})))
-
-    def find(self, query:dict, validate=False, get_data=False):
-        '''
-        query: {
-            'dataset': 'dsxxxx',
-
-        }'''
-        with self.uow as uow:
-            sessions = uow.data.find(query, validate=validate, get_data=get_data)
-            if sessions:
-                print(f'Found {len(sessions)} records')
-                return sessions
-            else:
-                return []
-
-    def get(self, query:dict, validate=False):
-        '''
-        query: {
-            'dataset': 'dsxxxx',
-
-        }'''
-        with self.uow as uow:
-            sessions = uow.data.find(query, validate=validate, get_data=True)
-            if sessions:
-                print(f'Found {len(sessions)} records')
-                return sessions
-            else:
-                return []
-
-class OpenneuroFileSystemDAO(FileSystemDAO):
-    def __init__(self):
-        filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
-        super().__init__(filesystem, project_dir='openneuro.org')
-    
-    def get(self, schema_ref, data_name, version_timestamp=0, nth_most_recent=1, data_adapter=None):
-        """Gets an object from the Openneuro S3 bucket.
-        Arguments:
-            schema_ref {str} -- The type of object to get.
-            data_name {str} -- The name of the object to get.
-            version_timestamp {str} -- The version_timestamp of the object to get.
-        Raises:
-            FileSystemDAOFileNotFoundError -- If the object is not found.
-        Returns:
-            dict -- The object.
-        """
-        self._check_args(
-            schema_ref=schema_ref,
-            data_name=data_name,
-            nth_most_recent=nth_most_recent,
-            version_timestamp=version_timestamp,
-            data_adapter=data_adapter
-            )
-        if data_adapter is None:
-            data_adapter = self._default_data_adapter
-        else:
-            data_adapter.set_filesystem(self._fs)
-        path = self._get_file_path(schema_ref, data_name, version_timestamp, nth_most_recent, data_adapter)
-        if path is None:
-            return None
-        data_object = data_adapter.read_file(path)
-        data_object = self._deserialize(data_object)
-        return data_object
-    
-        
-if __name__ == "__main__":
-    # sstore_hbn = SignalstoreHBN()
-    # sstore_hbn.add_data()
-    # sstore_ds004584 = SignalstoreHBN(
-    #     data_path='/mnt/nemar/openneuro/ds004584',
-    #     dataset_name='eegdash',
-    #     local_filesystem=False,
-    #     dbconnectionstring='mongodb://23.21.113.214:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.2.1'
-    # )
-    # sstore_ds004584.load_domain_models()
-    # sstore_ds004584.add_data()
-    pass

eegdash-0.0.2/eegdash.egg-info/SOURCES.txt

@@ -1,14 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-eegdash/__init__.py
-eegdash/data_utils.py
-eegdash/main.py
-eegdash/script.py
-eegdash/signalstore_data_utils.py
-eegdash.egg-info/PKG-INFO
-eegdash.egg-info/SOURCES.txt
-eegdash.egg-info/dependency_links.txt
-eegdash.egg-info/requires.txt
-eegdash.egg-info/top_level.txt
-tests/__init__.py

eegdash-0.0.2/eegdash.egg-info/top_level.txt

@@ -1 +0,0 @@
-eegdash

eegdash-0.0.2/tests/__init__.py

@@ -1,3 +0,0 @@
-from os import path
-import sys
-sys.path.append(path.abspath('../eegdash'))