eegdash 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

eegdash/__init__.py

@@ -1 +1 @@
-from .main import EEGDash
+from .main import EEGDash, EEGDashDataset

eegdash/data_utils.py

@@ -12,9 +12,106 @@ from mne._fiff.utils import _find_channels, _read_segments_file
 import s3fs
 import tempfile
 from mne._fiff.utils import _read_segments_file
+from braindecode.datasets import BaseDataset
+import mne_bids
+from mne_bids import (
+    BIDSPath,
+)
 
-class RawEEGDash(BaseRaw):
-    r"""Raw object from EEG-Dash connection with Openneuro S3 file.
+class EEGDashBaseDataset(BaseDataset):
+    """Returns samples from an mne.io.Raw object along with a target.
+
+    Dataset which serves samples from an mne.io.Raw object along with a target.
+    The target is unique for the dataset, and is obtained through the
+    `description` attribute.
+
+    Parameters
+    ----------
+    raw : mne.io.Raw
+        Continuous data.
+    description : dict | pandas.Series | None
+        Holds additional description about the continuous signal / subject.
+    target_name : str | tuple | None
+        Name(s) of the index in `description` that should be used to provide the
+        target (e.g., to be used in a prediction task later on).
+    transform : callable | None
+        On-the-fly transform applied to the example before it is returned.
+    """
+    AWS_BUCKET = 's3://openneuro.org'
+    def __init__(self, record, cache_dir, **kwargs):
+        super().__init__(None, **kwargs)
+        self.record = record
+        self.cache_dir = Path(cache_dir)
+        bids_kwargs = self.get_raw_bids_args()
+        self.bidspath = BIDSPath(root=self.cache_dir / record['dataset'], datatype='eeg', suffix='eeg', **bids_kwargs)
+        self.s3file = self.get_s3path(record['bidspath'])
+        self.filecache = self.cache_dir / record['bidspath']
+        self.bids_dependencies = record['bidsdependencies']
+        self._raw = None
+        # if os.path.exists(self.filecache):
+        #     self.raw = mne_bids.read_raw_bids(self.bidspath, verbose=False)
+
+    def get_s3path(self, filepath):
+        return f"{self.AWS_BUCKET}/{filepath}"
+
+    def _download_s3(self):
+        self.filecache.parent.mkdir(parents=True, exist_ok=True)
+        filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
+        filesystem.download(self.s3file, self.filecache)
+        self.filenames = [self.filecache]
+
+    def _download_dependencies(self):
+        filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
+        for dep in self.bids_dependencies:
+            s3path = self.get_s3path(dep)
+            filepath = self.cache_dir / dep
+            if not filepath.exists():
+                filepath.parent.mkdir(parents=True, exist_ok=True)
+                filesystem.download(s3path, filepath) 
+
+    def get_raw_bids_args(self):
+        desired_fields = ['subject', 'session', 'task', 'run']
+        return {k: self.record[k] for k in desired_fields if self.record[k]}
+
+    def check_and_get_raw(self):
+        if not os.path.exists(self.filecache): # not preload
+            if self.bids_dependencies:
+                self._download_dependencies()
+            self._download_s3()
+        if self._raw is None:
+            self._raw = mne_bids.read_raw_bids(self.bidspath, verbose=False)
+
+    def __getitem__(self, index):
+        # self.check_and_get_raw()
+
+        X = self.raw[:, index][0]
+        y = None
+        if self.target_name is not None:
+            y = self.description[self.target_name]
+        if isinstance(y, pd.Series):
+            y = y.to_list()
+        if self.transform is not None:
+            X = self.transform(X)
+        return X, y
+    
+    def __len__(self):
+        if self._raw is None:
+            return self.record['rawdatainfo']['ntimes']
+        else:
+            return len(self._raw)
+
+    @property
+    def raw(self):
+        if self._raw is None:
+            self.check_and_get_raw()
+        return self._raw
+
+    @raw.setter
+    def raw(self, raw):
+        self._raw = raw
+
+class EEGDashBaseRaw(BaseRaw):
+    r"""MNE Raw object from EEG-Dash connection with Openneuro S3 file.
 
     Parameters
     ----------
@@ -40,6 +137,7 @@ class RawEEGDash(BaseRaw):
     .. versionadded:: 0.11.0
     """
 
+    AWS_BUCKET = 's3://openneuro.org'
     def __init__(
         self,
         input_fname,
@@ -48,6 +146,7 @@ class RawEEGDash(BaseRaw):
         preload=False,
         *,
         cache_dir='./.eegdash_cache',
+        bids_dependencies:list = [],
         uint16_codec=None,
         montage_units="auto",
         verbose=None,
@@ -66,9 +165,10 @@ class RawEEGDash(BaseRaw):
                 chtype = 'eog'
             ch_types.append(chtype)
         info = mne.create_info(ch_names=ch_names, sfreq=sfreq, ch_types=ch_types)
-        self.s3file = input_fname
-        os.makedirs(cache_dir, exist_ok=True)
-        self.filecache = os.path.join(cache_dir, os.path.basename(self.s3file))
+        self.s3file = self.get_s3path(input_fname)
+        self.cache_dir = Path(cache_dir)
+        self.filecache = self.cache_dir / input_fname
+        self.bids_dependencies = bids_dependencies
 
         if preload and not os.path.exists(self.filecache):
             self._download_s3()
@@ -82,17 +182,30 @@ class RawEEGDash(BaseRaw):
             verbose=verbose,
         )
 
+    def get_s3path(self, filepath):
+        return f"{self.AWS_BUCKET}/{filepath}"
+
     def _download_s3(self):
+        self.filecache.parent.mkdir(parents=True, exist_ok=True)
         filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
-        print('s3file', self.s3file)
-        print('filecache', self.filecache)
         filesystem.download(self.s3file, self.filecache)
         self.filenames = [self.filecache]
 
+    def _download_dependencies(self):
+        filesystem = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
+        for dep in self.bids_dependencies:
+            s3path = self.get_s3path(dep)
+            filepath = self.cache_dir / dep
+            if not filepath.exists():
+                filepath.parent.mkdir(parents=True, exist_ok=True)
+                filesystem.download(s3path, filepath)
+
     def _read_segment(
         self, start=0, stop=None, sel=None, data_buffer=None, *, verbose=None
     ):
         if not os.path.exists(self.filecache): # not preload
+            if self.bids_dependencies:
+                self._download_dependencies()
             self._download_s3()
         else: # not preload and file is not cached
             self.filenames = [self.filecache]
@@ -121,6 +234,7 @@ class BIDSDataset():
             raise ValueError('data_dir must be specified and must exist')
         self.bidsdir = Path(data_dir)
         self.dataset = dataset
+        assert str(self.bidsdir).endswith(self.dataset)
 
         if raw_format.lower() not in self.ALLOWED_FILE_FORMAT:
             raise ValueError('raw_format must be one of {}'.format(self.ALLOWED_FILE_FORMAT))
@@ -136,6 +250,10 @@ class BIDSDataset():
         else:
             self.files = np.load(temp_dir / f'{dataset}_files.npy', allow_pickle=True)
 
+    def get_relative_bidspath(self, filename):
+        bids_parent_dir = self.bidsdir.parent
+        return str(Path(filename).relative_to(bids_parent_dir))
+
     def get_property_from_filename(self, property, filename):
         import platform
         if platform.system() == "Windows":
@@ -177,11 +295,17 @@ class BIDSDataset():
         for file in os.listdir(path):
             # target_file = path / f"{cur_file_basename}_{extension}"
             if os.path.isfile(path/file):
-                cur_file_basename = file[:file.rfind('_')] # TODO: change to just search for any file with extension
-                if file.endswith(extension) and cur_file_basename in basename:
+                # check if file has extension extension
+                # check if file basename has extension
+                if file.endswith(extension):
                     filepath = path / file
                     bids_files.append(filepath)
 
+                # cur_file_basename = file[:file.rfind('_')] # TODO: change to just search for any file with extension
+                # if file.endswith(extension) and cur_file_basename in basename:
+                #     filepath = path / file
+                #     bids_files.append(filepath)
+
         # check if file is in top level directory
         if any(file in os.listdir(path) for file in top_level_files):
             return bids_files
@@ -210,10 +334,7 @@ class BIDSDataset():
         basename = filename[:filename.rfind('_')]
         # metadata files
         meta_files = self.get_bids_file_inheritance(path, basename, metadata_file_extension)
-        if not meta_files:
-            raise ValueError('No metadata files found for filepath {filepath} and extension {metadata_file_extension}')
-        else:
-            return meta_files
+        return meta_files
         
     def scan_directory(self, directory, extension):
         result_files = []
@@ -336,4 +457,25 @@ class BIDSDataset():
     def num_times(self, data_filepath):
         eeg_jsons = self.get_bids_metadata_files(data_filepath, 'eeg.json')
         eeg_json_dict = self.merge_json_inheritance(eeg_jsons)
-        return int(eeg_json_dict['SamplingFrequency'] * eeg_json_dict['RecordingDuration'])
+        return int(eeg_json_dict['SamplingFrequency'] * eeg_json_dict['RecordingDuration'])
+    
+    def subject_participant_tsv(self, data_filepath):
+        '''Get participants_tsv info of a subject based on filepath'''
+        participants_tsv = pd.read_csv(self.get_bids_metadata_files(data_filepath, 'participants.tsv')[0], sep='\t')
+        # set 'participant_id' as index
+        participants_tsv.set_index('participant_id', inplace=True)
+        subject = f'sub-{self.subject(data_filepath)}'
+        return participants_tsv.loc[subject].to_dict()
+    
+    def eeg_json(self, data_filepath):
+        eeg_jsons = self.get_bids_metadata_files(data_filepath, 'eeg.json')
+        eeg_json_dict = self.merge_json_inheritance(eeg_jsons)
+        return eeg_json_dict
+    
+    def channel_tsv(self, data_filepath):
+        channels_tsv = pd.read_csv(self.get_bids_metadata_files(data_filepath, 'channels.tsv')[0], sep='\t')
+        channel_tsv = channels_tsv.to_dict()
+        # 'name' and 'type' now have a dictionary of index-value. Convert them to list
+        for list_field in ['name', 'type', 'units']:
+            channel_tsv[list_field] = list(channel_tsv[list_field].values())
+        return channel_tsv

eegdash/main.py

@@ -1,13 +1,19 @@
+from typing import List
 import pymongo
 from dotenv import load_dotenv
 import os
+from pathlib import Path
 import s3fs
 from joblib import Parallel, delayed
 import tempfile
 import mne
 import numpy as np
 import xarray as xr
-from .data_utils import BIDSDataset
+from .data_utils import BIDSDataset, EEGDashBaseRaw, EEGDashBaseDataset
+from braindecode.datasets import BaseDataset, BaseConcatDataset
+from collections import defaultdict
+from pymongo import MongoClient, InsertOne, UpdateOne, DeleteOne
+
 class EEGDash:
     AWS_BUCKET = 's3://openneuro.org'
     def __init__(self, 
@@ -31,22 +37,15 @@ class EEGDash:
         # convert to list using get_item on each element
         return [result for result in results]
 
-    def exist(self, schema_ref='eeg_signal', data_name=''):
+    def exist(self, data_name=''):
         query = {
-            "schema_ref": schema_ref,
             "data_name": data_name
         }
         sessions = self.find(query)
         return len(sessions) > 0
 
-    def add(self, record:dict):
-        input_record = self._validate_input(record)
-        print(input_record)
-        self.__collection.insert_one(input_record)
-
     def _validate_input(self, record:dict):
         input_types = {
-            'schema_ref': str,
             'data_name': str,
             'dataset': str,
             'bidspath': str,
@@ -61,7 +60,6 @@ class EEGDash:
             'channel_types': list,
             'channel_names': list,
         }
-        record['schema_ref'] = 'eeg_signal'
         if 'data_name' not in record:
             raise ValueError("Missing key: data_name")
         # check if args are in the keys and has correct type
@@ -117,21 +115,30 @@ class EEGDash:
         # extract openneuro path by finding the first occurrence of the dataset name in the filename and remove the path before that
         openneuro_path = dsnumber + bids_file.split(dsnumber)[1]
 
+        bids_dependencies_files = ['dataset_description.json', 'participants.tsv', 'events.tsv', 'events.json', 'eeg.json', 'electrodes.tsv', 'channels.tsv', 'coordsystem.json']
+        bidsdependencies = []
+        for extension in bids_dependencies_files:
+            dep_path = bids_dataset.get_bids_metadata_files(bids_file, extension)
+            dep_path = [str(bids_dataset.get_relative_bidspath(dep)) for dep in dep_path]
+
+            bidsdependencies.extend(dep_path)
+
+        participants_tsv = bids_dataset.subject_participant_tsv(bids_file)
+        eeg_json = bids_dataset.eeg_json(bids_file)
         attrs = {
-            'schema_ref': 'eeg_signal',
             'data_name': f'{bids_dataset.dataset}_{f}',
             'dataset': bids_dataset.dataset,
             'bidspath': openneuro_path,
             'subject': bids_dataset.subject(bids_file),
-            'nchans': bids_dataset.num_channels(bids_file),
-            'ntimes': bids_dataset.num_times(bids_file),
-            'channel_types': bids_dataset.channel_types(bids_file),
-            'channel_names': bids_dataset.channel_labels(bids_file),
             'task': bids_dataset.task(bids_file),
             'session': bids_dataset.session(bids_file),
             'run': bids_dataset.run(bids_file),
-            'sampling_frequency': bids_dataset.sfreq(bids_file), 
             'modality': 'EEG',
+            'nchans': bids_dataset.num_channels(bids_file),
+            'ntimes': bids_dataset.num_times(bids_file),
+            'participant_tsv': participants_tsv,
+            'eeg_json': eeg_json,
+            'bidsdependencies': bidsdependencies,
         }
 
         return attrs
@@ -148,27 +155,30 @@ class EEGDash:
             dataset=dataset,
             raw_format=raw_format,
         )
+        requests = []
         for bids_file in bids_dataset.get_files():
-            print('bids raw file', bids_file)
-
-            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
+            try:
+                data_id = f"{dataset}_{os.path.basename(bids_file)}"
 
-            if self.exist(data_name=signalstore_data_id):
-                if overwrite:
-                    eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                    print('updating record', eeg_attrs['data_name'])
-                    self.update(eeg_attrs)
+                if self.exist(data_name=data_id):
+                    if overwrite:
+                        eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                        requests.append(UpdateOne(self.update_request(eeg_attrs)))
                 else:
-                    print('data already exist and not overwriting. skipped')
-                    continue
-            else:
-                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                # Assume raw data already exists on Openneuro, recreating record only
-                print('adding record', eeg_attrs['data_name'])
-                self.add(eeg_attrs)
+                    eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                    requests.append(self.add_request(eeg_attrs))
+            except:
+                print('error adding record', bids_file)
 
-    def get_s3path(self, record):
-        return f"{self.AWS_BUCKET}/{record['bidspath']}"
+        print('Number of database requests', len(requests))
+
+        if requests:
+            result = self.__collection.bulk_write(requests, ordered=False)
+            print(f"Inserted: {result.inserted_count}")
+            print(f"Modified: {result.modified_count}")
+            print(f"Deleted: {result.deleted_count}")
+            print(f"Upserted: {result.upserted_count}")
+            print(f"Errors: {result.bulk_api_result.get('writeErrors', [])}")
 
     def get(self, query:dict):
         '''
@@ -185,11 +195,113 @@ class EEGDash:
             )
         return results
 
+    def add_request(self, record:dict):
+        return InsertOne(record)
+
+    def add(self, record:dict):
+        try:
+            # input_record = self._validate_input(record)
+            self.__collection.insert_one(record)
+        # silent failing
+        except ValueError as e:
+            print(f"Failed to validate record: {record['data_name']}")
+            print(e)
+        except: 
+            print(f"Error adding record: {record['data_name']}")
+
+    def update_request(self, record:dict):
+        return UpdateOne({'data_name': record['data_name']}, {'$set': record})
+
     def update(self, record:dict):
-        record['schema_ref'] = 'eeg_signal'
-        self.__collection.update_one({'schema_ref': record['schema_ref'], 'data_name': record['data_name']}, 
-                    {'$set': record}
-                )
+        try:
+            self.__collection.update_one({'data_name': record['data_name']}, {'$set': record})
+        except: # silent failure
+            print(f'Error updating record {record["data_name"]}')
+
+    def remove_field(self, record, field):
+        self.__collection.update_one({'data_name': record['data_name']}, {'$unset': {field: 1}})
+    
+    def remove_field_from_db(self, field):
+        self.__collection.update_many({}, {'$unset': {field: 1}})
+    
+
+class EEGDashDataset(BaseConcatDataset):
+    CACHE_DIR = '.eegdash_cache'
+    def __init__(
+        self,
+        query:dict=None,
+        data_dir:str | list =None,
+        dataset:str | list =None,
+        description_fields: list[str]=['subject', 'session', 'run', 'task', 'age', 'gender', 'sex'],
+        **kwargs
+    ):
+        if query:
+            datasets = self.find_datasets(query, description_fields, **kwargs)
+        elif data_dir:
+            if type(data_dir) == str:
+                datasets = self.load_bids_dataset(dataset, data_dir, description_fields)
+            else:
+                assert len(data_dir) == len(dataset), 'Number of datasets and their directories must match' 
+                datasets = []
+                for i in range(len(data_dir)):
+                    datasets.extend(self.load_bids_dataset(dataset[i], data_dir[i], description_fields))
+        # convert to list using get_item on each element
+        super().__init__(datasets)
+    
+    def find_key_in_nested_dict(self, data, target_key):
+        if isinstance(data, dict):
+            if target_key in data:
+                return data[target_key]
+            for value in data.values():
+                result = self.find_key_in_nested_dict(value, target_key)
+                if result is not None:
+                    return result
+        return None
+
+    def find_datasets(self, query:dict, description_fields:list[str], **kwargs):
+        eegdashObj = EEGDash()
+        datasets = []
+        for record in eegdashObj.find(query):
+            description = {}
+            for field in description_fields:
+                value = self.find_key_in_nested_dict(record, field)
+                if value:
+                    description[field] = value
+            datasets.append(EEGDashBaseDataset(record, self.CACHE_DIR, description=description, **kwargs))
+        return datasets
+
+    def load_bids_dataset(self, dataset, data_dir, description_fields: list[str],raw_format='eeglab', **kwargs):
+        '''
+        '''
+        def get_base_dataset_from_bids_file(bids_dataset, bids_file):
+            record = eegdashObj.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+            description = {}
+            for field in description_fields:
+                value = self.find_key_in_nested_dict(record, field)
+                if value:
+                    description[field] = value
+            return EEGDashBaseDataset(record, self.CACHE_DIR, description=description, **kwargs)
+
+        bids_dataset = BIDSDataset(
+            data_dir=data_dir,
+            dataset=dataset,
+            raw_format=raw_format,
+        )
+        eegdashObj = EEGDash()
+        datasets = Parallel(n_jobs=-1, prefer="threads", verbose=1)(
+                delayed(get_base_dataset_from_bids_file)(bids_dataset, bids_file) for bids_file in bids_dataset.get_files()
+            )
+        # datasets = []
+        # for bids_file in bids_dataset.get_files():
+        #     record = eegdashObj.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+        #     description = {}
+        #     for field in description_fields:
+        #         value = self.find_key_in_nested_dict(record, field)
+        #         if value:
+        #             description[field] = value
+        #     datasets.append(EEGDashBaseDataset(record, self.CACHE_DIR, description=description, **kwargs))
+        return datasets
+
 def main():
     eegdash = EEGDash()
     record = eegdash.find({'dataset': 'ds005511', 'subject': 'NDARUF236HM7'})

eegdash-0.0.8.dist-info/METADATA

@@ -0,0 +1,157 @@
+Metadata-Version: 2.4
+Name: eegdash
+Version: 0.0.8
+Summary: EEG data for machine learning
+Author-email: Young Truong <dt.young112@gmail.com>, Arnaud Delorme <adelorme@gmail.com>
+License: GNU General Public License
+        
+        Copyright (C) 2024-2025
+        
+        Young Truong, UCSD, dt.young112@gmail.com
+        Arnaud Delorme, UCSD, adelorme@ucsd.edu
+        
+        This program is free software; you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation; either version 2 of the License, or
+        (at your option) any later version.
+        
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+        
+        You should have received a copy of the GNU General Public License
+        along with this program; if not, write to the Free Software
+        Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1.07  USA
+        
+Project-URL: Homepage, https://github.com/sccn/EEG-Dash-Data
+Project-URL: Issues, https://github.com/sccn/EEG-Dash-Data/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: xarray
+Requires-Dist: python-dotenv
+Requires-Dist: s3fs
+Requires-Dist: mne
+Requires-Dist: pynwb
+Requires-Dist: h5py
+Requires-Dist: pymongo
+Requires-Dist: joblib
+Requires-Dist: braindecode
+Requires-Dist: mne-bids
+Dynamic: license-file
+
+# EEG-Dash
+To leverage recent and ongoing advancements in large-scale computational methods and to ensure the preservation of scientific data generated from publicly funded research, the EEG-DaSh data archive will create a data-sharing resource for MEEG (EEG, MEG) data contributed by collaborators for machine learning (ML) and deep learning (DL) applications. 
+
+## Data source
+The data in EEG-DaSh originates from a collaboration involving 25 laboratories, encompassing 27,053 participants. This extensive collection includes MEEG data, which is a combination of EEG and MEG signals. The data is sourced from various studies conducted by these labs, involving both healthy subjects and clinical populations with conditions such as ADHD, depression, schizophrenia, dementia, autism, and psychosis. Additionally, data spans different mental states like sleep, meditation, and cognitive tasks. In addition, EEG-DaSh will incorporate a subset of the data converted from NEMAR, which includes 330 MEEG BIDS-formatted datasets, further expanding the archive with well-curated, standardized neuroelectromagnetic data.
+
+## Available data
+
+The following datasets are currently available on EEGDash.
+
+| DatasetID | Participants | Files | Sessions | Population | Channels | Is 10-20? | Modality | Size |
+|---|---|---|---|---|---|---|---|---|
+| [ds002181](https://nemar.org/dataexplorer/detail?dataset_id=ds002181) | 20 | 949 | 1 | Healthy | 63 | 10-20 | Visual | 0.163 GB |
+| [ds002578](https://nemar.org/dataexplorer/detail?dataset_id=ds002578) | 2 | 22 | 1 | Healthy | 256 | 10-20 | Visual | 0.001 TB |
+| [ds002680](https://nemar.org/dataexplorer/detail?dataset_id=ds002680) | 14 | 4977 | 2 | Healthy | 0 | 10-20 | Visual | 0.01 TB |
+| [ds002691](https://nemar.org/dataexplorer/detail?dataset_id=ds002691) | 20 | 146 | 1 | Healthy | 32 | other | Visual | 0.001 TB |
+| [ds002718](https://nemar.org/dataexplorer/detail?dataset_id=ds002718) | 18 | 582 | 1 | Healthy | 70 | other | Visual | 0.005 TB |
+| [ds003061](https://nemar.org/dataexplorer/detail?dataset_id=ds003061) | 13 | 282 | 1 | Not specified | 64 | 10-20 | Auditory | 0.002 TB |
+| [ds003690](https://nemar.org/dataexplorer/detail?dataset_id=ds003690) | 75 | 2630 | 1 | Healthy | 61 | 10-20 | Auditory | 0.023 TB |
+| [ds003805](https://nemar.org/dataexplorer/detail?dataset_id=ds003805) | 1 | 10 | 1 | Healthy | 19 | 10-20 | Multisensory | 0 TB |
+| [ds003838](https://nemar.org/dataexplorer/detail?dataset_id=ds003838) | 65 | 947 | 1 | Healthy | 63 | 10-20 | Auditory | 100.2 GB |
+| [ds004010](https://nemar.org/dataexplorer/detail?dataset_id=ds004010) | 24 | 102 | 1 | Healthy | 64 | other | Multisensory | 0.025 TB |
+| [ds004040](https://nemar.org/dataexplorer/detail?dataset_id=ds004040) | 13 | 160 | 2 | Healthy | 64 | 10-20 | Auditory | 0.012 TB |
+| [ds004350](https://nemar.org/dataexplorer/detail?dataset_id=ds004350) | 24 | 960 | 2 | Healthy | 64 | other | Visual | 0.023 TB |
+| [ds004362](https://nemar.org/dataexplorer/detail?dataset_id=ds004362) | 109 | 9162 | 1 | Healthy | 64 | 10-20 | Visual | 0.008 TB |
+| [ds004504](https://nemar.org/dataexplorer/detail?dataset_id=ds004504) | 88 | 269 | 1 | Dementia | 19 | 10-20 | Resting State | 2.6 GB |
+| [ds004554](https://nemar.org/dataexplorer/detail?dataset_id=ds004554) | 16 | 101 | 1 | Healthy | 99 | 10-20 | Visual | 0.009 TB |
+| [ds004635](https://nemar.org/dataexplorer/detail?dataset_id=ds004635) | 48 | 292 | 1 | Healthy | 129 | other | Multisensory | 26.1 GB |
+| [ds004657](https://nemar.org/dataexplorer/detail?dataset_id=ds004657) | 24 | 838 | 6 | Not specified | 64 | 10-20 | Motor | 43.1 GB |
+| [ds004660](https://nemar.org/dataexplorer/detail?dataset_id=ds004660) | 21 | 299 | 1 | Healthy | 32 | 10-20 | Multisensory | 7.2 GB |
+| [ds004661](https://nemar.org/dataexplorer/detail?dataset_id=ds004661) | 17 | 90 | 1 | Not specified | 64 | 10-20 | Multisensory | 1.4 GB |
+| [ds004745](https://nemar.org/dataexplorer/detail?dataset_id=ds004745) | 52 | 762 | 1 | Healthy | 64 | ? | Auditory | 0 TB |
+| [ds004785](https://nemar.org/dataexplorer/detail?dataset_id=ds004785) | 17 | 74 | 1 | Healthy | 32 | ? | Motor | 0 TB |
+| [ds004841](https://nemar.org/dataexplorer/detail?dataset_id=ds004841) | 20 | 1034 | 2 | Not specified | 64 | 10-20 | Multisensory | 7.3 GB |
+| [ds004842](https://nemar.org/dataexplorer/detail?dataset_id=ds004842) | 14 | 719 | 2 | Not specified | 64 | ? | Multisensory | 5.2 GB |
+| [ds004843](https://nemar.org/dataexplorer/detail?dataset_id=ds004843) | 14 | 649 | 1 | Not specified | 64 | ? | Visual | 7.7 GB |
+| [ds004844](https://nemar.org/dataexplorer/detail?dataset_id=ds004844) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 22.3 GB |
+| [ds004849](https://nemar.org/dataexplorer/detail?dataset_id=ds004849) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds004850](https://nemar.org/dataexplorer/detail?dataset_id=ds004850) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds004851](https://nemar.org/dataexplorer/detail?dataset_id=ds004851) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds004852](https://nemar.org/dataexplorer/detail?dataset_id=ds004852) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds004853](https://nemar.org/dataexplorer/detail?dataset_id=ds004853) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds004854](https://nemar.org/dataexplorer/detail?dataset_id=ds004854) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds004855](https://nemar.org/dataexplorer/detail?dataset_id=ds004855) | 17 | 481 | 4 | Not specified | 64 | ? | Multisensory | 0.077 GB |
+| [ds005034](https://nemar.org/dataexplorer/detail?dataset_id=ds005034) | 25 | 406 | 2 | Healthy | 129 | ? | Visual | 61.4 GB |
+| [ds005079](https://nemar.org/dataexplorer/detail?dataset_id=ds005079) | 1 | 210 | 12 | Healthy | 64 | ? | Multisensory | 1.7 GB |
+| [ds005342](https://nemar.org/dataexplorer/detail?dataset_id=ds005342) | 32 | 134 | 1 | Healthy | 17 | ? | Visual | 2 GB |
+| [ds005410](https://nemar.org/dataexplorer/detail?dataset_id=ds005410) | 81 | 492 | 1 | Healthy | 63 | ? | ? | 19.8 GB |
+| [ds005505](https://nemar.org/dataexplorer/detail?dataset_id=ds005505) | 136 | 5393 | 1 | Healthy | 129 | other | Visual | 103 GB |
+| [ds005506](https://nemar.org/dataexplorer/detail?dataset_id=ds005506) | 150 | 5645 | 1 | Healthy | 129 | other | Visual | 112 GB |
+| [ds005507](https://nemar.org/dataexplorer/detail?dataset_id=ds005507) | 184 | 7273 | 1 | Healthy | 129 | other | Visual | 140 GB |
+| [ds005508](https://nemar.org/dataexplorer/detail?dataset_id=ds005508) | 324 | 13393 | 1 | Healthy | 129 | other | Visual | 230 GB |
+| [ds005509](https://nemar.org/dataexplorer/detail?dataset_id=ds005509) | 330 | 19980 | 1 | Healthy | 129 | other | Visual | 224 GB |
+| [ds005510](https://nemar.org/dataexplorer/detail?dataset_id=ds005510) | 135 | 4933 | 1 | Healthy | 129 | other | Visual | 91 GB |
+| [ds005511](https://nemar.org/dataexplorer/detail?dataset_id=ds005511) | 381 | 18604 | 1 | Healthy | 129 | other | Visual | 245 GB |
+| [ds005512](https://nemar.org/dataexplorer/detail?dataset_id=ds005512) | 257 | 9305 | 1 | Healthy | 129 | other | Visual | 157 GB |
+| [ds005514](https://nemar.org/dataexplorer/detail?dataset_id=ds005514) | 295 | 11565 | 1 | Healthy | 129 | other | Visual | 185 GB |
+| [ds005672](https://nemar.org/dataexplorer/detail?dataset_id=ds005672) | 3 | 18 | 1 | Healthy | 64 | 10-20 | Visual | 4.2 GB |
+| [ds005697](https://nemar.org/dataexplorer/detail?dataset_id=ds005697) | 52 | 210 | 1 | Healthy | 64 | 10-20 | Visual | 67 GB |
+| [ds005787](https://nemar.org/dataexplorer/detail?dataset_id=ds005787) | 30 | ? | 4 | Healthy | 64 | 10-20 | Visual | 185 GB |
+
+## Data format
+EEGDash queries return a **Pytorch Dataset** formatted to facilitate machine learning (ML) and deep learning (DL) applications. PyTorch Datasets are the best format for EEGDash queries because they provide an efficient, scalable, and flexible structure for machine learning (ML) and deep learning (DL) applications. They allow seamless integration with PyTorch’s DataLoader, enabling efficient batching, shuffling, and parallel data loading, which is essential for training deep learning models on large EEG datasets.
+
+## Data preprocessing
+EEGDash datasets are processed using the popular [BrainDecode](https://braindecode.org/stable/index.html) library. In fact, EEGDash datasets are BrainDecode datasets, which are themselves PyTorch datasets. This means that any preprocessing possible on BrainDecode datasets is also possible on EEGDash datasets. Refer to [BrainDecode](https://braindecode.org/stable/index.html) tutorials for guidance on preprocessing EEG data.
+
+## EEG-Dash usage
+
+### Install
+Use your preferred Python environment manager with Python > 3.9 to install the package.
+* To install the eegdash package, use the following temporary command (a direct pip install eegdash option will be available soon): `pip install -i https://test.pypi.org/simple/eegdash`
+* To verify the installation, start a Python session and type: `from eegdash import EEGDash`
+
+### Data access
+
+To use the data from a single subject, enter:
+
+```python
+from eegdash import EEGDashDataset
+ds_NDARDB033FW5 = EEGDashDataset({'dataset': 'ds005514', 'task': 'RestingState', 'subject': 'NDARDB033FW5'})
+```
+
+This will search and download the metadata for the task **RestingState** for subject **NDARDB033FW5** in BIDS dataset **ds005514**. The actual data will not be downloaded at this stage. Following standard practice, data is only downloaded once it is processed. The **ds_NDARDB033FW5** object is a fully functional BrainDecode dataset, which is itself a PyTorch dataset. This [tutorial](https://github.com/sccn/EEGDash/blob/develop/notebooks/tutorial_eoec.ipynb) shows how to preprocess the EEG data, extracting portions of the data containing eyes-open and eyes-closed segments, then perform eyes-open vs. eyes-closed classification using a (shallow) deep-learning model. 
+
+To use the data from multiple subjects, enter:
+
+```python
+from eegdash import EEGDashDataset
+ds_ds005505rest = EEGDashDataset({'dataset': 'ds005505', 'task': 'RestingState'}, target_name='sex')
+```
+
+This will search and download the metadata for the task 'RestingState' for all subjects in BIDS dataset 'ds005505' (a total of 136). As above, the actual data will not be downloaded at this stage so this command is quick to execute. Also, the target class for each subject is assigned using the target_name parameter. This means that this object is ready to be directly fed to a deep learning model, although the [tutorial script](https://github.com/sccn/EEGDash/blob/develop/notebooks/tutorial_sex_classification.ipynb) performs minimal processing on it, prior to training a deep-learning model. Because 14 gigabytes of data are downloaded, this tutorial takes about 10 minutes to execute.
+
+### Automatic caching
+
+EEGDash automatically caches the downloaded data in the .eegdash_cache folder of the current directory from which the script is called. This means that if you run the tutorial [scripts](https://github.com/sccn/EEGDash/tree/develop/notebooks), the data will only be downloaded the first time the script is executed.
+
+## Education -- Coming soon...
+
+We organize workshops and educational events to foster cross-cultural education and student training, offering both online and in-person opportunities in collaboration with US and Israeli partners. Events for 2025 will be announced via the EEGLABNEWS mailing list. Be sure to [subscribe](https://sccn.ucsd.edu/mailman/listinfo/eeglabnews).
+
+## About EEG-DaSh
+
+EEG-DaSh is a collaborative initiative between the United States and Israel, supported by the National Science Foundation (NSF). The partnership brings together experts from the Swartz Center for Computational Neuroscience (SCCN) at the University of California San Diego (UCSD) and Ben-Gurion University (BGU) in Israel. 
+
+![Screenshot 2024-10-03 at 09 14 06](https://github.com/user-attachments/assets/327639d3-c3b4-46b1-9335-37803209b0d3)
+
+
+
+

eegdash-0.0.8.dist-info/RECORD

@@ -0,0 +1,8 @@
+eegdash/__init__.py,sha256=DrliW5AazWcHJBznrmrS_YF8n8K48csOzfWWIvB6Esw,41
+eegdash/data_utils.py,sha256=1ETB2rW-6HvXdli75pHW9yz3xjEZN7IqApKHn9XKHv4,19205
+eegdash/main.py,sha256=QfFLzbs8iXmaJj4x_ylhFuEAuOCEKIKV6h_a__XPZ6Y,12048
+eegdash-0.0.8.dist-info/licenses/LICENSE,sha256=Xafu48R-h_kyaNj2tuhfgdEv9_ovciktjUEgRRwMZ6w,812
+eegdash-0.0.8.dist-info/METADATA,sha256=X3AsZ23uPQY71alITDp4cOHOvgD7QmlGVagiKHj4cjE,13881
+eegdash-0.0.8.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
+eegdash-0.0.8.dist-info/top_level.txt,sha256=zavO69HQ6MyZM0aQMR2zUS6TAFc7bnN5GEpDpOpFZzU,8
+eegdash-0.0.8.dist-info/RECORD,,

{eegdash-0.0.6.dist-info → eegdash-0.0.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: setuptools (78.0.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

eegdash-0.0.6.dist-info/METADATA

@@ -1,147 +0,0 @@
-Metadata-Version: 2.2
-Name: eegdash
-Version: 0.0.6
-Summary: EEG data for machine learning
-Author-email: Young Truong <dt.young112@gmail.com>, Arnaud Delorme <adelorme@gmail.com>
-License: GNU General Public License
-        
-        Copyright (C) 2024-2025
-        
-        Young Truong, UCSD, dt.young112@gmail.com
-        Arnaud Delorme, UCSD, adelorme@ucsd.edu
-        
-        This program is free software; you can redistribute it and/or modify
-        it under the terms of the GNU General Public License as published by
-        the Free Software Foundation; either version 2 of the License, or
-        (at your option) any later version.
-        
-        This program is distributed in the hope that it will be useful,
-        but WITHOUT ANY WARRANTY; without even the implied warranty of
-        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-        GNU General Public License for more details.
-        
-        You should have received a copy of the GNU General Public License
-        along with this program; if not, write to the Free Software
-        Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1.07  USA
-        
-Project-URL: Homepage, https://github.com/sccn/EEG-Dash-Data
-Project-URL: Issues, https://github.com/sccn/EEG-Dash-Data/issues
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: zarr==2.18.3
-Requires-Dist: xarray
-Requires-Dist: python-dotenv
-Requires-Dist: s3fs
-Requires-Dist: mne
-Requires-Dist: pynwb
-Requires-Dist: h5py
-Requires-Dist: pymongo
-Requires-Dist: joblib
-
-# EEG-Dash
-To leverage recent and ongoing advancements in large-scale computational methods and to ensure the preservation of scientific data generated from publicly funded research, the EEG-DaSh data archive will create a data-sharing resource for MEEG (EEG, MEG) data contributed by collaborators for machine learning (ML) and deep learning (DL) applications. 
-
-## Data source
-The data in EEG-DaSh originates from a collaboration involving 25 laboratories, encompassing 27,053 participants. This extensive collection includes MEEG data, which is a combination of EEG and MEG signals. The data is sourced from various studies conducted by these labs, involving both healthy subjects and clinical populations with conditions such as ADHD, depression, schizophrenia, dementia, autism, and psychosis. Additionally, data spans different mental states like sleep, meditation, and cognitive tasks. In addition, EEG-DaSh will also incorporate a subset of the data converted from NEMAR, which includes 330 MEEG BIDS-formatted datasets, further expanding the archive with well-curated, standardized neuroelectromagnetic data.
-
-## Datasets available
-
-There are currently only two datasets made available for testing purposes. 
-
-| Dataset ID | Description                                                                                 | Participants | Channels        | Task            | NEMAR Link                                                                                      |
-|------------|---------------------------------------------------------------------------------------------|--------------|-----------------|-----------------|------------------------------------------------------------------------------------------------|
-| ds002718   | EEG dataset focused on face processing with MRI for source localization                     | 18           | 70 EEG, 2 EOG   | FaceRecognition | [NEMAR ds002718](https://nemar.org/dataexplorer/detail?dataset_id=ds002718)                    |
-| ds004745   | 8-Channel SSVEP EEG dataset with trials including voluntary movements to introduce artifacts | 6            | 8 EEG           | SSVEP tasks     | [NEMAR ds004745](https://nemar.org/dataexplorer/detail?dataset_id=ds004745)                    |
-
-
-
-## Data formatting
-The data in EEG-DaSh is formatted to facilitate machine learning (ML) and deep learning (DL) applications by using a simplified structure commonly adopted by these communities. This will involve converting raw MEEG data into a matrix format, where samples (e.g., individual EEG or MEG recordings) are represented by rows, and values (such as time or channel data) are represented by columns. The data is also divided into training and testing sets, with 80% of the data allocated for training and 20% for testing, ensuring a balanced representation of relevant labels across sets. Hierarchical Event Descriptor (HED) tags will be used to annotate labels, which will be stored in a text table, and detailed metadata, including dataset origins and methods. This formatting process will ensure that data is ready for ML/DL models, allowing for efficient training and testing of algorithms while preserving data integrity and reusability.
-
-![Screenshot 2024-10-03 at 09 07 28](https://github.com/user-attachments/assets/b30a79bb-0d94-410a-843c-44c3fcea01fc)
-
-## Data access
-The data in EEG-DaSh is formatted to facilitate machine learning (ML) and deep learning (DL) applications by using a simplified structure commonly adopted by these communities. This will involve converting raw MEEG data into a matrix format, where samples (e.g., individual EEG or MEG recordings) are represented by rows, and values (such as time or channel data) are represented by columns. The data is also divided into training and testing sets, with 80% of the data allocated for training and 20% for testing, ensuring a balanced representation of relevant labels across sets. Hierarchical Event Descriptor (HED) tags will be used to annotate labels, which will be stored in a text table, and detailed metadata, including dataset origins and methods. This formatting process will ensure that data is ready for ML/DL models, allowing for efficient training and testing of algorithms while preserving data integrity and reusability.
-
-The data in EEG-DaSh is accessed through Python and MATLAB libraries specifically designed for this platform. These libraries will use objects compatible with deep learning data storage formats in each language, such as <i>Torchvision.dataset</i> in Python and <i>DataStore</i> in MATLAB. Users can dynamically fetch data from the EEG-DaSh server which is then cached locally. 
-
-### Install
-Use your preferred Python environment manager with Python > 3.9 to install the package. Here we show example using Conda environment with Python 3.11.5:
-* Create a new environment Python 3.11.5 -> `conda create --name eegdash python=3.11.5`
-* Switch to the right environment -> `conda activate eegdash`
-* Install dependencies (this is a temporary link that will be updated soon) -> `pip install -r https://raw.githubusercontent.com/sccn/EEG-Dash-Data/refs/heads/develop/requirements.txt`
-* Install _eegdash_ package (this is a temporary link that will be updated soon) -> `pip install -i https://test.pypi.org/simple/ eegdash`
-* Check installation. Start a Python session and type `from eegdash import EEGDash`
-
-### Python data access
-
-To create a local object for accessing the database, use the following code:
-
-```python
-from eegdash import EEGDash
-EEGDashInstance = EEGDash()
-```
-
-Once the object is instantiated, it can be utilized to search datasets. Providing an empty parameter will search the entire database and return all available datasets.
-
-```python
-EEGDashInstance.find({})
-```
-A list of dataset is returned.
-
-```python
-[{'schema_ref': 'eeg_signal',
-  'data_name': 'ds004745_sub-001_task-unnamed_eeg.set',
-  'dataset': 'ds004745',
-  'subject': '001',
-  'task': 'unnamed',
-  'session': '',
-  'run': '',
-  'modality': 'EEG',
-  'sampling_frequency': 1000,
-  'version_timestamp': 0,
-  'has_file': True,
-  'time_of_save': datetime.datetime(2024, 10, 25, 14, 11, 48, 843593, tzinfo=datetime.timezone.utc),
-  'time_of_removal': None}, ...
-
-```
-
-Additionally, users can search for a specific dataset by specifying criteria.
-
-```python
-EEGDashInstance.find({'task': 'FaceRecognition'})
-```
-
-After locating the desired dataset or data record, users can download it locally by executing the following command. This will return an xArray Python object.
-
-```python
-XArrayData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})
-```
-
-Optionally, this is how you may access the raw data for the first record. This will return an numpy array.
-
-```python
-npData = EEGDashInstance.get({'task': 'FaceRecognition', 'subject': '019'})[0].values
-```
-
-## Example use
-
-This [example](tests/eegdash.ipynb) demonstrates the full workflow from data retrieval with `EEGDash` to model definition, data handling, and training in PyTorch.
-
-## Education - Coming soon...
-
-We organize workshops and educational events to foster cross-cultural education and student training, offering both online and in-person opportunities in collaboration with US and Israeli partners. There is no event planned for 2024. Events for 2025 will be advertised on the EEGLABNEWS mailing list so make sure to [subscribe](https://sccn.ucsd.edu/mailman/listinfo/eeglabnews).
-
-## About EEG-DaSh
-
-EEG-DaSh is a collaborative initiative between the United States and Israel, supported by the National Science Foundation (NSF). The partnership brings together experts from the Swartz Center for Computational Neuroscience (SCCN) at the University of California San Diego (UCSD) and Ben-Gurion University (BGU) in Israel. 
-
-![Screenshot 2024-10-03 at 09 14 06](https://github.com/user-attachments/assets/327639d3-c3b4-46b1-9335-37803209b0d3)
-
-
-
-

eegdash-0.0.6.dist-info/RECORD

@@ -1,8 +0,0 @@
-eegdash/__init__.py,sha256=PjBBYCX47NLQxybOvz0WjcfFKGI8F3m1BBJxFhMJ6eA,25
-eegdash/data_utils.py,sha256=vzMGVp4PBWyRF8tbYNqkJs0QnUd5CzvmJUkpPfxdJh8,13491
-eegdash/main.py,sha256=fFZHHdVYNLqKr2X_NDB0XXla7A2QlHexgI9AD79_niY,7217
-eegdash-0.0.6.dist-info/LICENSE,sha256=Xafu48R-h_kyaNj2tuhfgdEv9_ovciktjUEgRRwMZ6w,812
-eegdash-0.0.6.dist-info/METADATA,sha256=halHsdTDj4by5BMtg4FXsCPMKnjoKPmrOz3FOXO_RPw,9609
-eegdash-0.0.6.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-eegdash-0.0.6.dist-info/top_level.txt,sha256=zavO69HQ6MyZM0aQMR2zUS6TAFc7bnN5GEpDpOpFZzU,8
-eegdash-0.0.6.dist-info/RECORD,,