eegdash 0.0.7__py3-none-any.whl → 0.0.9__py3-none-any.whl

eegdash/features/serialization.py

@@ -0,0 +1,87 @@
+"""
+Convenience functions for storing and loading of features datasets.
+
+see also: https://github.com/braindecode/braindecode//blob/master/braindecode/datautil/serialization.py#L165-L229
+"""
+
+import json
+from pathlib import Path
+
+import pandas as pd
+from joblib import Parallel, delayed
+
+from mne.io import read_info
+from braindecode.datautil.serialization import _load_kwargs_json
+
+from .datasets import (
+    FeaturesDataset,
+    FeaturesConcatDataset,
+)
+
+
+def load_features_concat_dataset(path, ids_to_load=None, n_jobs=1):
+    """Load a stored FeaturesConcatDataset of FeaturesDatasets from files.
+
+    Parameters
+    ----------
+    path: str | pathlib.Path
+        Path to the directory of the .fif / -epo.fif and .json files.
+    ids_to_load: list of int | None
+        Ids of specific files to load.
+    n_jobs: int
+        Number of jobs to be used to read files in parallel.
+
+    Returns
+    -------
+    concat_dataset: FeaturesConcatDataset of FeaturesDatasets
+    """
+    # Make sure we always work with a pathlib.Path
+    path = Path(path)
+
+    # else we have a dataset saved in the new way with subdirectories in path
+    # for every dataset with description.json and -feat.parquet,
+    # target_name.json, raw_preproc_kwargs.json, window_kwargs.json,
+    # window_preproc_kwargs.json, features_kwargs.json
+    if ids_to_load is None:
+        ids_to_load = [p.name for p in path.iterdir()]
+        ids_to_load = sorted(ids_to_load, key=lambda i: int(i))
+    ids_to_load = [str(i) for i in ids_to_load]
+
+    datasets = Parallel(n_jobs)(delayed(_load_parallel)(path, i) for i in ids_to_load)
+    return FeaturesConcatDataset(datasets)
+
+
+def _load_parallel(path, i):
+    sub_dir = path / i
+
+    parquet_name_pattern = "{}-feat.parquet"
+    parquet_file_name = parquet_name_pattern.format(i)
+    parquet_file_path = sub_dir / parquet_file_name
+
+    features = pd.read_parquet(parquet_file_path)
+
+    description_file_path = sub_dir / "description.json"
+    description = pd.read_json(description_file_path, typ="series")
+
+    raw_info_file_path = sub_dir / "raw-info.fif"
+    raw_info = None
+    if raw_info_file_path.exists():
+        raw_info = read_info(raw_info_file_path)
+
+    raw_preproc_kwargs = _load_kwargs_json("raw_preproc_kwargs", sub_dir)
+    window_kwargs = _load_kwargs_json("window_kwargs", sub_dir)
+    window_preproc_kwargs = _load_kwargs_json("window_preproc_kwargs", sub_dir)
+    features_kwargs = _load_kwargs_json("features_kwargs", sub_dir)
+    metadata = pd.read_pickle(path / i / "metadata_df.pkl")
+
+    dataset = FeaturesDataset(
+        features,
+        metadata=metadata,
+        description=description,
+        raw_info=raw_info,
+        raw_preproc_kwargs=raw_preproc_kwargs,
+        window_kwargs=window_kwargs,
+        window_preproc_kwargs=window_preproc_kwargs,
+        features_kwargs=features_kwargs,
+    )
+    return dataset

eegdash/features/utils.py

@@ -0,0 +1,114 @@
+from typing import Dict, List
+from collections.abc import Callable
+import copy
+import numpy as np
+import pandas as pd
+from joblib import Parallel, delayed
+from tqdm import tqdm
+from torch.utils.data import DataLoader
+from braindecode.datasets.base import (
+    EEGWindowsDataset,
+    WindowsDataset,
+    BaseConcatDataset,
+)
+
+from .datasets import FeaturesDataset, FeaturesConcatDataset
+from .extractors import FeatureExtractor
+
+
+def _extract_features_from_windowsdataset(
+    win_ds: EEGWindowsDataset | WindowsDataset,
+    feature_extractor: FeatureExtractor,
+    batch_size: int = 512,
+):
+    metadata = win_ds.metadata
+    if not win_ds.targets_from == "metadata":
+        metadata = copy.deepcopy(metadata)
+        metadata["orig_index"] = metadata.index
+        metadata.set_index(
+            ["i_window_in_trial", "i_start_in_trial", "i_stop_in_trial"],
+            drop=False,
+            inplace=True,
+        )
+    win_dl = DataLoader(win_ds, batch_size=batch_size, shuffle=False, drop_last=False)
+    features_dict = dict()
+    ch_names = win_ds.raw.ch_names
+    for X, y, crop_inds in win_dl:
+        X = X.numpy()
+        if hasattr(y, "tolist"):
+            y = y.tolist()
+        win_dict = dict()
+        win_dict.update(
+            feature_extractor(X, _batch_size=X.shape[0], _ch_names=ch_names)
+        )
+        if not win_ds.targets_from == "metadata":
+            metadata.loc[crop_inds, "target"] = y
+        for k, v in win_dict.items():
+            if k not in features_dict:
+                features_dict[k] = []
+            features_dict[k].extend(v)
+    features_df = pd.DataFrame(features_dict)
+    if not win_ds.targets_from == "metadata":
+        metadata.set_index("orig_index", drop=False, inplace=True)
+        metadata.reset_index(drop=True, inplace=True)
+        metadata.drop("orig_index", axis=1, inplace=True)
+
+    # FUTURE: truely support WindowsDataset objects
+    return FeaturesDataset(
+        features_df,
+        metadata=metadata,
+        description=win_ds.description,
+        raw_info=win_ds.raw.info,
+        raw_preproc_kwargs=win_ds.raw_preproc_kwargs,
+        window_kwargs=win_ds.window_kwargs,
+        features_kwargs=feature_extractor.features_kwargs,
+    )
+
+
+def extract_features(
+    concat_dataset: BaseConcatDataset,
+    features: FeatureExtractor | Dict[str, Callable] | List[Callable],
+    *,
+    batch_size: int = 512,
+    n_jobs: int = 1,
+):
+    if isinstance(features, list):
+        features = dict(enumerate(features))
+    if not isinstance(features, FeatureExtractor):
+        features = FeatureExtractor(features)
+    feature_ds_list = list(
+        tqdm(
+            Parallel(n_jobs=n_jobs, return_as="generator")(
+                delayed(_extract_features_from_windowsdataset)(
+                    win_ds, features, batch_size
+                )
+                for win_ds in concat_dataset.datasets
+            ),
+            total=len(concat_dataset.datasets),
+            desc="Extracting features",
+        )
+    )
+    return FeaturesConcatDataset(feature_ds_list)
+
+
+def fit_feature_extractors(
+    concat_dataset: BaseConcatDataset,
+    features: FeatureExtractor | Dict[str, Callable] | List[Callable],
+    batch_size: int = 8192,
+):
+    if isinstance(features, list):
+        features = dict(enumerate(features))
+    if not isinstance(features, FeatureExtractor):
+        features = FeatureExtractor(features)
+    if not features._is_fitable:
+        return features
+    features.clear()
+    concat_dl = DataLoader(
+        concat_dataset, batch_size=batch_size, shuffle=False, drop_last=False
+    )
+    for X, y, _ in tqdm(
+        concat_dl, total=len(concat_dl), desc="Fitting feature extractors"
+    ):
+        features.partial_fit(X.numpy(), y=np.array(y))
+    features.fit()
+    return features

eegdash/main.py

@@ -1,17 +1,30 @@
 import pymongo
 from dotenv import load_dotenv
 import os
+from pathlib import Path
 import s3fs
 from joblib import Parallel, delayed
+import json
 import tempfile
 import mne
 import numpy as np
 import xarray as xr
-from .data_utils import BIDSDataset
+from .data_utils import EEGBIDSDataset, EEGDashBaseRaw, EEGDashBaseDataset
+from .data_config import config as data_config
+from braindecode.datasets import BaseDataset, BaseConcatDataset
+from collections import defaultdict
+from pymongo import MongoClient, InsertOne, UpdateOne, DeleteOne
+
 class EEGDash:
     AWS_BUCKET = 's3://openneuro.org'
     def __init__(self, 
                  is_public=True):
+        # Load config file
+        # config_path = Path(__file__).parent / 'config.json'
+        # with open(config_path, 'r') as f:
+        #     self.config = json.load(f)
+
+        self.config = data_config
         if is_public:
             DB_CONNECTION_STRING="mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0"
         else:
@@ -31,22 +44,14 @@ class EEGDash:
         # convert to list using get_item on each element
         return [result for result in results]
 
-    def exist(self, schema_ref='eeg_signal', data_name=''):
-        query = {
-            "schema_ref": schema_ref,
-            "data_name": data_name
-        }
+    def exist(self, query:dict):
+        accepted_query_fields = ['data_name', 'dataset']
+        assert all(field in accepted_query_fields for field in query.keys())
         sessions = self.find(query)
         return len(sessions) > 0
 
-    def add(self, record:dict):
-        input_record = self._validate_input(record)
-        print(input_record)
-        self.__collection.insert_one(input_record)
-
     def _validate_input(self, record:dict):
         input_types = {
-            'schema_ref': str,
             'data_name': str,
             'dataset': str,
             'bidspath': str,
@@ -61,7 +66,6 @@ class EEGDash:
             'channel_types': list,
             'channel_names': list,
         }
-        record['schema_ref'] = 'eeg_signal'
         if 'data_name' not in record:
             raise ValueError("Missing key: data_name")
         # check if args are in the keys and has correct type
@@ -106,69 +110,126 @@ class EEGDash:
         )
         return eeg_xarray
 
-    def load_eeg_attrs_from_bids_file(self, bids_dataset: BIDSDataset, bids_file):
+    def get_raw_extensions(self, bids_file, bids_dataset: EEGBIDSDataset):
+        bids_file = Path(bids_file)
+        extensions = {
+            '.set': ['.set', '.fdt'], # eeglab
+            '.edf': ['.edf'], # european
+            '.vhdr': ['.eeg', '.vhdr', '.vmrk', '.dat', '.raw'], # brainvision
+            '.bdf': ['.bdf'], # biosemi
+        }
+        return [str(bids_dataset.get_relative_bidspath(bids_file.with_suffix(suffix))) for suffix in extensions[bids_file.suffix] if bids_file.with_suffix(suffix).exists()]
+
+    def load_eeg_attrs_from_bids_file(self, bids_dataset: EEGBIDSDataset, bids_file):
         '''
         bids_file must be a file of the bids_dataset
         '''
         if bids_file not in bids_dataset.files:
             raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
+
+        # Initialize attrs with None values for all expected fields
+        attrs = {field: None for field in self.config['attributes'].keys()}
+
         f = os.path.basename(bids_file)
         dsnumber = bids_dataset.dataset
         # extract openneuro path by finding the first occurrence of the dataset name in the filename and remove the path before that
         openneuro_path = dsnumber + bids_file.split(dsnumber)[1]
 
-        attrs = {
-            'schema_ref': 'eeg_signal',
-            'data_name': f'{bids_dataset.dataset}_{f}',
-            'dataset': bids_dataset.dataset,
-            'bidspath': openneuro_path,
-            'subject': bids_dataset.subject(bids_file),
-            'nchans': bids_dataset.num_channels(bids_file),
-            'ntimes': bids_dataset.num_times(bids_file),
-            'channel_types': bids_dataset.channel_types(bids_file),
-            'channel_names': bids_dataset.channel_labels(bids_file),
-            'task': bids_dataset.task(bids_file),
-            'session': bids_dataset.session(bids_file),
-            'run': bids_dataset.run(bids_file),
-            'sampling_frequency': bids_dataset.sfreq(bids_file), 
-            'modality': 'EEG',
+        # Update with actual values where available
+        try:
+            participants_tsv = bids_dataset.subject_participant_tsv(bids_file)
+        except Exception as e:
+            print(f"Error getting participants_tsv: {str(e)}")
+            participants_tsv = None
+            
+        try:
+            eeg_json = bids_dataset.eeg_json(bids_file)
+        except Exception as e:
+            print(f"Error getting eeg_json: {str(e)}")
+            eeg_json = None
+            
+        bids_dependencies_files = self.config['bids_dependencies_files']
+        bidsdependencies = []
+        for extension in bids_dependencies_files:
+            try:
+                dep_path = bids_dataset.get_bids_metadata_files(bids_file, extension)
+                dep_path = [str(bids_dataset.get_relative_bidspath(dep)) for dep in dep_path]
+                bidsdependencies.extend(dep_path)
+            except Exception as e:
+                pass
+                
+        bidsdependencies.extend(self.get_raw_extensions(bids_file, bids_dataset))
+
+        # Define field extraction functions with error handling
+        field_extractors = {
+            'data_name': lambda: f'{bids_dataset.dataset}_{f}',
+            'dataset': lambda: bids_dataset.dataset,
+            'bidspath': lambda: openneuro_path,
+            'subject': lambda: bids_dataset.get_bids_file_attribute('subject', bids_file),
+            'task': lambda: bids_dataset.get_bids_file_attribute('task', bids_file),
+            'session': lambda: bids_dataset.get_bids_file_attribute('session', bids_file),
+            'run': lambda: bids_dataset.get_bids_file_attribute('run', bids_file),
+            'modality': lambda: bids_dataset.get_bids_file_attribute('modality', bids_file),
+            'sampling_frequency': lambda: bids_dataset.get_bids_file_attribute('sfreq', bids_file),
+            'nchans': lambda: bids_dataset.get_bids_file_attribute('nchans', bids_file),
+            'ntimes': lambda: bids_dataset.get_bids_file_attribute('ntimes', bids_file),
+            'participant_tsv': lambda: participants_tsv,
+            'eeg_json': lambda: eeg_json,
+            'bidsdependencies': lambda: bidsdependencies,
         }
+        
+        # Dynamically populate attrs with error handling
+        for field, extractor in field_extractors.items():
+            try:
+                attrs[field] = extractor()
+            except Exception as e:
+                print(f"Error extracting {field}: {str(e)}")
+                attrs[field] = None
 
         return attrs
 
-    def add_bids_dataset(self, dataset, data_dir, raw_format='eeglab', overwrite=True):
+    def add_bids_dataset(self, dataset, data_dir, overwrite=True):
         '''
         Create new records for the dataset in the MongoDB database if not found
         '''
         if self.is_public:
             raise ValueError('This operation is not allowed for public users')
 
-        bids_dataset = BIDSDataset(
-            data_dir=data_dir,
-            dataset=dataset,
-            raw_format=raw_format,
-        )
+        if not overwrite and self.exist({'dataset': dataset}):
+            print(f'Dataset {dataset} already exists in the database')
+            return
+        try:
+            bids_dataset = EEGBIDSDataset(
+                data_dir=data_dir,
+                dataset=dataset,
+            )
+        except Exception as e:
+            print(f'Error creating bids dataset {dataset}: {str(e)}')
+            raise e
+        requests = []
         for bids_file in bids_dataset.get_files():
-            print('bids raw file', bids_file)
-
-            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
+            try:
+                data_id = f"{dataset}_{os.path.basename(bids_file)}"
 
-            if self.exist(data_name=signalstore_data_id):
-                if overwrite:
-                    eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                    print('updating record', eeg_attrs['data_name'])
-                    self.update(eeg_attrs)
+                if self.exist({'data_name':data_id}):
+                    if overwrite:
+                        eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                        requests.append(self.update_request(eeg_attrs))
                 else:
-                    print('data already exist and not overwriting. skipped')
-                    continue
-            else:
-                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
-                # Assume raw data already exists on Openneuro, recreating record only
-                print('adding record', eeg_attrs['data_name'])
-                self.add(eeg_attrs)
+                    eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                    requests.append(self.add_request(eeg_attrs))
+            except:
+                print('error adding record', bids_file)
 
-    def get_s3path(self, record):
-        return f"{self.AWS_BUCKET}/{record['bidspath']}"
+        print('Number of database requests', len(requests))
+
+        if requests:
+            result = self.__collection.bulk_write(requests, ordered=False)
+            print(f"Inserted: {result.inserted_count}")
+            print(f"Modified: {result.modified_count}")
+            print(f"Deleted: {result.deleted_count}")
+            print(f"Upserted: {result.upserted_count}")
+            print(f"Errors: {result.bulk_api_result.get('writeErrors', [])}")
 
     def get(self, query:dict):
         '''
@@ -185,11 +246,110 @@ class EEGDash:
             )
         return results
 
+    def add_request(self, record:dict):
+        return InsertOne(record)
+
+    def add(self, record:dict):
+        try:
+            # input_record = self._validate_input(record)
+            self.__collection.insert_one(record)
+        # silent failing
+        except ValueError as e:
+            print(f"Failed to validate record: {record['data_name']}")
+            print(e)
+        except: 
+            print(f"Error adding record: {record['data_name']}")
+
+    def update_request(self, record:dict):
+        return UpdateOne({'data_name': record['data_name']}, {'$set': record})
+
     def update(self, record:dict):
-        record['schema_ref'] = 'eeg_signal'
-        self.__collection.update_one({'schema_ref': record['schema_ref'], 'data_name': record['data_name']}, 
-                    {'$set': record}
-                )
+        try:
+            self.__collection.update_one({'data_name': record['data_name']}, {'$set': record})
+        except: # silent failure
+            print(f'Error updating record {record["data_name"]}')
+
+    def remove_field(self, record, field):
+        self.__collection.update_one({'data_name': record['data_name']}, {'$unset': {field: 1}})
+    
+    def remove_field_from_db(self, field):
+        self.__collection.update_many({}, {'$unset': {field: 1}})
+    
+    @property
+    def collection(self):
+        return self.__collection
+
+class EEGDashDataset(BaseConcatDataset):
+    # CACHE_DIR = '.eegdash_cache'
+    def __init__(
+        self,
+        query:dict=None,
+        data_dir:str | list =None,
+        dataset:str | list =None,
+        description_fields: list[str]=['subject', 'session', 'run', 'task', 'age', 'gender', 'sex'],
+        cache_dir:str='.eegdash_cache',
+        **kwargs
+    ):
+        self.cache_dir = cache_dir
+        if query:
+            datasets = self.find_datasets(query, description_fields, **kwargs)
+        elif data_dir:
+            if type(data_dir) == str:
+                datasets = self.load_bids_dataset(dataset, data_dir, description_fields)
+            else:
+                assert len(data_dir) == len(dataset), 'Number of datasets and their directories must match' 
+                datasets = []
+                for i in range(len(data_dir)):
+                    datasets.extend(self.load_bids_dataset(dataset[i], data_dir[i], description_fields))
+        # convert to list using get_item on each element
+        super().__init__(datasets)
+
+    
+    def find_key_in_nested_dict(self, data, target_key):
+        if isinstance(data, dict):
+            if target_key in data:
+                return data[target_key]
+            for value in data.values():
+                result = self.find_key_in_nested_dict(value, target_key)
+                if result is not None:
+                    return result
+        return None
+
+    def find_datasets(self, query:dict, description_fields:list[str], **kwargs):
+        eegdashObj = EEGDash()
+        datasets = []
+        for record in eegdashObj.find(query):
+            description = {}
+            for field in description_fields:
+                value = self.find_key_in_nested_dict(record, field)
+                if value:
+                    description[field] = value
+            datasets.append(EEGDashBaseDataset(record, self.cache_dir, description=description, **kwargs))
+        return datasets
+
+    def load_bids_dataset(self, dataset, data_dir, description_fields: list[str],raw_format='eeglab', **kwargs):
+        '''
+        '''
+        def get_base_dataset_from_bids_file(bids_dataset, bids_file):
+            record = eegdashObj.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+            description = {}
+            for field in description_fields:
+                value = self.find_key_in_nested_dict(record, field)
+                if value:
+                    description[field] = value
+            return EEGDashBaseDataset(record, self.cache_dir, description=description, **kwargs)
+
+        bids_dataset = EEGBIDSDataset(
+            data_dir=data_dir,
+            dataset=dataset,
+            raw_format=raw_format,
+        )
+        eegdashObj = EEGDash()
+        datasets = Parallel(n_jobs=-1, prefer="threads", verbose=1)(
+                delayed(get_base_dataset_from_bids_file)(bids_dataset, bids_file) for bids_file in bids_dataset.get_files()
+            )
+        return datasets
+
 def main():
     eegdash = EEGDash()
     record = eegdash.find({'dataset': 'ds005511', 'subject': 'NDARUF236HM7'})

eegdash-0.0.9.dist-info/METADATA

@@ -0,0 +1,123 @@
+Metadata-Version: 2.4
+Name: eegdash
+Version: 0.0.9
+Summary: EEG data for machine learning
+Author-email: Young Truong <dt.young112@gmail.com>, Arnaud Delorme <adelorme@gmail.com>
+License: GNU General Public License
+        
+        Copyright (C) 2024-2025
+        
+        Young Truong, UCSD, dt.young112@gmail.com
+        Arnaud Delorme, UCSD, adelorme@ucsd.edu
+        
+        This program is free software; you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation; either version 2 of the License, or
+        (at your option) any later version.
+        
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+        
+        You should have received a copy of the GNU General Public License
+        along with this program; if not, write to the Free Software
+        Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1.07  USA
+        
+Project-URL: Homepage, https://eegdash.org
+Project-URL: Issues, https://github.com/sccn/EEGDash/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: xarray
+Requires-Dist: python-dotenv
+Requires-Dist: s3fs
+Requires-Dist: mne
+Requires-Dist: pynwb
+Requires-Dist: h5py
+Requires-Dist: pymongo
+Requires-Dist: joblib
+Requires-Dist: braindecode
+Requires-Dist: mne-bids
+Requires-Dist: pybids
+Requires-Dist: pymatreader
+Requires-Dist: pyarrow
+Requires-Dist: tqdm
+Requires-Dist: numba
+Dynamic: license-file
+
+# EEG-Dash
+To leverage recent and ongoing advancements in large-scale computational methods and to ensure the preservation of scientific data generated from publicly funded research, the EEG-DaSh data archive will create a data-sharing resource for MEEG (EEG, MEG) data contributed by collaborators for machine learning (ML) and deep learning (DL) applications. 
+
+## Data source
+The data in EEG-DaSh originates from a collaboration involving 25 laboratories, encompassing 27,053 participants. This extensive collection includes MEEG data, which is a combination of EEG and MEG signals. The data is sourced from various studies conducted by these labs, involving both healthy subjects and clinical populations with conditions such as ADHD, depression, schizophrenia, dementia, autism, and psychosis. Additionally, data spans different mental states like sleep, meditation, and cognitive tasks. In addition, EEG-DaSh will incorporate a subset of the data converted from NEMAR, which includes 330 MEEG BIDS-formatted datasets, further expanding the archive with well-curated, standardized neuroelectromagnetic data.
+
+## Featured data
+
+The following HBN datasets are currently featured on EEGDash. Documentation about these datasets is available [here](https://neuromechanist.github.io/data/hbn/).
+
+| DatasetID | Participants | Files | Sessions | Population | Channels | Is 10-20? | Modality | Size |
+|---|---|---|---|---|---|---|---|---|
+| [ds005505](https://nemar.org/dataexplorer/detail?dataset_id=ds005505) | 136 | 5393 | 1 | Healthy | 129 | other | Visual | 103 GB |
+| [ds005506](https://nemar.org/dataexplorer/detail?dataset_id=ds005506) | 150 | 5645 | 1 | Healthy | 129 | other | Visual | 112 GB |
+| [ds005507](https://nemar.org/dataexplorer/detail?dataset_id=ds005507) | 184 | 7273 | 1 | Healthy | 129 | other | Visual | 140 GB |
+| [ds005508](https://nemar.org/dataexplorer/detail?dataset_id=ds005508) | 324 | 13393 | 1 | Healthy | 129 | other | Visual | 230 GB |
+| [ds005510](https://nemar.org/dataexplorer/detail?dataset_id=ds005510) | 135 | 4933 | 1 | Healthy | 129 | other | Visual | 91 GB |
+| [ds005512](https://nemar.org/dataexplorer/detail?dataset_id=ds005512) | 257 | 9305 | 1 | Healthy | 129 | other | Visual | 157 GB |
+| [ds005514](https://nemar.org/dataexplorer/detail?dataset_id=ds005514) | 295 | 11565 | 1 | Healthy | 129 | other | Visual | 185 GB |
+
+A total of [246 other datasets](datasets.md) are also available through EEGDash. 
+
+## Data format
+EEGDash queries return a **Pytorch Dataset** formatted to facilitate machine learning (ML) and deep learning (DL) applications. PyTorch Datasets are the best format for EEGDash queries because they provide an efficient, scalable, and flexible structure for machine learning (ML) and deep learning (DL) applications. They allow seamless integration with PyTorch’s DataLoader, enabling efficient batching, shuffling, and parallel data loading, which is essential for training deep learning models on large EEG datasets.
+
+## Data preprocessing
+EEGDash datasets are processed using the popular [BrainDecode](https://braindecode.org/stable/index.html) library. In fact, EEGDash datasets are BrainDecode datasets, which are themselves PyTorch datasets. This means that any preprocessing possible on BrainDecode datasets is also possible on EEGDash datasets. Refer to [BrainDecode](https://braindecode.org/stable/index.html) tutorials for guidance on preprocessing EEG data.
+
+## EEG-Dash usage
+
+### Install
+Use your preferred Python environment manager with Python > 3.9 to install the package.
+* To install the eegdash package, use the following command: `pip install eegdash`
+* To verify the installation, start a Python session and type: `from eegdash import EEGDash`
+
+### Data access
+
+To use the data from a single subject, enter:
+
+```python
+from eegdash import EEGDashDataset
+ds_NDARDB033FW5 = EEGDashDataset({'dataset': 'ds005514', 'task': 'RestingState', 'subject': 'NDARDB033FW5'})
+```
+
+This will search and download the metadata for the task **RestingState** for subject **NDARDB033FW5** in BIDS dataset **ds005514**. The actual data will not be downloaded at this stage. Following standard practice, data is only downloaded once it is processed. The **ds_NDARDB033FW5** object is a fully functional BrainDecode dataset, which is itself a PyTorch dataset. This [tutorial](https://github.com/sccn/EEGDash/blob/develop/notebooks/tutorial_eoec.ipynb) shows how to preprocess the EEG data, extracting portions of the data containing eyes-open and eyes-closed segments, then perform eyes-open vs. eyes-closed classification using a (shallow) deep-learning model. 
+
+To use the data from multiple subjects, enter:
+
+```python
+from eegdash import EEGDashDataset
+ds_ds005505rest = EEGDashDataset({'dataset': 'ds005505', 'task': 'RestingState'}, target_name='sex')
+```
+
+This will search and download the metadata for the task 'RestingState' for all subjects in BIDS dataset 'ds005505' (a total of 136). As above, the actual data will not be downloaded at this stage so this command is quick to execute. Also, the target class for each subject is assigned using the target_name parameter. This means that this object is ready to be directly fed to a deep learning model, although the [tutorial script](https://github.com/sccn/EEGDash/blob/develop/notebooks/tutorial_sex_classification.ipynb) performs minimal processing on it, prior to training a deep-learning model. Because 14 gigabytes of data are downloaded, this tutorial takes about 10 minutes to execute.
+
+### Automatic caching
+
+EEGDash automatically caches the downloaded data in the .eegdash_cache folder of the current directory from which the script is called. This means that if you run the tutorial [scripts](https://github.com/sccn/EEGDash/tree/develop/notebooks), the data will only be downloaded the first time the script is executed.
+
+## Education -- Coming soon...
+
+We organize workshops and educational events to foster cross-cultural education and student training, offering both online and in-person opportunities in collaboration with US and Israeli partners. Events for 2025 will be announced via the EEGLABNEWS mailing list. Be sure to [subscribe](https://sccn.ucsd.edu/mailman/listinfo/eeglabnews).
+
+## About EEG-DaSh
+
+EEG-DaSh is a collaborative initiative between the United States and Israel, supported by the National Science Foundation (NSF). The partnership brings together experts from the Swartz Center for Computational Neuroscience (SCCN) at the University of California San Diego (UCSD) and Ben-Gurion University (BGU) in Israel. 
+
+![Screenshot 2024-10-03 at 09 14 06](https://github.com/user-attachments/assets/327639d3-c3b4-46b1-9335-37803209b0d3)
+
+
+
+