eegdash 0.0.1__py3-none-any.whl

eegdash/SignalStore/tests/unit/store/test_unit_of_work.py

@@ -0,0 +1,7 @@
+
+
+class TestUnitOfWork:
+
+    def test_initialize(self, unit_of_work):
+        with unit_of_work as uow:
+            pass

eegdash/SignalStore/tests/unit/test_ci_cd.py

@@ -0,0 +1,8 @@
+import pytest
+
+def test_fast_ci_cd():
+    assert 5 == 5
+
+@pytest.mark.slow()
+def test_slow_ci_cd():
+    assert 10 == 10

eegdash/__init__.py

@@ -0,0 +1 @@
+from eegdash.main import EEGDash

eegdash/aws_ingest.py

@@ -0,0 +1,29 @@
+import sys 
+sys.path.append('..')
+import argparse
+from src.signalstore_data_utils import SignalstoreBIDS
+
+def add_bids_dataset(args):
+    signalstore_aws = SignalstoreBIDS(
+        dbconnectionstring='mongodb://23.21.113.214:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.2.1',
+        local_filesystem=False,
+        project_name='eegdash',
+    )
+    signalstore_aws.add_bids_dataset(dataset=args.dataset, data_dir=args.data, raw_format='eeglab')
+
+def main():
+    # Create the parser
+    parser = argparse.ArgumentParser(description="A simple command line argument parser")
+
+    # Add arguments
+    parser.add_argument('--data', type=str, default="/mnt/nemar/openneuro/ds004186", help="Path to data directory (Default: /mnt/nemar/openneuro/ds004186)")
+    parser.add_argument('--dataset', type=str, default="ds004186", help="Dataset name (Default: ds004186)")
+
+    # Parse the arguments
+    args = parser.parse_args()
+    print('Arguments:', args)
+
+    add_bids_dataset(args)
+
+if __name__ == "__main__":
+    main()

eegdash/data_utils.py

@@ -0,0 +1,213 @@
+import os
+import sys 
+from joblib import Parallel, delayed
+import mne
+import numpy as np
+from pathlib import Path
+import re
+import json
+
+verbose = False
+
+
+class BIDSDataset():
+    ALLOWED_FILE_FORMAT = ['eeglab', 'brainvision', 'biosemi', 'european']
+    RAW_EXTENSION = {
+        'eeglab': '.set',
+        'brainvision': '.vhdr',
+        'biosemi': '.bdf',
+        'european': '.edf'
+    }
+    METADATA_FILE_EXTENSIONS = ['eeg.json', 'channels.tsv', 'electrodes.tsv', 'events.tsv', 'events.json']
+    def __init__(self,
+            data_dir=None,                            # location of asr cleaned data 
+            dataset='',                               # dataset name
+            raw_format='eeglab',                      # format of raw data
+        ):                            
+        if data_dir is None or not os.path.exists(data_dir):
+            raise ValueError('data_dir must be specified and must exist')
+        self.bidsdir = Path(data_dir)
+        self.dataset = dataset
+
+        if raw_format.lower() not in self.ALLOWED_FILE_FORMAT:
+            raise ValueError('raw_format must be one of {}'.format(self.ALLOWED_FILE_FORMAT))
+        self.raw_format = raw_format.lower()
+
+        # get all .set files in the bids directory
+        temp_dir = (Path().resolve() / 'data')
+        if not os.path.exists(temp_dir):
+            os.mkdir(temp_dir)
+        if not os.path.exists(temp_dir / f'{dataset}_files.npy'):
+            self.files = self.get_files_with_extension_parallel(self.bidsdir, extension=self.RAW_EXTENSION[self.raw_format])
+            np.save(temp_dir / f'{dataset}_files.npy', self.files)
+        else:
+            self.files = np.load(temp_dir / f'{dataset}_files.npy', allow_pickle=True)
+
+    def get_property_from_filename(self, property, filename):
+        lookup = re.search(rf'{property}-(.*?)[_\/]', filename)
+        return lookup.group(1) if lookup else ''
+
+    def get_bids_file_inheritance(self, path, basename, extension):
+        '''
+        Get all files with given extension that applies to the basename file 
+        following the BIDS inheritance principle in the order of lowest level first
+        @param
+            basename: bids file basename without _eeg.set extension for example
+            extension: e.g. channels.tsv
+        '''
+        top_level_files = ['README', 'dataset_description.json', 'participants.tsv']
+        bids_files = []
+
+        # check if path is str object
+        if isinstance(path, str):
+            path = Path(path)
+        if not path.exists:
+            raise ValueError('path {path} does not exist')
+
+        # check if file is in current path
+        for file in os.listdir(path):
+            # target_file = path / f"{cur_file_basename}_{extension}"
+            if os.path.isfile(path/file):
+                cur_file_basename = file[:file.rfind('_')]
+                if file.endswith(extension) and cur_file_basename in basename:
+                    filepath = path / file
+                    bids_files.append(filepath)
+
+        # check if file is in top level directory
+        if any(file in os.listdir(path) for file in top_level_files):
+            return bids_files
+        else:
+            # call get_bids_file_inheritance recursively with parent directory
+            bids_files.extend(self.get_bids_file_inheritance(path.parent, basename, extension))
+            return bids_files
+
+    def get_bids_metadata_files(self, filepath, metadata_file_extension):
+        """
+        (Wrapper for self.get_bids_file_inheritance)
+        Get all BIDS metadata files that are associated with the given filepath, following the BIDS inheritance principle.
+        
+        Args:
+            filepath (str or Path): The filepath to get the associated metadata files for.
+            metadata_files_extensions (list): A list of file extensions to search for metadata files.
+        
+        Returns:
+            list: A list of filepaths for all the associated metadata files
+        """
+        if isinstance(filepath, str):
+            filepath = Path(filepath)
+        if not filepath.exists:
+            raise ValueError('filepath {filepath} does not exist')
+        path, filename = os.path.split(filepath)
+        basename = filename[:filename.rfind('_')]
+        # metadata files
+        meta_files = self.get_bids_file_inheritance(path, basename, metadata_file_extension)
+        if not meta_files:
+            raise ValueError('No metadata files found for filepath {filepath} and extension {metadata_file_extension}')
+        else:
+            return meta_files
+        
+    def scan_directory(self, directory, extension):
+        result_files = []
+        directory_to_ignore = ['.git']
+        with os.scandir(directory) as entries:
+            for entry in entries:
+                if entry.is_file() and entry.name.endswith(extension):
+                    print('Adding ', entry.path)
+                    result_files.append(entry.path)
+                elif entry.is_dir():
+                    # check that entry path doesn't contain any name in ignore list
+                    if not any(name in entry.name for name in directory_to_ignore):
+                        result_files.append(entry.path)  # Add directory to scan later
+        return result_files
+
+    def get_files_with_extension_parallel(self, directory, extension='.set', max_workers=-1):
+        result_files = []
+        dirs_to_scan = [directory]
+
+        # Use joblib.Parallel and delayed to parallelize directory scanning
+        while dirs_to_scan:
+            print(f"Scanning {len(dirs_to_scan)} directories...", dirs_to_scan)
+            # Run the scan_directory function in parallel across directories
+            results = Parallel(n_jobs=max_workers, prefer="threads", verbose=1)(
+                delayed(self.scan_directory)(d, extension) for d in dirs_to_scan
+            )
+            
+            # Reset the directories to scan and process the results
+            dirs_to_scan = []
+            for res in results:
+                for path in res:
+                    if os.path.isdir(path):
+                        dirs_to_scan.append(path)  # Queue up subdirectories to scan
+                    else:
+                        result_files.append(path)  # Add files to the final result
+            print(f"Current number of files: {len(result_files)}")
+
+        return result_files
+
+    def load_and_preprocess_raw(self, raw_file, preprocess=False):
+        print(f"Loading {raw_file}")
+        EEG = mne.io.read_raw_eeglab(raw_file, preload=True, verbose='error')
+        
+        if preprocess:
+            # highpass filter
+            EEG = EEG.filter(l_freq=0.25, h_freq=25, verbose=False)
+            # remove 60Hz line noise
+            EEG = EEG.notch_filter(freqs=(60), verbose=False)
+            # bring to common sampling rate
+            sfreq = 128
+            if EEG.info['sfreq'] != sfreq:
+                EEG = EEG.resample(sfreq)
+            # # normalize data to zero mean and unit variance
+            # scalar = preprocessing.StandardScaler()
+            # mat_data = scalar.fit_transform(mat_data.T).T # scalar normalize for each feature and expects shape data x features
+
+        mat_data = EEG.get_data()
+
+        if len(mat_data.shape) > 2:
+            raise ValueError('Expect raw data to be CxT dimension')
+        return mat_data
+    
+    def get_files(self):
+        return self.files
+    
+    def resolve_bids_json(self, json_files: list):
+        """
+        Resolve the BIDS JSON files and return a dictionary of the resolved values.
+        Args:
+            json_files (list): A list of JSON files to resolve in order of leaf level first
+
+        Returns:
+            dict: A dictionary of the resolved values.
+        """
+        if len(json_files) == 0:
+            raise ValueError('No JSON files provided')
+        json_files.reverse() # TODO undeterministic
+
+        json_dict = {}
+        for json_file in json_files:
+            with open(json_file) as f:
+                json_dict.update(json.load(f))
+        return json_dict
+
+    def sfreq(self, data_filepath):
+        json_files = self.get_bids_metadata_files(data_filepath, 'eeg.json')
+        if len(json_files) == 0:
+            raise ValueError('No eeg.json found')
+
+        metadata = self.resolve_bids_json(json_files)
+        if 'SamplingFrequency' not in metadata:
+            raise ValueError('SamplingFrequency not found in metadata')
+        else:
+            return metadata['SamplingFrequency']
+    
+    def task(self, data_filepath):
+        return self.get_property_from_filename('task', data_filepath)
+        
+    def session(self, data_filepath):
+        return self.get_property_from_filename('session', data_filepath)
+
+    def run(self, data_filepath):
+        return self.get_property_from_filename('run', data_filepath)
+
+    def subject(self, data_filepath):
+        return self.get_property_from_filename('sub', data_filepath)

eegdash/main.py

@@ -0,0 +1,17 @@
+from eegdash.signalstore_data_utils import SignalstoreBIDS
+
+class EEGDash:
+    def __init__(self):
+        self.sstore = SignalstoreBIDS(
+            # dbconnectionstring='mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.3.1',
+            dbconnectionstring='mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0',
+            is_public=True,
+            local_filesystem=False,
+            project_name='eegdash'
+        )
+    
+    def find(self, *args):
+        return self.sstore.find(*args)
+    
+    def get(self, *args):
+        return self.sstore.get(*args)

eegdash/signalstore_data_utils.py

@@ -0,0 +1,280 @@
+from pathlib import Path
+from dotenv import load_dotenv
+import re
+import numpy as np
+import xarray as xr
+import os
+from eegdash.SignalStore.signalstore.store import UnitOfWorkProvider
+# from mongomock import MongoClient
+from pymongo.mongo_client import MongoClient
+from pymongo.server_api import ServerApi
+from fsspec.implementations.local import LocalFileSystem
+from fsspec.implementations.dirfs import DirFileSystem
+import pandas as pd
+import json
+import s3fs
+from eegdash.data_utils import BIDSDataset
+
+
+class SignalstoreBIDS():
+    AWS_BUCKET = 'eegdash'
+    def __init__(self, 
+                 project_name=AWS_BUCKET,
+                 dbconnectionstring="mongodb://127.0.0.1:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.3.1",
+                 is_public=False,
+                 local_filesystem=True,
+                 ):
+        self.is_public = is_public
+        if is_public:
+            dbconnectionstring='mongodb+srv://eegdash-user:mdzoMjQcHWTVnKDq@cluster0.vz35p.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0',
+        else:
+            load_dotenv()
+            dbconnectionstring = os.getenv('DB_CONNECTION_STRING')
+
+        # Create a new client and connect to the server
+        client = MongoClient(dbconnectionstring, server_api=ServerApi('1'))
+        # Send a ping to confirm a successful connection
+        try:
+            client.admin.command('ping')
+            print("Pinged your deployment. You successfully connected to MongoDB!")
+        except Exception as e:
+            print(e)
+
+        memory_store = {}
+        filesystem = self.set_up_filesystem(is_local=local_filesystem)
+        self.uow_provider = UnitOfWorkProvider(
+            mongo_client=client,
+            filesystem=filesystem,
+            memory_store=memory_store,
+            default_filetype='zarr'
+        )
+
+        self.project_name=project_name
+        self.uow = self.uow_provider(self.project_name)
+        # self.load_domain_models()
+
+    def set_up_filesystem(self, is_local=True):
+        if is_local:
+            cache_path='/mnt/nemar/dtyoung/eeg-ssl-data'                  # path where signalstore netCDF files are stored
+            # Create a directory for the dataset
+            store_path = Path(cache_path)
+            if not os.path.exists(store_path):
+                os.makedirs(store_path)
+
+            filesystem = LocalFileSystem()
+            tmp_dir_fs = DirFileSystem(
+                store_path,
+                filesystem=filesystem
+            )
+            return tmp_dir_fs
+        else:
+            if self.is_public:
+                s3 = s3fs.S3FileSystem(anon=True, client_kwargs={'region_name': 'us-east-2'})
+            else:
+                s3 = s3fs.S3FileSystem(client_kwargs={'region_name': 'us-east-2'})
+            return s3
+
+    def load_domain_models(self):
+        cwd = Path.cwd()
+        domain_models_path = cwd / f"DomainModels/{self.project_name}/data_models.json"
+        metamodel_path = cwd / f"DomainModels/{self.project_name}/metamodels.json"
+        property_path = cwd / f"DomainModels/{self.project_name}/property_models.json"
+        with open(metamodel_path) as f:
+            metamodels = json.load(f)
+
+        with open(property_path) as f:
+            property_models = json.load(f)
+            
+        # load domain models json file
+        with open(domain_models_path) as f:
+            domain_models = json.load(f)
+            
+        with self.uow as uow:
+            for property_model in property_models:
+                uow.domain_models.add(property_model)
+                model = uow.domain_models.get(property_model['schema_name'])
+                print('property model: ', model['schema_name'])
+            for metamodel in metamodels:
+                uow.domain_models.add(metamodel)
+                model = uow.domain_models.get(metamodel['schema_name'])
+                print('meta model: ', model['schema_name'])
+            for domain_model in domain_models:
+                uow.domain_models.add(domain_model)
+                model = uow.domain_models.get(domain_model['schema_name'])
+                print('domain model: ', model['schema_name'])
+                uow.commit()
+
+    def extract_attribute(self, pattern, filename):
+        match = re.search(pattern, filename)
+        return match.group(1) if match else None
+
+    def load_eeg_attrs_from_bids_file(self, bids_dataset: BIDSDataset, bids_file):
+        '''
+        bids_file must be a file of the bids_dataset
+        '''
+        if bids_file not in bids_dataset.files:
+            raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
+        f = os.path.basename(bids_file)
+        attrs = {
+            'schema_ref': 'eeg_signal',
+            'data_name': f'{bids_dataset.dataset}_{f}',
+            'dataset': bids_dataset.dataset,
+            'subject': bids_dataset.subject(bids_file),
+            'task': bids_dataset.task(bids_file),
+            'session': bids_dataset.session(bids_file),
+            'run': bids_dataset.run(bids_file),
+            'sampling_frequency': bids_dataset.sfreq(bids_file), 
+            'modality': 'EEG',
+        }
+
+        return attrs
+
+    def load_eeg_data_from_bids_file(self, bids_dataset: BIDSDataset, bids_file, eeg_attrs=None):
+        '''
+        bids_file must be a file of the bids_dataset
+        '''
+        if bids_file not in bids_dataset.files:
+            raise ValueError(f'{bids_file} not in {bids_dataset.dataset}')
+
+        attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file) if eeg_attrs is None else eeg_attrs
+
+        eeg_data = bids_dataset.load_and_preprocess_raw(bids_file)
+        print('data shape:', eeg_data.shape)
+    
+        fs = attrs['sampling_frequency']
+        max_time = eeg_data.shape[1] / fs
+        time_steps = np.linspace(0, max_time, eeg_data.shape[1]).squeeze() # in seconds
+        # print('time steps', len(time_steps))
+
+        # replace eeg.set with channels.tsv
+        # todo this is still a hacky way
+        channels_tsv = bids_dataset.get_bids_metadata_files(bids_file, 'channels.tsv')
+        channels_tsv = Path(channels_tsv[0]) 
+        if channels_tsv.exists():
+            channels = pd.read_csv(channels_tsv, sep='\t') 
+            # get channel names from channel_coords
+            channel_names = channels['name'].values
+
+        eeg_xarray = xr.DataArray(
+            data=eeg_data,
+            dims=['channel','time'],
+            coords={
+                'time': time_steps,
+                'channel': channel_names
+            },
+            attrs=attrs
+        )
+        return eeg_xarray
+
+    def exist(self, schema_ref='eeg_signal', data_name=''):
+        with self.uow as uow:
+            query = {
+                "schema_ref": schema_ref,
+                "data_name": data_name
+            }
+            sessions = uow.data.find(query)
+            if len(sessions) > 0:
+                return True
+            else:
+                return False
+
+    def add_bids_dataset(self, dataset, data_dir, raw_format='eeglab', overwrite=False, record_only=False):
+        if self.is_public:
+            raise ValueError('This operation is not allowed for public users')
+
+        bids_dataset = BIDSDataset(
+            data_dir=data_dir,
+            dataset=dataset,
+            raw_format=raw_format,
+        )
+        for bids_file in bids_dataset.get_files():
+            print('bids raw file', bids_file)
+
+            signalstore_data_id = f"{dataset}_{os.path.basename(bids_file)}"
+            if overwrite:
+                self.remove(signalstore_data_id)
+
+            if self.exist(data_name=signalstore_data_id):
+                print('data already exist. skipped')
+                continue
+            else:
+                eeg_attrs = self.load_eeg_attrs_from_bids_file(bids_dataset, bids_file)
+                with self.uow as uow:
+                    # Assume raw data already exists, recreating record only
+                    eeg_attrs['has_file'] = True
+                    print('adding record', eeg_attrs['data_name'])
+                    uow.data.add(eeg_attrs)
+                    uow.commit()
+                if  not record_only:
+                    eeg_xarray = self.load_eeg_data_from_bids_file(bids_dataset, bids_file, eeg_attrs)
+                    with self.uow as uow:
+                        print('adding data', eeg_xarray.attrs['data_name'])
+                        uow.data.add(eeg_xarray)
+                        uow.commit()
+
+    def remove(self, schema_ref='eeg_signal', data_name=''):
+        if self.is_public:
+            raise ValueError('This operation is not allowed for public users')
+
+        with self.uow as uow:
+            sessions = uow.data.find({'schema_ref': schema_ref, 'data_name': data_name})
+            if len(session) > 0:
+                for session in range(len(sessions)):
+                    uow.data.remove(session['schema_ref'], session['data_name'])
+                    uow.commit()
+
+    def remove_all(self):
+        if self.is_public:
+            raise ValueError('This operation is not allowed for public users')
+
+        with self.uow as uow:
+            sessions = uow.data.find({})
+            print(len(sessions))
+            for session in range(len(sessions)):
+                uow.data.remove(session['schema_ref'], session['data_name'])
+                uow.commit()
+
+            uow.purge()
+            
+            print('Verifying deletion job. Dataset length: ', len(uow.data.find({})))
+
+    def find(self, query:dict, validate=False, get_data=False):
+        '''
+        query: {
+            'dataset': 'dsxxxx',
+
+        }'''
+        with self.uow as uow:
+            sessions = uow.data.find(query, validate=validate, get_data=get_data)
+            if sessions:
+                print(f'Found {len(sessions)} records')
+                return sessions
+            else:
+                return []
+
+    def get(self, query:dict, validate=False):
+        '''
+        query: {
+            'dataset': 'dsxxxx',
+
+        }'''
+        with self.uow as uow:
+            sessions = uow.data.find(query, validate=validate, get_data=True)
+            if sessions:
+                print(f'Found {len(sessions)} records')
+                return sessions
+            else:
+                return []
+
+if __name__ == "__main__":
+    # sstore_hbn = SignalstoreHBN()
+    # sstore_hbn.add_data()
+    # sstore_ds004584 = SignalstoreHBN(
+    #     data_path='/mnt/nemar/openneuro/ds004584',
+    #     dataset_name='eegdash',
+    #     local_filesystem=False,
+    #     dbconnectionstring='mongodb://23.21.113.214:27017/?directConnection=true&serverSelectionTimeoutMS=2000&appName=mongosh+2.2.1'
+    # )
+    # sstore_ds004584.load_domain_models()
+    # sstore_ds004584.add_data()
+    pass

eegdash-0.0.1.dist-info/LICENSE

@@ -0,0 +1,20 @@
+GNU General Public License
+
+Copyright (C) 2024-2025
+
+Young Truong, UCSD, dt.young112@gmail.com
+Arnaud Delorme, UCSD, adelorme@ucsd.edu
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1.07  USA

eegdash-0.0.1.dist-info/METADATA

@@ -0,0 +1,72 @@
+Metadata-Version: 2.1
+Name: eegdash
+Version: 0.0.1
+Summary: EEG data for machine learning
+Author-email: Young Truong <dt.young112@gmail.com>, Arnaud Delorme <adelorme@gmail.com>
+License: GNU General Public License
+        
+        Copyright (C) 2024-2025
+        
+        Young Truong, UCSD, dt.young112@gmail.com
+        Arnaud Delorme, UCSD, adelorme@ucsd.edu
+        
+        This program is free software; you can redistribute it and/or modify
+        it under the terms of the GNU General Public License as published by
+        the Free Software Foundation; either version 2 of the License, or
+        (at your option) any later version.
+        
+        This program is distributed in the hope that it will be useful,
+        but WITHOUT ANY WARRANTY; without even the implied warranty of
+        MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+        GNU General Public License for more details.
+        
+        You should have received a copy of the GNU General Public License
+        along with this program; if not, write to the Free Software
+        Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1.07  USA
+        
+Project-URL: Homepage, https://github.com/sccn/EEG-Dash-Data
+Project-URL: Issues, https://github.com/sccn/EEG-Dash-Data/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# EEG-Dash
+To leverage recent and ongoing advancements in large-scale computational methods and to ensure the preservation of scientific data generated from publicly funded research, the EEG-DaSh data archive will create a data-sharing resource for MEEG (EEG, MEG) data contributed by collaborators for machine learning (ML) and deep learning (DL) applications. 
+
+## Data source
+The data in EEG-DaSh originates from a collaboration involving 25 laboratories, encompassing 27,053 participants. This extensive collection includes MEEG data, which is a combination of EEG and MEG signals. The data is sourced from various studies conducted by these labs, involving both healthy subjects and clinical populations with conditions such as ADHD, depression, schizophrenia, dementia, autism, and psychosis. Additionally, data spans different mental states like sleep, meditation, and cognitive tasks. In addition, EEG-DaSh will also incorporate data converted from NEMAR, which includes a subset of the 330 MEEG BIDS-formatted datasets available on OpenNeuro, further expanding the archive with well-curated, standardized neuroelectromagnetic data.
+
+## Data formatting
+The data in EEG-DaSh is formatted to facilitate machine learning (ML) and deep learning (DL) applications by using a simplified structure commonly adopted by these communities. This will involve converting raw MEEG data into a matrix format, where samples (e.g., individual EEG or MEG recordings) are represented by rows, and values (such as time or channel data) are represented by columns. The data is also divided into training and testing sets, with 80% of the data allocated for training and 20% for testing, ensuring a balanced representation of relevant labels across sets. Hierarchical Event Descriptor (HED) tags will be used to annotate labels, which will be stored in a text table, and detailed metadata, including dataset origins and methods. This formatting process will ensure that data is ready for ML/DL models, allowing for efficient training and testing of algorithms while preserving data integrity and reusability.
+
+![Screenshot 2024-10-03 at 09 07 28](https://github.com/user-attachments/assets/b30a79bb-0d94-410a-843c-44c3fcea01fc)
+
+## Data access
+The data in EEG-DaSh is formatted to facilitate machine learning (ML) and deep learning (DL) applications by using a simplified structure commonly adopted by these communities. This will involve converting raw MEEG data into a matrix format, where samples (e.g., individual EEG or MEG recordings) are represented by rows, and values (such as time or channel data) are represented by columns. The data is also divided into training and testing sets, with 80% of the data allocated for training and 20% for testing, ensuring a balanced representation of relevant labels across sets. Hierarchical Event Descriptor (HED) tags will be used to annotate labels, which will be stored in a text table, and detailed metadata, including dataset origins and methods. This formatting process will ensure that data is ready for ML/DL models, allowing for efficient training and testing of algorithms while preserving data integrity and reusability.
+
+The data in EEG-DaSh is accessed through Python and MATLAB libraries specifically designed for this platform. These libraries will use objects compatible with deep learning data storage formats in each language, such as <i>Torchvision.dataset</i> in Python and <i>DataStore</i> in MATLAB. Users can dynamically fetch data from the EEG-DaSh server which is then cached locally. 
+
+### AWS S3
+
+Coming soon...
+
+### EEG-Dash API
+
+Coming soon...
+
+## Education
+
+We organize workshops and educational events to foster cross-cultural education and student training, offering both online and in-person opportunities in collaboration with US and Israeli partners. There is no event planned for 2024. Events for 2025 will be advertised on the EEGLABNEWS mailing list so make sure to [subscribe](https://sccn.ucsd.edu/mailman/listinfo/eeglabnews).
+
+## About EEG-DaSh
+
+EEG-DaSh is a collaborative initiative between the United States and Israel, supported by the National Science Foundation (NSF). The partnership brings together experts from the Swartz Center for Computational Neuroscience (SCCN) at the University of California San Diego (UCSD) and Ben-Gurion University (BGU) in Israel. 
+
+![Screenshot 2024-10-03 at 09 14 06](https://github.com/user-attachments/assets/327639d3-c3b4-46b1-9335-37803209b0d3)
+
+
+
+