eegdash 0.3.9.dev182388821__py3-none-any.whl → 0.4.0.dev132__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of eegdash might be problematic. Click here for more details.
- eegdash/__init__.py +1 -1
- eegdash/api.py +68 -145
- eegdash/bids_eeg_metadata.py +149 -27
- eegdash/data_utils.py +63 -254
- eegdash/dataset/dataset.py +27 -21
- eegdash/downloader.py +176 -0
- eegdash/features/datasets.py +4 -3
- eegdash/hbn/preprocessing.py +1 -3
- eegdash/hbn/windows.py +0 -2
- eegdash/logging.py +23 -0
- {eegdash-0.3.9.dev182388821.dist-info → eegdash-0.4.0.dev132.dist-info}/METADATA +5 -56
- {eegdash-0.3.9.dev182388821.dist-info → eegdash-0.4.0.dev132.dist-info}/RECORD +15 -13
- {eegdash-0.3.9.dev182388821.dist-info → eegdash-0.4.0.dev132.dist-info}/WHEEL +0 -0
- {eegdash-0.3.9.dev182388821.dist-info → eegdash-0.4.0.dev132.dist-info}/licenses/LICENSE +0 -0
- {eegdash-0.3.9.dev182388821.dist-info → eegdash-0.4.0.dev132.dist-info}/top_level.txt +0 -0
eegdash/data_utils.py
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import io
|
|
2
2
|
import json
|
|
3
|
-
import logging
|
|
4
3
|
import os
|
|
5
4
|
import re
|
|
6
5
|
import traceback
|
|
7
|
-
import warnings
|
|
8
6
|
from contextlib import redirect_stderr
|
|
9
7
|
from pathlib import Path
|
|
10
8
|
from typing import Any
|
|
@@ -13,9 +11,7 @@ import mne
|
|
|
13
11
|
import mne_bids
|
|
14
12
|
import numpy as np
|
|
15
13
|
import pandas as pd
|
|
16
|
-
import s3fs
|
|
17
14
|
from bids import BIDSLayout
|
|
18
|
-
from fsspec.callbacks import TqdmCallback
|
|
19
15
|
from joblib import Parallel, delayed
|
|
20
16
|
from mne._fiff.utils import _read_segments_file
|
|
21
17
|
from mne.io import BaseRaw
|
|
@@ -23,10 +19,11 @@ from mne_bids import BIDSPath
|
|
|
23
19
|
|
|
24
20
|
from braindecode.datasets import BaseDataset
|
|
25
21
|
|
|
22
|
+
from . import downloader
|
|
23
|
+
from .bids_eeg_metadata import enrich_from_participants
|
|
24
|
+
from .logging import logger
|
|
26
25
|
from .paths import get_default_cache_dir
|
|
27
26
|
|
|
28
|
-
logger = logging.getLogger("eegdash")
|
|
29
|
-
|
|
30
27
|
|
|
31
28
|
class EEGDashBaseDataset(BaseDataset):
|
|
32
29
|
"""A single EEG recording hosted on AWS S3 and cached locally upon first access.
|
|
@@ -73,6 +70,7 @@ class EEGDashBaseDataset(BaseDataset):
|
|
|
73
70
|
# Compute a dataset folder name under cache_dir that encodes preprocessing
|
|
74
71
|
# (e.g., bdf, mini) to avoid overlapping with the original dataset cache.
|
|
75
72
|
self.dataset_folder = record.get("dataset", "")
|
|
73
|
+
# TODO: remove this hack when competition is over
|
|
76
74
|
if s3_bucket:
|
|
77
75
|
suffixes: list[str] = []
|
|
78
76
|
bucket_lower = str(s3_bucket).lower()
|
|
@@ -91,6 +89,7 @@ class EEGDashBaseDataset(BaseDataset):
|
|
|
91
89
|
rel = Path(self.dataset_folder) / rel
|
|
92
90
|
self.filecache = self.cache_dir / rel
|
|
93
91
|
self.bids_root = self.cache_dir / self.dataset_folder
|
|
92
|
+
|
|
94
93
|
self.bidspath = BIDSPath(
|
|
95
94
|
root=self.bids_root,
|
|
96
95
|
datatype="eeg",
|
|
@@ -98,113 +97,18 @@ class EEGDashBaseDataset(BaseDataset):
|
|
|
98
97
|
**self.bids_kwargs,
|
|
99
98
|
)
|
|
100
99
|
|
|
101
|
-
self.s3file = self.
|
|
100
|
+
self.s3file = downloader.get_s3path(self.s3_bucket, record["bidspath"])
|
|
102
101
|
self.bids_dependencies = record["bidsdependencies"]
|
|
103
|
-
|
|
104
|
-
#
|
|
102
|
+
self.bids_dependencies_original = record["bidsdependencies"]
|
|
103
|
+
# TODO: removing temporary fix for BIDS dependencies path
|
|
104
|
+
# when the competition is over and dataset is digested properly
|
|
105
105
|
if not self.s3_open_neuro:
|
|
106
|
-
self.bids_dependencies_original = self.bids_dependencies
|
|
107
106
|
self.bids_dependencies = [
|
|
108
107
|
dep.split("/", 1)[1] for dep in self.bids_dependencies
|
|
109
108
|
]
|
|
110
109
|
|
|
111
110
|
self._raw = None
|
|
112
111
|
|
|
113
|
-
def _get_s3path(self, filepath: str) -> str:
|
|
114
|
-
"""Helper to form an AWS S3 URI for the given relative filepath."""
|
|
115
|
-
return f"{self.s3_bucket}/{filepath}"
|
|
116
|
-
|
|
117
|
-
def _download_s3(self) -> None:
|
|
118
|
-
"""Download function that gets the raw EEG data from S3."""
|
|
119
|
-
filesystem = s3fs.S3FileSystem(
|
|
120
|
-
anon=True, client_kwargs={"region_name": "us-east-2"}
|
|
121
|
-
)
|
|
122
|
-
if not self.s3_open_neuro:
|
|
123
|
-
self.s3file = re.sub(r"(^|/)ds\d{6}/", r"\1", self.s3file, count=1)
|
|
124
|
-
if self.s3file.endswith(".set"):
|
|
125
|
-
self.s3file = self.s3file[:-4] + ".bdf"
|
|
126
|
-
self.filecache = self.filecache.with_suffix(".bdf")
|
|
127
|
-
|
|
128
|
-
self.filecache.parent.mkdir(parents=True, exist_ok=True)
|
|
129
|
-
info = filesystem.info(self.s3file)
|
|
130
|
-
size = info.get("size") or info.get("Size")
|
|
131
|
-
|
|
132
|
-
callback = TqdmCallback(
|
|
133
|
-
size=size,
|
|
134
|
-
tqdm_kwargs=dict(
|
|
135
|
-
desc=f"Downloading {Path(self.s3file).name}",
|
|
136
|
-
unit="B",
|
|
137
|
-
unit_scale=True,
|
|
138
|
-
unit_divisor=1024,
|
|
139
|
-
dynamic_ncols=True,
|
|
140
|
-
leave=True,
|
|
141
|
-
mininterval=0.2,
|
|
142
|
-
smoothing=0.1,
|
|
143
|
-
miniters=1,
|
|
144
|
-
bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
|
|
145
|
-
"[{elapsed}<{remaining}, {rate_fmt}]",
|
|
146
|
-
),
|
|
147
|
-
)
|
|
148
|
-
filesystem.get(self.s3file, self.filecache, callback=callback)
|
|
149
|
-
|
|
150
|
-
self.filenames = [self.filecache]
|
|
151
|
-
|
|
152
|
-
def _download_dependencies(self) -> None:
|
|
153
|
-
"""Download all BIDS dependency files (metadata files, recording sidecar files)
|
|
154
|
-
from S3 and cache them locally.
|
|
155
|
-
"""
|
|
156
|
-
filesystem = s3fs.S3FileSystem(
|
|
157
|
-
anon=True, client_kwargs={"region_name": "us-east-2"}
|
|
158
|
-
)
|
|
159
|
-
for i, dep in enumerate(self.bids_dependencies):
|
|
160
|
-
if not self.s3_open_neuro:
|
|
161
|
-
# fix this when our bucket is integrated into the
|
|
162
|
-
# mongodb
|
|
163
|
-
# if the file have ".set" replace to ".bdf"
|
|
164
|
-
if dep.endswith(".set"):
|
|
165
|
-
dep = dep[:-4] + ".bdf"
|
|
166
|
-
|
|
167
|
-
s3path = self._get_s3path(dep)
|
|
168
|
-
if not self.s3_open_neuro:
|
|
169
|
-
dep = self.bids_dependencies_original[i]
|
|
170
|
-
|
|
171
|
-
dep_path = Path(dep)
|
|
172
|
-
if dep_path.parts and dep_path.parts[0] == self.record.get("dataset"):
|
|
173
|
-
dep_local = Path(self.dataset_folder, *dep_path.parts[1:])
|
|
174
|
-
else:
|
|
175
|
-
dep_local = Path(self.dataset_folder) / dep_path
|
|
176
|
-
filepath = self.cache_dir / dep_local
|
|
177
|
-
if not self.s3_open_neuro:
|
|
178
|
-
if filepath.suffix == ".set":
|
|
179
|
-
filepath = filepath.with_suffix(".bdf")
|
|
180
|
-
if self.filecache.suffix == ".set":
|
|
181
|
-
self.filecache = self.filecache.with_suffix(".bdf")
|
|
182
|
-
|
|
183
|
-
# here, we download the dependency and it is fine
|
|
184
|
-
# in the case of the competition.
|
|
185
|
-
if not filepath.exists():
|
|
186
|
-
filepath.parent.mkdir(parents=True, exist_ok=True)
|
|
187
|
-
info = filesystem.info(s3path)
|
|
188
|
-
size = info.get("size") or info.get("Size")
|
|
189
|
-
|
|
190
|
-
callback = TqdmCallback(
|
|
191
|
-
size=size,
|
|
192
|
-
tqdm_kwargs=dict(
|
|
193
|
-
desc=f"Downloading {Path(s3path).name}",
|
|
194
|
-
unit="B",
|
|
195
|
-
unit_scale=True,
|
|
196
|
-
unit_divisor=1024,
|
|
197
|
-
dynamic_ncols=True,
|
|
198
|
-
leave=True,
|
|
199
|
-
mininterval=0.2,
|
|
200
|
-
smoothing=0.1,
|
|
201
|
-
miniters=1,
|
|
202
|
-
bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
|
|
203
|
-
"[{elapsed}<{remaining}, {rate_fmt}]",
|
|
204
|
-
),
|
|
205
|
-
)
|
|
206
|
-
filesystem.get(s3path, filepath, callback=callback)
|
|
207
|
-
|
|
208
112
|
def _get_raw_bids_args(self) -> dict[str, Any]:
|
|
209
113
|
"""Helper to restrict the metadata record to the fields needed to locate a BIDS
|
|
210
114
|
recording.
|
|
@@ -222,130 +126,43 @@ class EEGDashBaseDataset(BaseDataset):
|
|
|
222
126
|
|
|
223
127
|
if not os.path.exists(self.filecache): # not preload
|
|
224
128
|
if self.bids_dependencies:
|
|
225
|
-
|
|
226
|
-
|
|
129
|
+
downloader.download_dependencies(
|
|
130
|
+
s3_bucket=self.s3_bucket,
|
|
131
|
+
bids_dependencies=self.bids_dependencies,
|
|
132
|
+
bids_dependencies_original=self.bids_dependencies_original,
|
|
133
|
+
cache_dir=self.cache_dir,
|
|
134
|
+
dataset_folder=self.dataset_folder,
|
|
135
|
+
record=self.record,
|
|
136
|
+
s3_open_neuro=self.s3_open_neuro,
|
|
137
|
+
)
|
|
138
|
+
self.filecache = downloader.download_s3_file(
|
|
139
|
+
self.s3file, self.filecache, self.s3_open_neuro
|
|
140
|
+
)
|
|
141
|
+
self.filenames = [self.filecache]
|
|
227
142
|
if self._raw is None:
|
|
228
|
-
# capturing any warnings
|
|
229
|
-
# to-do: remove this once is fixed on the mne-bids side.
|
|
230
|
-
with warnings.catch_warnings(record=True) as w:
|
|
231
|
-
# Ensure all warnings are captured into 'w' and not shown to users
|
|
232
|
-
warnings.simplefilter("always")
|
|
233
|
-
try:
|
|
234
|
-
# mne-bids emits RuntimeWarnings to stderr; silence stderr during read
|
|
235
|
-
_stderr_buffer = io.StringIO()
|
|
236
|
-
with redirect_stderr(_stderr_buffer):
|
|
237
|
-
self._raw = mne_bids.read_raw_bids(
|
|
238
|
-
bids_path=self.bidspath, verbose="ERROR"
|
|
239
|
-
)
|
|
240
|
-
# Parse unmapped participants.tsv fields reported by mne-bids and
|
|
241
|
-
# inject them into Raw.info and the dataset description generically.
|
|
242
|
-
extras = self._extract_unmapped_participants_from_warnings(w)
|
|
243
|
-
if extras:
|
|
244
|
-
# 1) Attach to Raw.info under subject_info.participants_extras
|
|
245
|
-
try:
|
|
246
|
-
subject_info = self._raw.info.get("subject_info") or {}
|
|
247
|
-
if not isinstance(subject_info, dict):
|
|
248
|
-
subject_info = {}
|
|
249
|
-
pe = subject_info.get("participants_extras") or {}
|
|
250
|
-
if not isinstance(pe, dict):
|
|
251
|
-
pe = {}
|
|
252
|
-
# Merge without overwriting
|
|
253
|
-
for k, v in extras.items():
|
|
254
|
-
pe.setdefault(k, v)
|
|
255
|
-
subject_info["participants_extras"] = pe
|
|
256
|
-
self._raw.info["subject_info"] = subject_info
|
|
257
|
-
except Exception:
|
|
258
|
-
# Non-fatal; continue
|
|
259
|
-
pass
|
|
260
|
-
|
|
261
|
-
# 2) Also add to this dataset's description, if possible, so
|
|
262
|
-
# targets can be selected later without naming specifics.
|
|
263
|
-
try:
|
|
264
|
-
if isinstance(self.description, dict):
|
|
265
|
-
for k, v in extras.items():
|
|
266
|
-
self.description.setdefault(k, v)
|
|
267
|
-
elif isinstance(self.description, pd.Series):
|
|
268
|
-
for k, v in extras.items():
|
|
269
|
-
if k not in self.description.index:
|
|
270
|
-
self.description.loc[k] = v
|
|
271
|
-
except Exception:
|
|
272
|
-
pass
|
|
273
|
-
except Exception as e:
|
|
274
|
-
logger.error(
|
|
275
|
-
f"Error while reading BIDS file: {self.bidspath}\n"
|
|
276
|
-
"This may be due to a missing or corrupted file.\n"
|
|
277
|
-
"Please check the file and try again."
|
|
278
|
-
)
|
|
279
|
-
logger.error(f"Exception: {e}")
|
|
280
|
-
logger.error(traceback.format_exc())
|
|
281
|
-
raise e
|
|
282
|
-
# Filter noisy mapping notices from mne-bids; surface others
|
|
283
|
-
for captured_warning in w:
|
|
284
|
-
try:
|
|
285
|
-
msg = str(captured_warning.message)
|
|
286
|
-
except Exception:
|
|
287
|
-
continue
|
|
288
|
-
# Suppress verbose participants mapping messages
|
|
289
|
-
if "Unable to map the following column" in msg and "MNE" in msg:
|
|
290
|
-
logger.debug(
|
|
291
|
-
"Suppressed mne-bids mapping warning while reading BIDS file: %s",
|
|
292
|
-
msg,
|
|
293
|
-
)
|
|
294
|
-
continue
|
|
295
|
-
|
|
296
|
-
def _extract_unmapped_participants_from_warnings(
|
|
297
|
-
self, warnings_list: list[Any]
|
|
298
|
-
) -> dict[str, Any]:
|
|
299
|
-
"""Scan captured warnings from mne-bids and extract unmapped participants.tsv
|
|
300
|
-
entries in a generic way.
|
|
301
|
-
|
|
302
|
-
Optionally, the column name can carry a note in parentheses that we ignore
|
|
303
|
-
for key/value extraction. Returns a mapping of column name -> raw value.
|
|
304
|
-
"""
|
|
305
|
-
extras: dict[str, Any] = {}
|
|
306
|
-
header = "Unable to map the following column(s) to MNE:"
|
|
307
|
-
for wr in warnings_list:
|
|
308
|
-
try:
|
|
309
|
-
msg = str(wr.message)
|
|
310
|
-
except Exception:
|
|
311
|
-
continue
|
|
312
|
-
if header not in msg:
|
|
313
|
-
continue
|
|
314
|
-
lines = msg.splitlines()
|
|
315
|
-
# Find the header line, then parse subsequent lines as entries
|
|
316
143
|
try:
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
"""Main function to access a sample from the dataset."""
|
|
340
|
-
X = self.raw[:, index][0]
|
|
341
|
-
y = None
|
|
342
|
-
if self.target_name is not None:
|
|
343
|
-
y = self.description[self.target_name]
|
|
344
|
-
if isinstance(y, pd.Series):
|
|
345
|
-
y = y.to_list()
|
|
346
|
-
if self.transform is not None:
|
|
347
|
-
X = self.transform(X)
|
|
348
|
-
return X, y
|
|
144
|
+
# mne-bids can emit noisy warnings to stderr; keep user logs clean
|
|
145
|
+
_stderr_buffer = io.StringIO()
|
|
146
|
+
with redirect_stderr(_stderr_buffer):
|
|
147
|
+
self._raw = mne_bids.read_raw_bids(
|
|
148
|
+
bids_path=self.bidspath, verbose="ERROR"
|
|
149
|
+
)
|
|
150
|
+
# Enrich Raw.info and description with participants.tsv extras
|
|
151
|
+
enrich_from_participants(
|
|
152
|
+
self.bids_root, self.bidspath, self._raw, self.description
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
except Exception as e:
|
|
156
|
+
logger.error(
|
|
157
|
+
f"Error while reading BIDS file: {self.bidspath}\n"
|
|
158
|
+
"This may be due to a missing or corrupted file.\n"
|
|
159
|
+
"Please check the file and try again.\n"
|
|
160
|
+
"Usually erasing the local cache and re-downloading helps.\n"
|
|
161
|
+
f"`rm {self.bidspath}`"
|
|
162
|
+
)
|
|
163
|
+
logger.error(f"Exception: {e}")
|
|
164
|
+
logger.error(traceback.format_exc())
|
|
165
|
+
raise e
|
|
349
166
|
|
|
350
167
|
def __len__(self) -> int:
|
|
351
168
|
"""Return the number of samples in the dataset."""
|
|
@@ -426,13 +243,16 @@ class EEGDashBaseRaw(BaseRaw):
|
|
|
426
243
|
ch_types.append(chtype)
|
|
427
244
|
info = mne.create_info(ch_names=ch_names, sfreq=sfreq, ch_types=ch_types)
|
|
428
245
|
|
|
429
|
-
self.s3file = self.
|
|
246
|
+
self.s3file = downloader.get_s3path(self._AWS_BUCKET, input_fname)
|
|
430
247
|
self.cache_dir = Path(cache_dir) if cache_dir else get_default_cache_dir()
|
|
431
248
|
self.filecache = self.cache_dir / input_fname
|
|
432
249
|
self.bids_dependencies = bids_dependencies
|
|
433
250
|
|
|
434
251
|
if preload and not os.path.exists(self.filecache):
|
|
435
|
-
self.
|
|
252
|
+
self.filecache = downloader.download_s3_file(
|
|
253
|
+
self.s3file, self.filecache, self.s3_open_neuro
|
|
254
|
+
)
|
|
255
|
+
self.filenames = [self.filecache]
|
|
436
256
|
preload = self.filecache
|
|
437
257
|
|
|
438
258
|
super().__init__(
|
|
@@ -443,35 +263,24 @@ class EEGDashBaseRaw(BaseRaw):
|
|
|
443
263
|
verbose=verbose,
|
|
444
264
|
)
|
|
445
265
|
|
|
446
|
-
def _get_s3path(self, filepath):
|
|
447
|
-
return f"{self._AWS_BUCKET}/{filepath}"
|
|
448
|
-
|
|
449
|
-
def _download_s3(self) -> None:
|
|
450
|
-
self.filecache.parent.mkdir(parents=True, exist_ok=True)
|
|
451
|
-
filesystem = s3fs.S3FileSystem(
|
|
452
|
-
anon=True, client_kwargs={"region_name": "us-east-2"}
|
|
453
|
-
)
|
|
454
|
-
filesystem.download(self.s3file, self.filecache)
|
|
455
|
-
self.filenames = [self.filecache]
|
|
456
|
-
|
|
457
|
-
def _download_dependencies(self):
|
|
458
|
-
filesystem = s3fs.S3FileSystem(
|
|
459
|
-
anon=True, client_kwargs={"region_name": "us-east-2"}
|
|
460
|
-
)
|
|
461
|
-
for dep in self.bids_dependencies:
|
|
462
|
-
s3path = self._get_s3path(dep)
|
|
463
|
-
filepath = self.cache_dir / dep
|
|
464
|
-
if not filepath.exists():
|
|
465
|
-
filepath.parent.mkdir(parents=True, exist_ok=True)
|
|
466
|
-
filesystem.download(s3path, filepath)
|
|
467
|
-
|
|
468
266
|
def _read_segment(
|
|
469
267
|
self, start=0, stop=None, sel=None, data_buffer=None, *, verbose=None
|
|
470
268
|
):
|
|
471
269
|
if not os.path.exists(self.filecache): # not preload
|
|
472
|
-
if self.bids_dependencies:
|
|
473
|
-
|
|
474
|
-
|
|
270
|
+
if self.bids_dependencies: # this is use only to sidecars for now
|
|
271
|
+
downloader.download_dependencies(
|
|
272
|
+
s3_bucket=self._AWS_BUCKET,
|
|
273
|
+
bids_dependencies=self.bids_dependencies,
|
|
274
|
+
bids_dependencies_original=None,
|
|
275
|
+
cache_dir=self.cache_dir,
|
|
276
|
+
dataset_folder=self.filecache,
|
|
277
|
+
record={},
|
|
278
|
+
s3_open_neuro=self.s3_open_neuro,
|
|
279
|
+
)
|
|
280
|
+
self.filecache = downloader.download_s3_file(
|
|
281
|
+
self.s3file, self.filecache, self.s3_open_neuro
|
|
282
|
+
)
|
|
283
|
+
self.filenames = [self.filecache]
|
|
475
284
|
else: # not preload and file is not cached
|
|
476
285
|
self.filenames = [self.filecache]
|
|
477
286
|
return super()._read_segment(start, stop, sel, data_buffer, verbose=verbose)
|
eegdash/dataset/dataset.py
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import logging
|
|
2
1
|
from pathlib import Path
|
|
3
2
|
|
|
4
|
-
from
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
from rich.panel import Panel
|
|
5
|
+
from rich.text import Text
|
|
5
6
|
|
|
6
7
|
from ..api import EEGDashDataset
|
|
7
8
|
from ..bids_eeg_metadata import build_query_from_kwargs
|
|
8
9
|
from ..const import RELEASE_TO_OPENNEURO_DATASET_MAP, SUBJECT_MINI_RELEASE_MAP
|
|
10
|
+
from ..logging import logger
|
|
9
11
|
from .registry import register_openneuro_datasets
|
|
10
12
|
|
|
11
|
-
logger = logging.getLogger("eegdash")
|
|
12
|
-
|
|
13
13
|
|
|
14
14
|
class EEGChallengeDataset(EEGDashDataset):
|
|
15
15
|
"""EEG 2025 Challenge dataset helper.
|
|
@@ -23,8 +23,6 @@ class EEGChallengeDataset(EEGDashDataset):
|
|
|
23
23
|
----------
|
|
24
24
|
release : str
|
|
25
25
|
Release name. One of ["R1", ..., "R11"].
|
|
26
|
-
cache_dir : str
|
|
27
|
-
Local cache directory for data files.
|
|
28
26
|
mini : bool, default True
|
|
29
27
|
If True, restrict subjects to the challenge mini subset.
|
|
30
28
|
query : dict | None
|
|
@@ -123,24 +121,32 @@ class EEGChallengeDataset(EEGDashDataset):
|
|
|
123
121
|
else:
|
|
124
122
|
s3_bucket = f"{s3_bucket}/{release}_L100_bdf"
|
|
125
123
|
|
|
126
|
-
|
|
127
|
-
"\n\n"
|
|
128
|
-
"[EEGChallengeDataset] EEG 2025 Competition Data Notice:\n"
|
|
129
|
-
"-------------------------------------------------------\n"
|
|
124
|
+
message_text = Text.from_markup(
|
|
130
125
|
"This object loads the HBN dataset that has been preprocessed for the EEG Challenge:\n"
|
|
131
|
-
"
|
|
132
|
-
"
|
|
133
|
-
"
|
|
134
|
-
"
|
|
135
|
-
"
|
|
136
|
-
"\n"
|
|
137
|
-
"IMPORTANT: The data accessed via `EEGChallengeDataset` is NOT identical to what you get from
|
|
138
|
-
"If you are participating in the competition, always use `EEGChallengeDataset` to ensure consistency with the challenge data
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
126
|
+
" * Downsampled from 500Hz to 100Hz\n"
|
|
127
|
+
" * Bandpass filtered (0.5-50 Hz)\n\n"
|
|
128
|
+
"For full preprocessing applied for competition details, see:\n"
|
|
129
|
+
" [link=https://github.com/eeg2025/downsample-datasets]https://github.com/eeg2025/downsample-datasets[/link]\n\n"
|
|
130
|
+
"The HBN dataset have some preprocessing applied by the HBN team:\n"
|
|
131
|
+
" * Re-reference (Cz Channel)\n\n"
|
|
132
|
+
"[bold red]IMPORTANT[/bold red]: The data accessed via `EEGChallengeDataset` is [u]NOT[/u] identical to what you get from [link=https://github.com/sccn/EEGDash/blob/develop/eegdash/api.py]EEGDashDataset[/link] directly.\n"
|
|
133
|
+
"If you are participating in the competition, always use `EEGChallengeDataset` to ensure consistency with the challenge data."
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
warning_panel = Panel(
|
|
137
|
+
message_text,
|
|
138
|
+
title="[yellow]EEG 2025 Competition Data Notice[/yellow]",
|
|
139
|
+
subtitle="[cyan]Source: EEGChallengeDataset[/cyan]",
|
|
140
|
+
border_style="yellow",
|
|
142
141
|
)
|
|
143
142
|
|
|
143
|
+
# Render the panel directly to the console so it displays in IPython/terminals
|
|
144
|
+
try:
|
|
145
|
+
Console().print(warning_panel)
|
|
146
|
+
except Exception:
|
|
147
|
+
warning_message = str(message_text)
|
|
148
|
+
logger.warning(warning_message)
|
|
149
|
+
|
|
144
150
|
super().__init__(
|
|
145
151
|
dataset=RELEASE_TO_OPENNEURO_DATASET_MAP[release],
|
|
146
152
|
query=query,
|
eegdash/downloader.py
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import tempfile
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
|
|
7
|
+
import mne
|
|
8
|
+
import numpy as np
|
|
9
|
+
import s3fs
|
|
10
|
+
import xarray as xr
|
|
11
|
+
from fsspec.callbacks import TqdmCallback
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_s3_filesystem():
|
|
15
|
+
"""Returns an S3FileSystem object."""
|
|
16
|
+
return s3fs.S3FileSystem(anon=True, client_kwargs={"region_name": "us-east-2"})
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def get_s3path(s3_bucket: str, filepath: str) -> str:
|
|
20
|
+
"""Helper to form an AWS S3 URI for the given relative filepath."""
|
|
21
|
+
return f"{s3_bucket}/{filepath}"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def download_s3_file(s3_path: str, local_path: Path, s3_open_neuro: bool):
|
|
25
|
+
"""Download function that gets the raw EEG data from S3."""
|
|
26
|
+
filesystem = get_s3_filesystem()
|
|
27
|
+
if not s3_open_neuro:
|
|
28
|
+
s3_path = re.sub(r"(^|/)ds\d{6}/", r"\1", s3_path, count=1)
|
|
29
|
+
# TODO: remove this hack when competition is over
|
|
30
|
+
if s3_path.endswith(".set"):
|
|
31
|
+
s3_path = s3_path[:-4] + ".bdf"
|
|
32
|
+
local_path = local_path.with_suffix(".bdf")
|
|
33
|
+
|
|
34
|
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
35
|
+
_filesystem_get(filesystem=filesystem, s3path=s3_path, filepath=local_path)
|
|
36
|
+
|
|
37
|
+
return local_path
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def download_dependencies(
|
|
41
|
+
s3_bucket: str,
|
|
42
|
+
bids_dependencies: list[str],
|
|
43
|
+
bids_dependencies_original: list[str],
|
|
44
|
+
cache_dir: Path,
|
|
45
|
+
dataset_folder: Path,
|
|
46
|
+
record: dict[str, Any],
|
|
47
|
+
s3_open_neuro: bool,
|
|
48
|
+
):
|
|
49
|
+
"""Download all BIDS dependency files from S3 and cache them locally."""
|
|
50
|
+
filesystem = get_s3_filesystem()
|
|
51
|
+
for i, dep in enumerate(bids_dependencies):
|
|
52
|
+
if not s3_open_neuro:
|
|
53
|
+
if dep.endswith(".set"):
|
|
54
|
+
dep = dep[:-4] + ".bdf"
|
|
55
|
+
|
|
56
|
+
s3path = get_s3path(s3_bucket, dep)
|
|
57
|
+
if not s3_open_neuro:
|
|
58
|
+
dep = bids_dependencies_original[i]
|
|
59
|
+
|
|
60
|
+
dep_path = Path(dep)
|
|
61
|
+
if dep_path.parts and dep_path.parts[0] == record.get("dataset"):
|
|
62
|
+
dep_local = Path(dataset_folder, *dep_path.parts[1:])
|
|
63
|
+
else:
|
|
64
|
+
dep_local = Path(dataset_folder) / dep_path
|
|
65
|
+
filepath = cache_dir / dep_local
|
|
66
|
+
if not s3_open_neuro:
|
|
67
|
+
if filepath.suffix == ".set":
|
|
68
|
+
filepath = filepath.with_suffix(".bdf")
|
|
69
|
+
|
|
70
|
+
if not filepath.exists():
|
|
71
|
+
filepath.parent.mkdir(parents=True, exist_ok=True)
|
|
72
|
+
_filesystem_get(filesystem=filesystem, s3path=s3path, filepath=filepath)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _filesystem_get(filesystem: s3fs.S3FileSystem, s3path: str, filepath: Path):
|
|
76
|
+
"""Helper to download a file from S3 with a progress bar."""
|
|
77
|
+
info = filesystem.info(s3path)
|
|
78
|
+
size = info.get("size") or info.get("Size")
|
|
79
|
+
|
|
80
|
+
callback = TqdmCallback(
|
|
81
|
+
size=size,
|
|
82
|
+
tqdm_kwargs=dict(
|
|
83
|
+
desc=f"Downloading {Path(s3path).name}",
|
|
84
|
+
unit="B",
|
|
85
|
+
unit_scale=True,
|
|
86
|
+
unit_divisor=1024,
|
|
87
|
+
dynamic_ncols=True,
|
|
88
|
+
leave=True,
|
|
89
|
+
mininterval=0.2,
|
|
90
|
+
smoothing=0.1,
|
|
91
|
+
miniters=1,
|
|
92
|
+
bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
|
|
93
|
+
"[{elapsed}<{remaining}, {rate_fmt}]",
|
|
94
|
+
),
|
|
95
|
+
)
|
|
96
|
+
filesystem.get(s3path, str(filepath), callback=callback)
|
|
97
|
+
return filepath
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def load_eeg_from_s3(s3path: str):
|
|
101
|
+
"""Load EEG data from an S3 URI into an ``xarray.DataArray``.
|
|
102
|
+
|
|
103
|
+
Preserves the original filename, downloads sidecar files when applicable
|
|
104
|
+
(e.g., ``.fdt`` for EEGLAB, ``.vmrk``/``.eeg`` for BrainVision), and uses
|
|
105
|
+
MNE's direct readers.
|
|
106
|
+
|
|
107
|
+
Parameters
|
|
108
|
+
----------
|
|
109
|
+
s3path : str
|
|
110
|
+
An S3 URI (should start with "s3://").
|
|
111
|
+
|
|
112
|
+
Returns
|
|
113
|
+
-------
|
|
114
|
+
xr.DataArray
|
|
115
|
+
EEG data with dimensions ``("channel", "time")``.
|
|
116
|
+
|
|
117
|
+
Raises
|
|
118
|
+
------
|
|
119
|
+
ValueError
|
|
120
|
+
If the file extension is unsupported.
|
|
121
|
+
|
|
122
|
+
"""
|
|
123
|
+
filesystem = get_s3_filesystem()
|
|
124
|
+
# choose a temp dir so sidecars can be colocated
|
|
125
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
126
|
+
# Derive local filenames from the S3 key to keep base name consistent
|
|
127
|
+
s3_key = urlsplit(s3path).path # e.g., "/dsXXXX/sub-.../..._eeg.set"
|
|
128
|
+
basename = Path(s3_key).name
|
|
129
|
+
ext = Path(basename).suffix.lower()
|
|
130
|
+
local_main = Path(tmpdir) / basename
|
|
131
|
+
|
|
132
|
+
# Download main file
|
|
133
|
+
with (
|
|
134
|
+
filesystem.open(s3path, mode="rb") as fsrc,
|
|
135
|
+
open(local_main, "wb") as fdst,
|
|
136
|
+
):
|
|
137
|
+
fdst.write(fsrc.read())
|
|
138
|
+
|
|
139
|
+
# Determine and fetch any required sidecars
|
|
140
|
+
sidecars: list[str] = []
|
|
141
|
+
if ext == ".set": # EEGLAB
|
|
142
|
+
sidecars = [".fdt"]
|
|
143
|
+
elif ext == ".vhdr": # BrainVision
|
|
144
|
+
sidecars = [".vmrk", ".eeg", ".dat", ".raw"]
|
|
145
|
+
|
|
146
|
+
for sc_ext in sidecars:
|
|
147
|
+
sc_key = s3_key[: -len(ext)] + sc_ext
|
|
148
|
+
sc_uri = f"s3://{urlsplit(s3path).netloc}{sc_key}"
|
|
149
|
+
try:
|
|
150
|
+
# If sidecar exists, download next to the main file
|
|
151
|
+
info = filesystem.info(sc_uri)
|
|
152
|
+
if info:
|
|
153
|
+
sc_local = Path(tmpdir) / Path(sc_key).name
|
|
154
|
+
with (
|
|
155
|
+
filesystem.open(sc_uri, mode="rb") as fsrc,
|
|
156
|
+
open(sc_local, "wb") as fdst,
|
|
157
|
+
):
|
|
158
|
+
fdst.write(fsrc.read())
|
|
159
|
+
except Exception:
|
|
160
|
+
# Sidecar not present; skip silently
|
|
161
|
+
pass
|
|
162
|
+
|
|
163
|
+
# Read using appropriate MNE reader
|
|
164
|
+
raw = mne.io.read_raw(str(local_main), preload=True, verbose=False)
|
|
165
|
+
|
|
166
|
+
data = raw.get_data()
|
|
167
|
+
fs = raw.info["sfreq"]
|
|
168
|
+
max_time = data.shape[1] / fs
|
|
169
|
+
time_steps = np.linspace(0, max_time, data.shape[1]).squeeze()
|
|
170
|
+
channel_names = raw.ch_names
|
|
171
|
+
|
|
172
|
+
return xr.DataArray(
|
|
173
|
+
data=data,
|
|
174
|
+
dims=["channel", "time"],
|
|
175
|
+
coords={"time": time_steps, "channel": channel_names},
|
|
176
|
+
)
|