eegdash 0.3.9.dev114__py3-none-any.whl → 0.3.9.dev129__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of eegdash might be problematic. Click here for more details.

eegdash/data_utils.py CHANGED
@@ -1,10 +1,8 @@
1
1
  import io
2
2
  import json
3
- import logging
4
3
  import os
5
4
  import re
6
5
  import traceback
7
- import warnings
8
6
  from contextlib import redirect_stderr
9
7
  from pathlib import Path
10
8
  from typing import Any
@@ -13,9 +11,7 @@ import mne
13
11
  import mne_bids
14
12
  import numpy as np
15
13
  import pandas as pd
16
- import s3fs
17
14
  from bids import BIDSLayout
18
- from fsspec.callbacks import TqdmCallback
19
15
  from joblib import Parallel, delayed
20
16
  from mne._fiff.utils import _read_segments_file
21
17
  from mne.io import BaseRaw
@@ -23,10 +19,11 @@ from mne_bids import BIDSPath
23
19
 
24
20
  from braindecode.datasets import BaseDataset
25
21
 
22
+ from . import downloader
23
+ from .bids_eeg_metadata import enrich_from_participants
24
+ from .logging import logger
26
25
  from .paths import get_default_cache_dir
27
26
 
28
- logger = logging.getLogger("eegdash")
29
-
30
27
 
31
28
  class EEGDashBaseDataset(BaseDataset):
32
29
  """A single EEG recording hosted on AWS S3 and cached locally upon first access.
@@ -73,6 +70,7 @@ class EEGDashBaseDataset(BaseDataset):
73
70
  # Compute a dataset folder name under cache_dir that encodes preprocessing
74
71
  # (e.g., bdf, mini) to avoid overlapping with the original dataset cache.
75
72
  self.dataset_folder = record.get("dataset", "")
73
+ # TODO: remove this hack when competition is over
76
74
  if s3_bucket:
77
75
  suffixes: list[str] = []
78
76
  bucket_lower = str(s3_bucket).lower()
@@ -91,6 +89,7 @@ class EEGDashBaseDataset(BaseDataset):
91
89
  rel = Path(self.dataset_folder) / rel
92
90
  self.filecache = self.cache_dir / rel
93
91
  self.bids_root = self.cache_dir / self.dataset_folder
92
+
94
93
  self.bidspath = BIDSPath(
95
94
  root=self.bids_root,
96
95
  datatype="eeg",
@@ -98,113 +97,18 @@ class EEGDashBaseDataset(BaseDataset):
98
97
  **self.bids_kwargs,
99
98
  )
100
99
 
101
- self.s3file = self._get_s3path(record["bidspath"])
100
+ self.s3file = downloader.get_s3path(self.s3_bucket, record["bidspath"])
102
101
  self.bids_dependencies = record["bidsdependencies"]
103
- # Temporary fix for BIDS dependencies path
104
- # just to release to the competition
102
+ self.bids_dependencies_original = record["bidsdependencies"]
103
+ # TODO: removing temporary fix for BIDS dependencies path
104
+ # when the competition is over and dataset is digested properly
105
105
  if not self.s3_open_neuro:
106
- self.bids_dependencies_original = self.bids_dependencies
107
106
  self.bids_dependencies = [
108
107
  dep.split("/", 1)[1] for dep in self.bids_dependencies
109
108
  ]
110
109
 
111
110
  self._raw = None
112
111
 
113
- def _get_s3path(self, filepath: str) -> str:
114
- """Helper to form an AWS S3 URI for the given relative filepath."""
115
- return f"{self.s3_bucket}/{filepath}"
116
-
117
- def _download_s3(self) -> None:
118
- """Download function that gets the raw EEG data from S3."""
119
- filesystem = s3fs.S3FileSystem(
120
- anon=True, client_kwargs={"region_name": "us-east-2"}
121
- )
122
- if not self.s3_open_neuro:
123
- self.s3file = re.sub(r"(^|/)ds\d{6}/", r"\1", self.s3file, count=1)
124
- if self.s3file.endswith(".set"):
125
- self.s3file = self.s3file[:-4] + ".bdf"
126
- self.filecache = self.filecache.with_suffix(".bdf")
127
-
128
- self.filecache.parent.mkdir(parents=True, exist_ok=True)
129
- info = filesystem.info(self.s3file)
130
- size = info.get("size") or info.get("Size")
131
-
132
- callback = TqdmCallback(
133
- size=size,
134
- tqdm_kwargs=dict(
135
- desc=f"Downloading {Path(self.s3file).name}",
136
- unit="B",
137
- unit_scale=True,
138
- unit_divisor=1024,
139
- dynamic_ncols=True,
140
- leave=True,
141
- mininterval=0.2,
142
- smoothing=0.1,
143
- miniters=1,
144
- bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
145
- "[{elapsed}<{remaining}, {rate_fmt}]",
146
- ),
147
- )
148
- filesystem.get(self.s3file, self.filecache, callback=callback)
149
-
150
- self.filenames = [self.filecache]
151
-
152
- def _download_dependencies(self) -> None:
153
- """Download all BIDS dependency files (metadata files, recording sidecar files)
154
- from S3 and cache them locally.
155
- """
156
- filesystem = s3fs.S3FileSystem(
157
- anon=True, client_kwargs={"region_name": "us-east-2"}
158
- )
159
- for i, dep in enumerate(self.bids_dependencies):
160
- if not self.s3_open_neuro:
161
- # fix this when our bucket is integrated into the
162
- # mongodb
163
- # if the file have ".set" replace to ".bdf"
164
- if dep.endswith(".set"):
165
- dep = dep[:-4] + ".bdf"
166
-
167
- s3path = self._get_s3path(dep)
168
- if not self.s3_open_neuro:
169
- dep = self.bids_dependencies_original[i]
170
-
171
- dep_path = Path(dep)
172
- if dep_path.parts and dep_path.parts[0] == self.record.get("dataset"):
173
- dep_local = Path(self.dataset_folder, *dep_path.parts[1:])
174
- else:
175
- dep_local = Path(self.dataset_folder) / dep_path
176
- filepath = self.cache_dir / dep_local
177
- if not self.s3_open_neuro:
178
- if filepath.suffix == ".set":
179
- filepath = filepath.with_suffix(".bdf")
180
- if self.filecache.suffix == ".set":
181
- self.filecache = self.filecache.with_suffix(".bdf")
182
-
183
- # here, we download the dependency and it is fine
184
- # in the case of the competition.
185
- if not filepath.exists():
186
- filepath.parent.mkdir(parents=True, exist_ok=True)
187
- info = filesystem.info(s3path)
188
- size = info.get("size") or info.get("Size")
189
-
190
- callback = TqdmCallback(
191
- size=size,
192
- tqdm_kwargs=dict(
193
- desc=f"Downloading {Path(s3path).name}",
194
- unit="B",
195
- unit_scale=True,
196
- unit_divisor=1024,
197
- dynamic_ncols=True,
198
- leave=True,
199
- mininterval=0.2,
200
- smoothing=0.1,
201
- miniters=1,
202
- bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
203
- "[{elapsed}<{remaining}, {rate_fmt}]",
204
- ),
205
- )
206
- filesystem.get(s3path, filepath, callback=callback)
207
-
208
112
  def _get_raw_bids_args(self) -> dict[str, Any]:
209
113
  """Helper to restrict the metadata record to the fields needed to locate a BIDS
210
114
  recording.
@@ -222,130 +126,43 @@ class EEGDashBaseDataset(BaseDataset):
222
126
 
223
127
  if not os.path.exists(self.filecache): # not preload
224
128
  if self.bids_dependencies:
225
- self._download_dependencies()
226
- self._download_s3()
129
+ downloader.download_dependencies(
130
+ s3_bucket=self.s3_bucket,
131
+ bids_dependencies=self.bids_dependencies,
132
+ bids_dependencies_original=self.bids_dependencies_original,
133
+ cache_dir=self.cache_dir,
134
+ dataset_folder=self.dataset_folder,
135
+ record=self.record,
136
+ s3_open_neuro=self.s3_open_neuro,
137
+ )
138
+ self.filecache = downloader.download_s3_file(
139
+ self.s3file, self.filecache, self.s3_open_neuro
140
+ )
141
+ self.filenames = [self.filecache]
227
142
  if self._raw is None:
228
- # capturing any warnings
229
- # to-do: remove this once is fixed on the mne-bids side.
230
- with warnings.catch_warnings(record=True) as w:
231
- # Ensure all warnings are captured into 'w' and not shown to users
232
- warnings.simplefilter("always")
233
- try:
234
- # mne-bids emits RuntimeWarnings to stderr; silence stderr during read
235
- _stderr_buffer = io.StringIO()
236
- with redirect_stderr(_stderr_buffer):
237
- self._raw = mne_bids.read_raw_bids(
238
- bids_path=self.bidspath, verbose="ERROR"
239
- )
240
- # Parse unmapped participants.tsv fields reported by mne-bids and
241
- # inject them into Raw.info and the dataset description generically.
242
- extras = self._extract_unmapped_participants_from_warnings(w)
243
- if extras:
244
- # 1) Attach to Raw.info under subject_info.participants_extras
245
- try:
246
- subject_info = self._raw.info.get("subject_info") or {}
247
- if not isinstance(subject_info, dict):
248
- subject_info = {}
249
- pe = subject_info.get("participants_extras") or {}
250
- if not isinstance(pe, dict):
251
- pe = {}
252
- # Merge without overwriting
253
- for k, v in extras.items():
254
- pe.setdefault(k, v)
255
- subject_info["participants_extras"] = pe
256
- self._raw.info["subject_info"] = subject_info
257
- except Exception:
258
- # Non-fatal; continue
259
- pass
260
-
261
- # 2) Also add to this dataset's description, if possible, so
262
- # targets can be selected later without naming specifics.
263
- try:
264
- if isinstance(self.description, dict):
265
- for k, v in extras.items():
266
- self.description.setdefault(k, v)
267
- elif isinstance(self.description, pd.Series):
268
- for k, v in extras.items():
269
- if k not in self.description.index:
270
- self.description.loc[k] = v
271
- except Exception:
272
- pass
273
- except Exception as e:
274
- logger.error(
275
- f"Error while reading BIDS file: {self.bidspath}\n"
276
- "This may be due to a missing or corrupted file.\n"
277
- "Please check the file and try again."
278
- )
279
- logger.error(f"Exception: {e}")
280
- logger.error(traceback.format_exc())
281
- raise e
282
- # Filter noisy mapping notices from mne-bids; surface others
283
- for captured_warning in w:
284
- try:
285
- msg = str(captured_warning.message)
286
- except Exception:
287
- continue
288
- # Suppress verbose participants mapping messages
289
- if "Unable to map the following column" in msg and "MNE" in msg:
290
- logger.debug(
291
- "Suppressed mne-bids mapping warning while reading BIDS file: %s",
292
- msg,
293
- )
294
- continue
295
-
296
- def _extract_unmapped_participants_from_warnings(
297
- self, warnings_list: list[Any]
298
- ) -> dict[str, Any]:
299
- """Scan captured warnings from mne-bids and extract unmapped participants.tsv
300
- entries in a generic way.
301
-
302
- Optionally, the column name can carry a note in parentheses that we ignore
303
- for key/value extraction. Returns a mapping of column name -> raw value.
304
- """
305
- extras: dict[str, Any] = {}
306
- header = "Unable to map the following column(s) to MNE:"
307
- for wr in warnings_list:
308
- try:
309
- msg = str(wr.message)
310
- except Exception:
311
- continue
312
- if header not in msg:
313
- continue
314
- lines = msg.splitlines()
315
- # Find the header line, then parse subsequent lines as entries
316
143
  try:
317
- idx = next(i for i, ln in enumerate(lines) if header in ln)
318
- except StopIteration:
319
- idx = -1
320
- for line in lines[idx + 1 :]:
321
- line = line.strip()
322
- if not line:
323
- continue
324
- # Pattern: <col>(optional note): <value>
325
- # Examples: "gender: F", "Ethnicity: Indian", "foo (ignored): bar"
326
- m = re.match(r"^([^:]+?)(?:\s*\([^)]*\))?\s*:\s*(.*)$", line)
327
- if not m:
328
- continue
329
- col = m.group(1).strip()
330
- val = m.group(2).strip()
331
- # Keep original column names as provided to stay agnostic
332
- if col and col not in extras:
333
- extras[col] = val
334
- return extras
335
-
336
- # === BaseDataset and PyTorch Dataset interface ===
337
-
338
- def __getitem__(self, index):
339
- """Main function to access a sample from the dataset."""
340
- X = self.raw[:, index][0]
341
- y = None
342
- if self.target_name is not None:
343
- y = self.description[self.target_name]
344
- if isinstance(y, pd.Series):
345
- y = y.to_list()
346
- if self.transform is not None:
347
- X = self.transform(X)
348
- return X, y
144
+ # mne-bids can emit noisy warnings to stderr; keep user logs clean
145
+ _stderr_buffer = io.StringIO()
146
+ with redirect_stderr(_stderr_buffer):
147
+ self._raw = mne_bids.read_raw_bids(
148
+ bids_path=self.bidspath, verbose="ERROR"
149
+ )
150
+ # Enrich Raw.info and description with participants.tsv extras
151
+ enrich_from_participants(
152
+ self.bids_root, self.bidspath, self._raw, self.description
153
+ )
154
+
155
+ except Exception as e:
156
+ logger.error(
157
+ f"Error while reading BIDS file: {self.bidspath}\n"
158
+ "This may be due to a missing or corrupted file.\n"
159
+ "Please check the file and try again.\n"
160
+ "Usually erasing the local cache and re-downloading helps.\n"
161
+ f"`rm {self.bidspath}`"
162
+ )
163
+ logger.error(f"Exception: {e}")
164
+ logger.error(traceback.format_exc())
165
+ raise e
349
166
 
350
167
  def __len__(self) -> int:
351
168
  """Return the number of samples in the dataset."""
@@ -426,13 +243,16 @@ class EEGDashBaseRaw(BaseRaw):
426
243
  ch_types.append(chtype)
427
244
  info = mne.create_info(ch_names=ch_names, sfreq=sfreq, ch_types=ch_types)
428
245
 
429
- self.s3file = self._get_s3path(input_fname)
246
+ self.s3file = downloader.get_s3path(self._AWS_BUCKET, input_fname)
430
247
  self.cache_dir = Path(cache_dir) if cache_dir else get_default_cache_dir()
431
248
  self.filecache = self.cache_dir / input_fname
432
249
  self.bids_dependencies = bids_dependencies
433
250
 
434
251
  if preload and not os.path.exists(self.filecache):
435
- self._download_s3()
252
+ self.filecache = downloader.download_s3_file(
253
+ self.s3file, self.filecache, self.s3_open_neuro
254
+ )
255
+ self.filenames = [self.filecache]
436
256
  preload = self.filecache
437
257
 
438
258
  super().__init__(
@@ -443,35 +263,24 @@ class EEGDashBaseRaw(BaseRaw):
443
263
  verbose=verbose,
444
264
  )
445
265
 
446
- def _get_s3path(self, filepath):
447
- return f"{self._AWS_BUCKET}/{filepath}"
448
-
449
- def _download_s3(self) -> None:
450
- self.filecache.parent.mkdir(parents=True, exist_ok=True)
451
- filesystem = s3fs.S3FileSystem(
452
- anon=True, client_kwargs={"region_name": "us-east-2"}
453
- )
454
- filesystem.download(self.s3file, self.filecache)
455
- self.filenames = [self.filecache]
456
-
457
- def _download_dependencies(self):
458
- filesystem = s3fs.S3FileSystem(
459
- anon=True, client_kwargs={"region_name": "us-east-2"}
460
- )
461
- for dep in self.bids_dependencies:
462
- s3path = self._get_s3path(dep)
463
- filepath = self.cache_dir / dep
464
- if not filepath.exists():
465
- filepath.parent.mkdir(parents=True, exist_ok=True)
466
- filesystem.download(s3path, filepath)
467
-
468
266
  def _read_segment(
469
267
  self, start=0, stop=None, sel=None, data_buffer=None, *, verbose=None
470
268
  ):
471
269
  if not os.path.exists(self.filecache): # not preload
472
- if self.bids_dependencies:
473
- self._download_dependencies()
474
- self._download_s3()
270
+ if self.bids_dependencies: # this is use only to sidecars for now
271
+ downloader.download_dependencies(
272
+ s3_bucket=self._AWS_BUCKET,
273
+ bids_dependencies=self.bids_dependencies,
274
+ bids_dependencies_original=None,
275
+ cache_dir=self.cache_dir,
276
+ dataset_folder=self.filecache,
277
+ record={},
278
+ s3_open_neuro=self.s3_open_neuro,
279
+ )
280
+ self.filecache = downloader.download_s3_file(
281
+ self.s3file, self.filecache, self.s3_open_neuro
282
+ )
283
+ self.filenames = [self.filecache]
475
284
  else: # not preload and file is not cached
476
285
  self.filenames = [self.filecache]
477
286
  return super()._read_segment(start, stop, sel, data_buffer, verbose=verbose)
@@ -1,15 +1,15 @@
1
- import logging
2
1
  from pathlib import Path
3
2
 
4
- from mne.utils import warn
3
+ from rich.console import Console
4
+ from rich.panel import Panel
5
+ from rich.text import Text
5
6
 
6
7
  from ..api import EEGDashDataset
7
8
  from ..bids_eeg_metadata import build_query_from_kwargs
8
9
  from ..const import RELEASE_TO_OPENNEURO_DATASET_MAP, SUBJECT_MINI_RELEASE_MAP
10
+ from ..logging import logger
9
11
  from .registry import register_openneuro_datasets
10
12
 
11
- logger = logging.getLogger("eegdash")
12
-
13
13
 
14
14
  class EEGChallengeDataset(EEGDashDataset):
15
15
  """EEG 2025 Challenge dataset helper.
@@ -23,8 +23,6 @@ class EEGChallengeDataset(EEGDashDataset):
23
23
  ----------
24
24
  release : str
25
25
  Release name. One of ["R1", ..., "R11"].
26
- cache_dir : str
27
- Local cache directory for data files.
28
26
  mini : bool, default True
29
27
  If True, restrict subjects to the challenge mini subset.
30
28
  query : dict | None
@@ -123,24 +121,32 @@ class EEGChallengeDataset(EEGDashDataset):
123
121
  else:
124
122
  s3_bucket = f"{s3_bucket}/{release}_L100_bdf"
125
123
 
126
- warn(
127
- "\n\n"
128
- "[EEGChallengeDataset] EEG 2025 Competition Data Notice:\n"
129
- "-------------------------------------------------------\n"
124
+ message_text = Text.from_markup(
130
125
  "This object loads the HBN dataset that has been preprocessed for the EEG Challenge:\n"
131
- " - Downsampled from 500Hz to 100Hz\n"
132
- " - Bandpass filtered (0.5–50 Hz)\n"
133
- "\n"
134
- "For full preprocessing details, see:\n"
135
- " https://github.com/eeg2025/downsample-datasets\n"
136
- "\n"
137
- "IMPORTANT: The data accessed via `EEGChallengeDataset` is NOT identical to what you get from `EEGDashDataset` directly.\n"
138
- "If you are participating in the competition, always use `EEGChallengeDataset` to ensure consistency with the challenge data.\n"
139
- "\n",
140
- UserWarning,
141
- module="eegdash",
126
+ " Downsampled from 500Hz to 100Hz\n"
127
+ " Bandpass filtered (0.5–50 Hz)\n\n"
128
+ "For full preprocessing applied for competition details, see:\n"
129
+ " [link=https://github.com/eeg2025/downsample-datasets]https://github.com/eeg2025/downsample-datasets[/link]\n\n"
130
+ "The HBN dataset have some preprocessing applied by the HBN team:\n"
131
+ " • Re-reference (Cz Channel)\n\n"
132
+ "[bold red]IMPORTANT[/bold red]: The data accessed via `EEGChallengeDataset` is [u]NOT[/u] identical to what you get from [link=https://github.com/sccn/EEGDash/blob/develop/eegdash/api.py]EEGDashDataset[/link] directly.\n"
133
+ "If you are participating in the competition, always use `EEGChallengeDataset` to ensure consistency with the challenge data."
134
+ )
135
+
136
+ warning_panel = Panel(
137
+ message_text,
138
+ title="[yellow]EEG 2025 Competition Data Notice[/yellow]",
139
+ subtitle="[cyan]Source: EEGChallengeDataset[/cyan]",
140
+ border_style="yellow",
142
141
  )
143
142
 
143
+ # Render the panel directly to the console so it displays in IPython/terminals
144
+ try:
145
+ Console().print(warning_panel)
146
+ except Exception:
147
+ warning_message = str(message_text)
148
+ logger.warning(warning_message)
149
+
144
150
  super().__init__(
145
151
  dataset=RELEASE_TO_OPENNEURO_DATASET_MAP[release],
146
152
  query=query,
eegdash/downloader.py ADDED
@@ -0,0 +1,176 @@
1
+ import re
2
+ import tempfile
3
+ from pathlib import Path
4
+ from typing import Any
5
+ from urllib.parse import urlsplit
6
+
7
+ import mne
8
+ import numpy as np
9
+ import s3fs
10
+ import xarray as xr
11
+ from fsspec.callbacks import TqdmCallback
12
+
13
+
14
+ def get_s3_filesystem():
15
+ """Returns an S3FileSystem object."""
16
+ return s3fs.S3FileSystem(anon=True, client_kwargs={"region_name": "us-east-2"})
17
+
18
+
19
+ def get_s3path(s3_bucket: str, filepath: str) -> str:
20
+ """Helper to form an AWS S3 URI for the given relative filepath."""
21
+ return f"{s3_bucket}/{filepath}"
22
+
23
+
24
+ def download_s3_file(s3_path: str, local_path: Path, s3_open_neuro: bool):
25
+ """Download function that gets the raw EEG data from S3."""
26
+ filesystem = get_s3_filesystem()
27
+ if not s3_open_neuro:
28
+ s3_path = re.sub(r"(^|/)ds\d{6}/", r"\1", s3_path, count=1)
29
+ # TODO: remove this hack when competition is over
30
+ if s3_path.endswith(".set"):
31
+ s3_path = s3_path[:-4] + ".bdf"
32
+ local_path = local_path.with_suffix(".bdf")
33
+
34
+ local_path.parent.mkdir(parents=True, exist_ok=True)
35
+ _filesystem_get(filesystem=filesystem, s3path=s3_path, filepath=local_path)
36
+
37
+ return local_path
38
+
39
+
40
+ def download_dependencies(
41
+ s3_bucket: str,
42
+ bids_dependencies: list[str],
43
+ bids_dependencies_original: list[str],
44
+ cache_dir: Path,
45
+ dataset_folder: Path,
46
+ record: dict[str, Any],
47
+ s3_open_neuro: bool,
48
+ ):
49
+ """Download all BIDS dependency files from S3 and cache them locally."""
50
+ filesystem = get_s3_filesystem()
51
+ for i, dep in enumerate(bids_dependencies):
52
+ if not s3_open_neuro:
53
+ if dep.endswith(".set"):
54
+ dep = dep[:-4] + ".bdf"
55
+
56
+ s3path = get_s3path(s3_bucket, dep)
57
+ if not s3_open_neuro:
58
+ dep = bids_dependencies_original[i]
59
+
60
+ dep_path = Path(dep)
61
+ if dep_path.parts and dep_path.parts[0] == record.get("dataset"):
62
+ dep_local = Path(dataset_folder, *dep_path.parts[1:])
63
+ else:
64
+ dep_local = Path(dataset_folder) / dep_path
65
+ filepath = cache_dir / dep_local
66
+ if not s3_open_neuro:
67
+ if filepath.suffix == ".set":
68
+ filepath = filepath.with_suffix(".bdf")
69
+
70
+ if not filepath.exists():
71
+ filepath.parent.mkdir(parents=True, exist_ok=True)
72
+ _filesystem_get(filesystem=filesystem, s3path=s3path, filepath=filepath)
73
+
74
+
75
+ def _filesystem_get(filesystem: s3fs.S3FileSystem, s3path: str, filepath: Path):
76
+ """Helper to download a file from S3 with a progress bar."""
77
+ info = filesystem.info(s3path)
78
+ size = info.get("size") or info.get("Size")
79
+
80
+ callback = TqdmCallback(
81
+ size=size,
82
+ tqdm_kwargs=dict(
83
+ desc=f"Downloading {Path(s3path).name}",
84
+ unit="B",
85
+ unit_scale=True,
86
+ unit_divisor=1024,
87
+ dynamic_ncols=True,
88
+ leave=True,
89
+ mininterval=0.2,
90
+ smoothing=0.1,
91
+ miniters=1,
92
+ bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
93
+ "[{elapsed}<{remaining}, {rate_fmt}]",
94
+ ),
95
+ )
96
+ filesystem.get(s3path, str(filepath), callback=callback)
97
+ return filepath
98
+
99
+
100
+ def load_eeg_from_s3(s3path: str):
101
+ """Load EEG data from an S3 URI into an ``xarray.DataArray``.
102
+
103
+ Preserves the original filename, downloads sidecar files when applicable
104
+ (e.g., ``.fdt`` for EEGLAB, ``.vmrk``/``.eeg`` for BrainVision), and uses
105
+ MNE's direct readers.
106
+
107
+ Parameters
108
+ ----------
109
+ s3path : str
110
+ An S3 URI (should start with "s3://").
111
+
112
+ Returns
113
+ -------
114
+ xr.DataArray
115
+ EEG data with dimensions ``("channel", "time")``.
116
+
117
+ Raises
118
+ ------
119
+ ValueError
120
+ If the file extension is unsupported.
121
+
122
+ """
123
+ filesystem = get_s3_filesystem()
124
+ # choose a temp dir so sidecars can be colocated
125
+ with tempfile.TemporaryDirectory() as tmpdir:
126
+ # Derive local filenames from the S3 key to keep base name consistent
127
+ s3_key = urlsplit(s3path).path # e.g., "/dsXXXX/sub-.../..._eeg.set"
128
+ basename = Path(s3_key).name
129
+ ext = Path(basename).suffix.lower()
130
+ local_main = Path(tmpdir) / basename
131
+
132
+ # Download main file
133
+ with (
134
+ filesystem.open(s3path, mode="rb") as fsrc,
135
+ open(local_main, "wb") as fdst,
136
+ ):
137
+ fdst.write(fsrc.read())
138
+
139
+ # Determine and fetch any required sidecars
140
+ sidecars: list[str] = []
141
+ if ext == ".set": # EEGLAB
142
+ sidecars = [".fdt"]
143
+ elif ext == ".vhdr": # BrainVision
144
+ sidecars = [".vmrk", ".eeg", ".dat", ".raw"]
145
+
146
+ for sc_ext in sidecars:
147
+ sc_key = s3_key[: -len(ext)] + sc_ext
148
+ sc_uri = f"s3://{urlsplit(s3path).netloc}{sc_key}"
149
+ try:
150
+ # If sidecar exists, download next to the main file
151
+ info = filesystem.info(sc_uri)
152
+ if info:
153
+ sc_local = Path(tmpdir) / Path(sc_key).name
154
+ with (
155
+ filesystem.open(sc_uri, mode="rb") as fsrc,
156
+ open(sc_local, "wb") as fdst,
157
+ ):
158
+ fdst.write(fsrc.read())
159
+ except Exception:
160
+ # Sidecar not present; skip silently
161
+ pass
162
+
163
+ # Read using appropriate MNE reader
164
+ raw = mne.io.read_raw(str(local_main), preload=True, verbose=False)
165
+
166
+ data = raw.get_data()
167
+ fs = raw.info["sfreq"]
168
+ max_time = data.shape[1] / fs
169
+ time_steps = np.linspace(0, max_time, data.shape[1]).squeeze()
170
+ channel_names = raw.ch_names
171
+
172
+ return xr.DataArray(
173
+ data=data,
174
+ dims=["channel", "time"],
175
+ coords={"time": time_steps, "channel": channel_names},
176
+ )