eegdash 0.3.9.dev182388821__py3-none-any.whl → 0.4.0.dev132__py3-none-any.whl

eegdash/__init__.py

@@ -7,4 +7,4 @@ _init_mongo_client()
 
 __all__ = ["EEGDash", "EEGDashDataset", "EEGChallengeDataset", "preprocessing"]
 
-__version__ = "0.3.9.dev182388821"
+__version__ = "0.4.0.dev132"

eegdash/api.py

@@ -1,9 +1,6 @@
-import logging
 import os
-import tempfile
 from pathlib import Path
 from typing import Any, Mapping
-from urllib.parse import urlsplit
 
 import mne
 import numpy as np
@@ -11,13 +8,15 @@ import xarray as xr
 from docstring_inheritance import NumpyDocstringInheritanceInitMeta
 from dotenv import load_dotenv
 from joblib import Parallel, delayed
-from mne.utils import warn
 from mne_bids import find_matching_paths, get_bids_path_from_fname, read_raw_bids
 from pymongo import InsertOne, UpdateOne
-from s3fs import S3FileSystem
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
 
 from braindecode.datasets import BaseConcatDataset
 
+from . import downloader
 from .bids_eeg_metadata import (
     build_query_from_kwargs,
     load_eeg_attrs_from_bids_file,
@@ -33,10 +32,10 @@ from .data_utils import (
     EEGBIDSDataset,
     EEGDashBaseDataset,
 )
+from .logging import logger
 from .mongodb import MongoConnectionManager
 from .paths import get_default_cache_dir
-
-logger = logging.getLogger("eegdash")
+from .utils import _init_mongo_client
 
 
 class EEGDash:
@@ -74,19 +73,26 @@ class EEGDash:
 
         if self.is_public:
             DB_CONNECTION_STRING = mne.utils.get_config("EEGDASH_DB_URI")
+            if not DB_CONNECTION_STRING:
+                try:
+                    _init_mongo_client()
+                    DB_CONNECTION_STRING = mne.utils.get_config("EEGDASH_DB_URI")
+                except Exception:
+                    DB_CONNECTION_STRING = None
         else:
             load_dotenv()
             DB_CONNECTION_STRING = os.getenv("DB_CONNECTION_STRING")
 
         # Use singleton to get MongoDB client, database, and collection
+        if not DB_CONNECTION_STRING:
+            raise RuntimeError(
+                "No MongoDB connection string configured. Set MNE config 'EEGDASH_DB_URI' "
+                "or environment variable 'DB_CONNECTION_STRING'."
+            )
         self.__client, self.__db, self.__collection = MongoConnectionManager.get_client(
             DB_CONNECTION_STRING, is_staging
         )
 
-        self.filesystem = S3FileSystem(
-            anon=True, client_kwargs={"region_name": "us-east-2"}
-        )
-
     def find(
         self, query: dict[str, Any] = None, /, **kwargs
     ) -> list[Mapping[str, Any]]:
@@ -310,83 +316,6 @@ class EEGDash:
                         f"Conflicting constraints for '{key}': disjoint sets {r_val!r} and {k_val!r}"
                     )
 
-    def load_eeg_data_from_s3(self, s3path: str) -> xr.DataArray:
-        """Load EEG data from an S3 URI into an ``xarray.DataArray``.
-
-        Preserves the original filename, downloads sidecar files when applicable
-        (e.g., ``.fdt`` for EEGLAB, ``.vmrk``/``.eeg`` for BrainVision), and uses
-        MNE's direct readers.
-
-        Parameters
-        ----------
-        s3path : str
-            An S3 URI (should start with "s3://").
-
-        Returns
-        -------
-        xr.DataArray
-            EEG data with dimensions ``("channel", "time")``.
-
-        Raises
-        ------
-        ValueError
-            If the file extension is unsupported.
-
-        """
-        # choose a temp dir so sidecars can be colocated
-        with tempfile.TemporaryDirectory() as tmpdir:
-            # Derive local filenames from the S3 key to keep base name consistent
-            s3_key = urlsplit(s3path).path  # e.g., "/dsXXXX/sub-.../..._eeg.set"
-            basename = Path(s3_key).name
-            ext = Path(basename).suffix.lower()
-            local_main = Path(tmpdir) / basename
-
-            # Download main file
-            with (
-                self.filesystem.open(s3path, mode="rb") as fsrc,
-                open(local_main, "wb") as fdst,
-            ):
-                fdst.write(fsrc.read())
-
-            # Determine and fetch any required sidecars
-            sidecars: list[str] = []
-            if ext == ".set":  # EEGLAB
-                sidecars = [".fdt"]
-            elif ext == ".vhdr":  # BrainVision
-                sidecars = [".vmrk", ".eeg", ".dat", ".raw"]
-
-            for sc_ext in sidecars:
-                sc_key = s3_key[: -len(ext)] + sc_ext
-                sc_uri = f"s3://{urlsplit(s3path).netloc}{sc_key}"
-                try:
-                    # If sidecar exists, download next to the main file
-                    info = self.filesystem.info(sc_uri)
-                    if info:
-                        sc_local = Path(tmpdir) / Path(sc_key).name
-                        with (
-                            self.filesystem.open(sc_uri, mode="rb") as fsrc,
-                            open(sc_local, "wb") as fdst,
-                        ):
-                            fdst.write(fsrc.read())
-                except Exception:
-                    # Sidecar not present; skip silently
-                    pass
-
-            # Read using appropriate MNE reader
-            raw = mne.io.read_raw(str(local_main), preload=True, verbose=False)
-
-            data = raw.get_data()
-            fs = raw.info["sfreq"]
-            max_time = data.shape[1] / fs
-            time_steps = np.linspace(0, max_time, data.shape[1]).squeeze()
-            channel_names = raw.ch_names
-
-            return xr.DataArray(
-                data=data,
-                dims=["channel", "time"],
-                coords={"time": time_steps, "channel": channel_names},
-            )
-
     def load_eeg_data_from_bids_file(self, bids_file: str) -> xr.DataArray:
         """Load EEG data from a local BIDS-formatted file.
 
@@ -508,39 +437,13 @@ class EEGDash:
             results = Parallel(
                 n_jobs=-1 if len(sessions) > 1 else 1, prefer="threads", verbose=1
             )(
-                delayed(self.load_eeg_data_from_s3)(self._get_s3path(session))
+                delayed(downloader.load_eeg_from_s3)(
+                    downloader.get_s3path("s3://openneuro.org", session["bidspath"])
+                )
                 for session in sessions
             )
         return results
 
-    def _get_s3path(self, record: Mapping[str, Any] | str) -> str:
-        """Build an S3 URI from a DB record or a relative path.
-
-        Parameters
-        ----------
-        record : dict or str
-            Either a DB record containing a ``'bidspath'`` key, or a relative
-            path string under the OpenNeuro bucket.
-
-        Returns
-        -------
-        str
-            Fully qualified S3 URI.
-
-        Raises
-        ------
-        ValueError
-            If a mapping is provided but ``'bidspath'`` is missing.
-
-        """
-        if isinstance(record, str):
-            rel = record
-        else:
-            rel = record.get("bidspath")
-            if not rel:
-                raise ValueError("Record missing 'bidspath' for S3 path resolution")
-        return f"s3://openneuro.org/{rel}"
-
     def _add_request(self, record: dict):
         """Internal helper method to create a MongoDB insertion request for a record."""
         return InsertOne(record)
@@ -552,8 +455,11 @@ class EEGDash:
         except ValueError as e:
             logger.error("Validation error for record: %s ", record["data_name"])
             logger.error(e)
-        except:
-            logger.error("Error adding record: %s ", record["data_name"])
+        except Exception as exc:
+            logger.error(
+                "Error adding record: %s ", record.get("data_name", "<unknown>")
+            )
+            logger.debug("Add operation failed", exc_info=exc)
 
     def _update_request(self, record: dict):
         """Internal helper method to create a MongoDB update request for a record."""
@@ -572,8 +478,11 @@ class EEGDash:
             self.__collection.update_one(
                 {"data_name": record["data_name"]}, {"$set": record}
             )
-        except:  # silent failure
-            logger.error("Error updating record: %s", record["data_name"])
+        except Exception as exc:  # log and continue
+            logger.error(
+                "Error updating record: %s", record.get("data_name", "<unknown>")
+            )
+            logger.debug("Update operation failed", exc_info=exc)
 
     def exists(self, query: dict[str, Any]) -> bool:
         """Alias for :meth:`exist` provided for API clarity."""
@@ -654,8 +563,7 @@ class EEGDashDataset(BaseConcatDataset, metaclass=NumpyDocstringInheritanceInitM
     Parameters
     ----------
     cache_dir : str | Path
-        Directory where data are cached locally. If not specified, a default
-        cache directory under the user cache is used.
+        Directory where data are cached locally.
     query : dict | None
         Raw MongoDB query to filter records. If provided, it is merged with
         keyword filtering arguments (see ``**kwargs``) using logical AND.
@@ -726,13 +634,21 @@ class EEGDashDataset(BaseConcatDataset, metaclass=NumpyDocstringInheritanceInitM
         self.records = records
         self.download = download
         self.n_jobs = n_jobs
-        self.eeg_dash_instance = eeg_dash_instance or EEGDash()
+        self.eeg_dash_instance = eeg_dash_instance
 
-        # Resolve a unified cache directory across code/tests/CI
-        self.cache_dir = Path(cache_dir or get_default_cache_dir())
+        self.cache_dir = cache_dir
+        if self.cache_dir == "" or self.cache_dir is None:
+            self.cache_dir = get_default_cache_dir()
+            logger.warning(
+                f"Cache directory is empty, using the eegdash default path: {self.cache_dir}"
+            )
+
+        self.cache_dir = Path(self.cache_dir)
 
         if not self.cache_dir.exists():
-            warn(f"Cache directory does not exist, creating it: {self.cache_dir}")
+            logger.warning(
+                f"Cache directory does not exist, creating it: {self.cache_dir}"
+            )
             self.cache_dir.mkdir(exist_ok=True, parents=True)
 
         # Separate query kwargs from other kwargs passed to the BaseDataset constructor
@@ -772,21 +688,29 @@ class EEGDashDataset(BaseConcatDataset, metaclass=NumpyDocstringInheritanceInitM
             not _suppress_comp_warning
             and self.query["dataset"] in RELEASE_TO_OPENNEURO_DATASET_MAP.values()
         ):
-            warn(
-                "If you are not participating in the competition, you can ignore this warning!"
-                "\n\n"
-                "EEG 2025 Competition Data Notice:\n"
-                "---------------------------------\n"
-                " You are loading the dataset that is used in the EEG 2025 Competition:\n"
-                "IMPORTANT: The data accessed via `EEGDashDataset` is NOT identical to what you get from `EEGChallengeDataset` object directly.\n"
-                "and it is not what you will use for the competition. Downsampling and filtering were applied to the data"
-                "to allow more people to participate.\n"
-                "\n"
-                "If you are participating in the competition, always use `EEGChallengeDataset` to ensure consistency with the challenge data.\n"
-                "\n",
-                UserWarning,
-                module="eegdash",
+            message_text = Text.from_markup(
+                "[italic]This notice is only for users who are participating in the [link=https://eeg2025.github.io/]EEG 2025 Competition[/link].[/italic]\n\n"
+                "[bold]EEG 2025 Competition Data Notice![/bold]\n"
+                "You are loading one of the datasets that is used in competition, but via `EEGDashDataset`.\n\n"
+                "[bold red]IMPORTANT[/bold red]: \n"
+                "If you download data from `EEGDashDataset`, it is [u]NOT[/u] identical to the official \n"
+                "competition data, which is accessed via `EEGChallengeDataset`. "
+                "The competition data has been downsampled and filtered.\n\n"
+                "[bold]If you are participating in the competition, \nyou must use the `EEGChallengeDataset` object to ensure consistency.[/bold] \n\n"
+                "If you are not participating in the competition, you can ignore this message."
             )
+            warning_panel = Panel(
+                message_text,
+                title="[yellow]EEG 2025 Competition Data Notice[/yellow]",
+                subtitle="[cyan]Source: EEGDashDataset[/cyan]",
+                border_style="yellow",
+            )
+
+            try:
+                Console().print(warning_panel)
+            except Exception:
+                logger.warning(str(message_text))
+
         if records is not None:
             self.records = records
             datasets = [
@@ -848,16 +772,15 @@ class EEGDashDataset(BaseConcatDataset, metaclass=NumpyDocstringInheritanceInitM
                     )
                 )
         elif self.query:
-            # This is the DB query path that we are improving
+            if self.eeg_dash_instance is None:
+                self.eeg_dash_instance = EEGDash()
             datasets = self._find_datasets(
                 query=build_query_from_kwargs(**self.query),
                 description_fields=description_fields,
                 base_dataset_kwargs=base_dataset_kwargs,
             )
             # We only need filesystem if we need to access S3
-            self.filesystem = S3FileSystem(
-                anon=True, client_kwargs={"region_name": "us-east-2"}
-            )
+            self.filesystem = downloader.get_s3_filesystem()
         else:
             raise ValueError(
                 "You must provide either 'records', a 'data_dir', or a query/keyword arguments for filtering."

eegdash/bids_eeg_metadata.py

@@ -1,18 +1,23 @@
-import logging
 import re
 from pathlib import Path
 from typing import Any
 
+import pandas as pd
+from mne_bids import BIDSPath
+
 from .const import ALLOWED_QUERY_FIELDS
 from .const import config as data_config
-
-logger = logging.getLogger("eegdash")
+from .logging import logger
 
 __all__ = [
     "build_query_from_kwargs",
     "load_eeg_attrs_from_bids_file",
     "merge_participants_fields",
     "normalize_key",
+    "participants_row_for_subject",
+    "participants_extras_from_tsv",
+    "attach_participants_extras",
+    "enrich_from_participants",
 ]
 
 
@@ -72,28 +77,6 @@ def build_query_from_kwargs(**kwargs) -> dict[str, Any]:
     return query
 
 
-def _get_raw_extensions(bids_file: str, bids_dataset) -> list[str]:
-    """Helper to find paths to additional "sidecar" files that may be associated
-    with a given main data file in a BIDS dataset; paths are returned as relative to
-    the parent dataset path.
-
-    For example, if the input file is a .set file, this will return the relative path
-    to a corresponding .fdt file (if any).
-    """
-    bids_file = Path(bids_file)
-    extensions = {
-        ".set": [".set", ".fdt"],  # eeglab
-        ".edf": [".edf"],  # european
-        ".vhdr": [".eeg", ".vhdr", ".vmrk", ".dat", ".raw"],  # brainvision
-        ".bdf": [".bdf"],  # biosemi
-    }
-    return [
-        str(bids_dataset._get_relative_bidspath(bids_file.with_suffix(suffix)))
-        for suffix in extensions[bids_file.suffix]
-        if bids_file.with_suffix(suffix).exists()
-    ]
-
-
 def load_eeg_attrs_from_bids_file(bids_dataset, bids_file: str) -> dict[str, Any]:
     """Build the metadata record for a given BIDS file (single recording) in a BIDS dataset.
 
@@ -140,7 +123,7 @@ def load_eeg_attrs_from_bids_file(bids_dataset, bids_file: str) -> dict[str, Any
         eeg_json = None
 
     bids_dependencies_files = data_config["bids_dependencies_files"]
-    bidsdependencies = []
+    bidsdependencies: list[str] = []
     for extension in bids_dependencies_files:
         try:
             dep_path = bids_dataset.get_bids_metadata_files(bids_file, extension)
@@ -151,7 +134,26 @@ def load_eeg_attrs_from_bids_file(bids_dataset, bids_file: str) -> dict[str, Any
         except Exception:
             pass
 
-    bidsdependencies.extend(_get_raw_extensions(bids_file, bids_dataset))
+    bids_path = BIDSPath(
+        subject=bids_dataset.get_bids_file_attribute("subject", bids_file),
+        session=bids_dataset.get_bids_file_attribute("session", bids_file),
+        task=bids_dataset.get_bids_file_attribute("task", bids_file),
+        run=bids_dataset.get_bids_file_attribute("run", bids_file),
+        root=bids_dataset.bidsdir,
+        datatype=bids_dataset.get_bids_file_attribute("modality", bids_file),
+        suffix="eeg",
+        extension=Path(bids_file).suffix,
+        check=False,
+    )
+
+    sidecars_map = {
+        ".set": [".fdt"],
+        ".vhdr": [".eeg", ".vmrk", ".dat", ".raw"],
+    }
+    for ext in sidecars_map.get(bids_path.extension, []):
+        sidecar = bids_path.find_matching_sidecar(extension=ext, on_error="ignore")
+        if sidecar is not None:
+            bidsdependencies.append(str(bids_dataset._get_relative_bidspath(sidecar)))
 
     # Define field extraction functions with error handling
     field_extractors = {
@@ -252,3 +254,123 @@ def merge_participants_fields(
         if norm_key not in description:
             description[norm_key] = part_value
     return description
+
+
+def participants_row_for_subject(
+    bids_root: str | Path,
+    subject: str,
+    id_columns: tuple[str, ...] = ("participant_id", "participant", "subject"),
+) -> pd.Series | None:
+    """Load participants.tsv and return the row for a subject.
+
+    - Accepts either "01" or "sub-01" as the subject identifier.
+    - Returns a pandas Series for the first matching row, or None if not found.
+    """
+    try:
+        participants_tsv = Path(bids_root) / "participants.tsv"
+        if not participants_tsv.exists():
+            return None
+
+        df = pd.read_csv(
+            participants_tsv, sep="\t", dtype="string", keep_default_na=False
+        )
+        if df.empty:
+            return None
+
+        candidates = {str(subject), f"sub-{subject}"}
+        present_cols = [c for c in id_columns if c in df.columns]
+        if not present_cols:
+            return None
+
+        mask = pd.Series(False, index=df.index)
+        for col in present_cols:
+            mask |= df[col].isin(candidates)
+        match = df.loc[mask]
+        if match.empty:
+            return None
+        return match.iloc[0]
+    except Exception:
+        return None
+
+
+def participants_extras_from_tsv(
+    bids_root: str | Path,
+    subject: str,
+    *,
+    id_columns: tuple[str, ...] = ("participant_id", "participant", "subject"),
+    na_like: tuple[str, ...] = ("", "n/a", "na", "nan", "unknown", "none"),
+) -> dict[str, Any]:
+    """Return non-identifier, non-empty participants.tsv fields for a subject.
+
+    Uses vectorized pandas operations to drop id columns and NA-like values.
+    """
+    row = participants_row_for_subject(bids_root, subject, id_columns=id_columns)
+    if row is None:
+        return {}
+
+    # Drop identifier columns and clean values
+    extras = row.drop(labels=[c for c in id_columns if c in row.index], errors="ignore")
+    s = extras.astype("string").str.strip()
+    valid = ~s.isna() & ~s.str.lower().isin(na_like)
+    return s[valid].to_dict()
+
+
+def attach_participants_extras(
+    raw: Any,
+    description: Any,
+    extras: dict[str, Any],
+) -> None:
+    """Attach extras to Raw.info and dataset description without overwriting.
+
+    - Adds to ``raw.info['subject_info']['participants_extras']``.
+    - Adds to ``description`` if dict or pandas Series (only missing keys).
+    """
+    if not extras:
+        return
+
+    # Raw.info enrichment
+    try:
+        subject_info = raw.info.get("subject_info") or {}
+        if not isinstance(subject_info, dict):
+            subject_info = {}
+        pe = subject_info.get("participants_extras") or {}
+        if not isinstance(pe, dict):
+            pe = {}
+        for k, v in extras.items():
+            pe.setdefault(k, v)
+        subject_info["participants_extras"] = pe
+        raw.info["subject_info"] = subject_info
+    except Exception:
+        pass
+
+    # Description enrichment
+    try:
+        import pandas as _pd  # local import to avoid hard dependency at import time
+
+        if isinstance(description, dict):
+            for k, v in extras.items():
+                description.setdefault(k, v)
+        elif isinstance(description, _pd.Series):
+            missing = [k for k in extras.keys() if k not in description.index]
+            if missing:
+                description.loc[missing] = [extras[m] for m in missing]
+    except Exception:
+        pass
+
+
+def enrich_from_participants(
+    bids_root: str | Path,
+    bidspath: BIDSPath,
+    raw: Any,
+    description: Any,
+) -> dict[str, Any]:
+    """Convenience wrapper: read participants.tsv and attach extras for this subject.
+
+    Returns the extras dictionary for further use if needed.
+    """
+    subject = getattr(bidspath, "subject", None)
+    if not subject:
+        return {}
+    extras = participants_extras_from_tsv(bids_root, subject)
+    attach_participants_extras(raw, description, extras)
+    return extras