eegdash 0.3.9.dev170082126__py3-none-any.whl → 0.4.0__py3-none-any.whl

eegdash/dataset/registry.py

@@ -14,7 +14,35 @@ def register_openneuro_datasets(
     namespace: Dict[str, Any] | None = None,
     add_to_all: bool = True,
 ) -> Dict[str, type]:
-    """Dynamically create dataset classes from a summary file."""
+    """Dynamically create and register dataset classes from a summary file.
+
+    This function reads a CSV file containing summaries of OpenNeuro datasets
+    and dynamically creates a Python class for each dataset. These classes
+    inherit from a specified base class and are pre-configured with the
+    dataset's ID.
+
+    Parameters
+    ----------
+    summary_file : str or pathlib.Path
+        The path to the CSV file containing the dataset summaries.
+    base_class : type, optional
+        The base class from which the new dataset classes will inherit. If not
+        provided, :class:`eegdash.api.EEGDashDataset` is used.
+    namespace : dict, optional
+        The namespace (e.g., `globals()`) into which the newly created classes
+        will be injected. Defaults to the local `globals()` of this module.
+    add_to_all : bool, default True
+        If True, the names of the newly created classes are added to the
+        `__all__` list of the target namespace, making them importable with
+        `from ... import *`.
+
+    Returns
+    -------
+    dict[str, type]
+        A dictionary mapping the names of the registered classes to the class
+        types themselves.
+
+    """
     if base_class is None:
         from ..api import EEGDashDataset as base_class  # lazy import
 
@@ -57,14 +85,8 @@ def register_openneuro_datasets(
 
         init = make_init(dataset_id)
 
-        doc = f"""OpenNeuro dataset ``{dataset_id}``.
-
-        {_markdown_table(row_series)}
-
-        This class is a thin convenience wrapper for the dataset ``{dataset_id}``.
-        Constructor arguments are forwarded to :class:`{base_class.__name__}`; see the
-        base class documentation for parameter details and examples.
-        """
+        # Generate rich docstring with dataset metadata
+        doc = _generate_rich_docstring(dataset_id, row_series, base_class)
 
         # init.__doc__ = doc
 
@@ -90,8 +112,133 @@ def register_openneuro_datasets(
     return registered
 
 
+def _generate_rich_docstring(
+    dataset_id: str, row_series: pd.Series, base_class: type
+) -> str:
+    """Generate a comprehensive, well-formatted docstring for a dataset class.
+
+    Parameters
+    ----------
+    dataset_id : str
+        The identifier of the dataset (e.g., "ds002718").
+    row_series : pandas.Series
+        A pandas Series containing the metadata for the dataset, extracted
+        from the summary CSV file.
+    base_class : type
+        The base class from which the new dataset class inherits. Used to
+        generate the "See Also" section of the docstring.
+
+    Returns
+    -------
+    str
+        A formatted docstring.
+
+    """
+    # Extract metadata with safe defaults
+    n_subjects = row_series.get("n_subjects", "Unknown")
+    n_records = row_series.get("n_records", "Unknown")
+    n_tasks = row_series.get("n_tasks", "Unknown")
+    modality = row_series.get("modality of exp", "")
+    exp_type = row_series.get("type of exp", "")
+    subject_type = row_series.get("Type Subject", "")
+    duration = row_series.get("duration_hours_total", "Unknown")
+    size = row_series.get("size", "Unknown")
+
+    # Create description based on available metadata
+    description_parts = []
+    if modality and str(modality).strip():
+        description_parts.append(f"**Modality**: {modality}")
+    if exp_type and str(exp_type).strip():
+        description_parts.append(f"**Type**: {exp_type}")
+    if subject_type and str(subject_type).strip():
+        description_parts.append(f"**Subjects**: {subject_type}")
+
+    description = (
+        " | ".join(description_parts)
+        if description_parts
+        else "EEG dataset from OpenNeuro"
+    )
+
+    # Generate the docstring
+    docstring = f"""OpenNeuro dataset ``{dataset_id}``.
+
+{description}
+
+This dataset contains {n_subjects} subjects with {n_records} recordings across {n_tasks} tasks.
+Total duration: {duration} hours. Dataset size: {size}.
+
+{_markdown_table(row_series)}
+
+This dataset class provides convenient access to the ``{dataset_id}`` dataset through the EEGDash interface.
+It inherits all functionality from :class:`~{base_class.__module__}.{base_class.__name__}` with the dataset filter pre-configured.
+
+Parameters
+----------
+cache_dir : str
+    Directory to cache downloaded data.
+query : dict, optional
+    Additional MongoDB-style filters to AND with the dataset selection.
+    Must not contain the key ``dataset``.
+s3_bucket : str, optional
+    Base S3 bucket used to locate the data.
+**kwargs
+    Additional arguments passed to the base dataset class.
+
+Examples
+--------
+Basic usage:
+
+>>> from eegdash.dataset import {dataset_id.upper()}
+>>> dataset = {dataset_id.upper()}(cache_dir="./data")
+>>> print(f"Number of recordings: {{len(dataset)}}")
+
+Load a specific recording:
+
+>>> if len(dataset) > 0:
+...     recording = dataset[0]
+...     raw = recording.load()
+...     print(f"Sampling rate: {{raw.info['sfreq']}} Hz")
+...     print(f"Number of channels: {{len(raw.ch_names)}}")
+
+Filter by additional criteria:
+
+>>> # Get subset with specific task or subject
+>>> filtered_dataset = {dataset_id.upper()}(
+...     cache_dir="./data",
+...     query={{"task": "RestingState"}}  # if applicable
+... )
+
+Notes
+-----
+More details available in the `NEMAR documentation <https://nemar.org/dataexplorer/detail?dataset_id={dataset_id}>`__.
+
+See Also
+--------
+{base_class.__name__} : Base dataset class with full API documentation
+"""
+
+    return docstring
+
+
 def _markdown_table(row_series: pd.Series) -> str:
-    """Create a reStructuredText grid table from a pandas Series."""
+    """Create a reStructuredText grid table from a pandas Series.
+
+    This helper function takes a pandas Series containing dataset metadata
+    and formats it into a reStructuredText grid table for inclusion in
+    docstrings.
+
+    Parameters
+    ----------
+    row_series : pandas.Series
+        A Series where each index is a metadata field and each value is the
+        corresponding metadata value.
+
+    Returns
+    -------
+    str
+        A string containing the formatted reStructuredText table.
+
+    """
     if row_series.empty:
         return ""
     dataset_id = row_series["dataset"]
@@ -128,7 +275,12 @@ def _markdown_table(row_series: pd.Series) -> str:
     table = tabulate(df, headers="keys", tablefmt="rst", showindex=False)
 
     # Add a caption for the table
-    caption = f"Short overview of dataset {dataset_id} more details in the `Nemar documentation <https://nemar.org/dataexplorer/detail?dataset_id={dataset_id}>`_."
+    # Use an anonymous external link (double underscore) to avoid duplicate
+    # target warnings when this docstring is repeated across many classes.
+    caption = (
+        f"Short overview of dataset {dataset_id} more details in the "
+        f"`NeMAR documentation <https://nemar.org/dataexplorer/detail?dataset_id={dataset_id}>`__."
+    )
     # adding caption below the table
     # Indent the table to fit within the admonition block
     indented_table = "\n".join("    " + line for line in table.split("\n"))

eegdash/downloader.py

@@ -0,0 +1,197 @@
+# Authors: The EEGDash contributors.
+# License: GNU General Public License
+# Copyright the EEGDash contributors.
+
+"""File downloading utilities for EEG data from cloud storage.
+
+This module provides functions for downloading EEG data files and BIDS dependencies from
+AWS S3 storage, with support for caching and progress tracking. It handles the communication
+between the EEGDash metadata database and the actual EEG data stored in the cloud.
+"""
+
+import re
+from pathlib import Path
+from typing import Any
+
+import s3fs
+from fsspec.callbacks import TqdmCallback
+
+
+def get_s3_filesystem() -> s3fs.S3FileSystem:
+    """Get an anonymous S3 filesystem object.
+
+    Initializes and returns an ``s3fs.S3FileSystem`` for anonymous access
+    to public S3 buckets, configured for the 'us-east-2' region.
+
+    Returns
+    -------
+    s3fs.S3FileSystem
+        An S3 filesystem object.
+
+    """
+    return s3fs.S3FileSystem(anon=True, client_kwargs={"region_name": "us-east-2"})
+
+
+def get_s3path(s3_bucket: str, filepath: str) -> str:
+    """Construct an S3 URI from a bucket and file path.
+
+    Parameters
+    ----------
+    s3_bucket : str
+        The S3 bucket name (e.g., "s3://my-bucket").
+    filepath : str
+        The path to the file within the bucket.
+
+    Returns
+    -------
+    str
+        The full S3 URI (e.g., "s3://my-bucket/path/to/file").
+
+    """
+    return f"{s3_bucket}/{filepath}"
+
+
+def download_s3_file(s3_path: str, local_path: Path, s3_open_neuro: bool) -> Path:
+    """Download a single file from S3 to a local path.
+
+    Handles the download of a raw EEG data file from an S3 bucket, caching it
+    at the specified local path. Creates parent directories if they do not exist.
+
+    Parameters
+    ----------
+    s3_path : str
+        The full S3 URI of the file to download.
+    local_path : pathlib.Path
+        The local file path where the downloaded file will be saved.
+    s3_open_neuro : bool
+        A flag indicating if the S3 bucket is the OpenNeuro main bucket, which
+        may affect path handling.
+
+    Returns
+    -------
+    pathlib.Path
+        The local path to the downloaded file.
+
+    """
+    filesystem = get_s3_filesystem()
+    if not s3_open_neuro:
+        s3_path = re.sub(r"(^|/)ds\d{6}/", r"\1", s3_path, count=1)
+        # TODO: remove this hack when competition is over
+        if s3_path.endswith(".set"):
+            s3_path = s3_path[:-4] + ".bdf"
+            local_path = local_path.with_suffix(".bdf")
+
+    local_path.parent.mkdir(parents=True, exist_ok=True)
+    _filesystem_get(filesystem=filesystem, s3path=s3_path, filepath=local_path)
+
+    return local_path
+
+
+def download_dependencies(
+    s3_bucket: str,
+    bids_dependencies: list[str],
+    bids_dependencies_original: list[str],
+    cache_dir: Path,
+    dataset_folder: Path,
+    record: dict[str, Any],
+    s3_open_neuro: bool,
+) -> None:
+    """Download all BIDS dependency files from S3.
+
+    Iterates through a list of BIDS dependency files, downloads each from the
+    specified S3 bucket, and caches them in the appropriate local directory
+    structure.
+
+    Parameters
+    ----------
+    s3_bucket : str
+        The S3 bucket to download from.
+    bids_dependencies : list of str
+        A list of dependency file paths relative to the S3 bucket root.
+    bids_dependencies_original : list of str
+        The original dependency paths, used for resolving local cache paths.
+    cache_dir : pathlib.Path
+        The root directory for caching.
+    dataset_folder : pathlib.Path
+        The specific folder for the dataset within the cache directory.
+    record : dict
+        The metadata record for the main data file, used to resolve paths.
+    s3_open_neuro : bool
+        Flag for OpenNeuro-specific path handling.
+
+    """
+    filesystem = get_s3_filesystem()
+    for i, dep in enumerate(bids_dependencies):
+        if not s3_open_neuro:
+            if dep.endswith(".set"):
+                dep = dep[:-4] + ".bdf"
+
+        s3path = get_s3path(s3_bucket, dep)
+        if not s3_open_neuro:
+            dep = bids_dependencies_original[i]
+
+        dep_path = Path(dep)
+        if dep_path.parts and dep_path.parts[0] == record.get("dataset"):
+            dep_local = Path(dataset_folder, *dep_path.parts[1:])
+        else:
+            dep_local = Path(dataset_folder) / dep_path
+        filepath = cache_dir / dep_local
+        if not s3_open_neuro:
+            if filepath.suffix == ".set":
+                filepath = filepath.with_suffix(".bdf")
+
+        if not filepath.exists():
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+            _filesystem_get(filesystem=filesystem, s3path=s3path, filepath=filepath)
+
+
+def _filesystem_get(filesystem: s3fs.S3FileSystem, s3path: str, filepath: Path) -> Path:
+    """Perform the file download using fsspec with a progress bar.
+
+    Internal helper function that wraps the ``filesystem.get`` call to include
+    a TQDM progress bar.
+
+    Parameters
+    ----------
+    filesystem : s3fs.S3FileSystem
+        The filesystem object to use for the download.
+    s3path : str
+        The full S3 URI of the source file.
+    filepath : pathlib.Path
+        The local destination path.
+
+    Returns
+    -------
+    pathlib.Path
+        The local path to the downloaded file.
+
+    """
+    info = filesystem.info(s3path)
+    size = info.get("size") or info.get("Size")
+
+    callback = TqdmCallback(
+        size=size,
+        tqdm_kwargs=dict(
+            desc=f"Downloading {Path(s3path).name}",
+            unit="B",
+            unit_scale=True,
+            unit_divisor=1024,
+            dynamic_ncols=True,
+            leave=True,
+            mininterval=0.2,
+            smoothing=0.1,
+            miniters=1,
+            bar_format="{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} "
+            "[{elapsed}<{remaining}, {rate_fmt}]",
+        ),
+    )
+    filesystem.get(s3path, str(filepath), callback=callback)
+    return filepath
+
+
+__all__ = [
+    "download_s3_file",
+    "download_dependencies",
+    "get_s3path",
+    "get_s3_filesystem",
+]