Braindecode 1.3.2.dev182321725__tar.gz → 1.3.2.dev182822510__tar.gz

{braindecode-1.3.2.dev182321725/braindecode.egg-info → braindecode-1.3.2.dev182822510}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braindecode
-Version: 1.3.2.dev182321725
+Version: 1.3.2.dev182822510
 Summary: Deep learning software to decode EEG, ECG or MEG signals
 Author-email: Robin Tibor Schirrmeister <robintibor@gmail.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Alexandre Gramfort <agramfort@meta.com>
 Maintainer-email: Alexandre Gramfort <agramfort@meta.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Robin Tibor Schirrmeister <robintibor@gmail.com>

{braindecode-1.3.2.dev182321725 → braindecode-1.3.2.dev182822510}/braindecode/datasets/base.py

@@ -617,6 +617,15 @@ class WindowsDataset(RecordDataset):
         last_target_only: bool = True,
     ):
         super().__init__(description, transform)
+        self._fast_disk = self._can_use_fast_get_epoch_from_raw(windows)
+        if not (self._fast_disk or windows.preload):
+            warnings.warn(
+                "The provided mne.Epochs object does not meet the requirements for "
+                "fast epoch access. This may lead to slow data loading from disk. "
+                "Consider preloading the epochs or checking the conditions in "
+                "WindowsDataset._can_use_fast_get_epoch_from_raw.",
+                UserWarning,
+            )
         self.windows = windows
         self.last_target_only = last_target_only
         if targets_from not in ("metadata", "channels"):
@@ -633,6 +642,20 @@ class WindowsDataset(RecordDataset):
         self.raw_preproc_kwargs: list[dict[str, Any]] = []
         self.window_preproc_kwargs: list[dict[str, Any]] = []
 
+    @staticmethod
+    def _can_use_fast_get_epoch_from_raw(epochs: mne.BaseEpochs) -> bool:
+        """Check if we can use the fast _get_epoch_from_raw method,
+        or if we need to use the slow get_data method."""
+        return (
+            not epochs.preload
+            and epochs._bad_dropped
+            and epochs.detrend is None
+            and not epochs._do_baseline  # baseline is None
+            and (epochs._decim == 1)
+            and epochs._offset is None
+            and epochs._projector is None
+        )
+
     def __getitem__(self, index: int):
         """Get a window and its target.
 
@@ -650,7 +673,10 @@ class WindowsDataset(RecordDataset):
         np.ndarray
             Crop indices.
         """
-        X = self.windows.get_data(item=index)[0].astype("float32")
+        if self._fast_disk:
+            X = self.windows._get_epoch_from_raw(index).astype("float32")
+        else:
+            X = self.windows.get_data(item=index)[0].astype("float32")
         if self.transform is not None:
             X = self.transform(X)
         if self.targets_from == "metadata":

{braindecode-1.3.2.dev182321725 → braindecode-1.3.2.dev182822510}/braindecode/datasets/bids/hub.py

@@ -27,6 +27,7 @@ The format follows a BIDS-inspired sourcedata structure:
 #
 # License: BSD (3-clause)
 
+import contextlib
 import json
 import logging
 import tempfile
@@ -69,6 +70,8 @@ huggingface_hub = _soft_import(
 
 log = logging.getLogger(__name__)
 
+_LOCK_FILE = "format_info.json"
+
 
 class HubDatasetMixin:
     """
@@ -84,7 +87,6 @@ class HubDatasetMixin:
     >>> dataset = NMT(path=path, preload=True)
     >>> dataset.push_to_hub(
     ...     repo_id="username/nmt-dataset",
-    ...     commit_message="Add NMT dataset"
     ... )
     >>>
     >>> # Load dataset from Hub
@@ -96,13 +98,14 @@ class HubDatasetMixin:
     def push_to_hub(
         self,
         repo_id: str,
-        commit_message: Optional[str] = None,
         private: bool = False,
         token: Optional[str] = None,
-        create_pr: bool = False,
         compression: str = "blosc",
         compression_level: int = 5,
         pipeline_name: str = "braindecode",
+        chunk_size: int = 10000,
+        local_cache_dir: str | Path | None = None,
+        **kwargs,
     ) -> str:
         """
         Upload the dataset to the Hugging Face Hub in BIDS-like Zarr format.
@@ -116,20 +119,38 @@ class HubDatasetMixin:
         ----------
         repo_id : str
             Repository ID on the Hugging Face Hub (e.g., "username/dataset-name").
-        commit_message : str | None
-            Commit message. If None, a default message is generated.
         private : bool, default=False
             Whether to create a private repository.
         token : str | None
             Hugging Face API token. If None, uses cached token.
-        create_pr : bool, default=False
-            Whether to create a Pull Request instead of directly committing.
         compression : str, default="blosc"
             Compression algorithm for Zarr. Options: "blosc", "zstd", "gzip", None.
         compression_level : int, default=5
             Compression level (0-9). Level 5 provides optimal balance.
         pipeline_name : str, default="braindecode"
             Name of the processing pipeline for BIDS sourcedata.
+        chunk_size : int, default=10000
+            Number of samples per chunk in Zarr. Larger chunk size will create fewer
+            but larger files. Only used for RawDataset and EEGWindowsDataset (continuous data).
+            With WindowsDataset (pre-cut epochs), each window is saved as a separate chunk.
+        local_cache_dir : str | Path | None
+            Local directory to use for temporary files during upload. If None, uses
+            the system temp directory and cleans it up after upload. If provided,
+            the directory is used as a persistent cache:
+
+            - If the directory is empty (or does not exist), the cache is built
+              there and a lock file (``format_info.json``) is written once
+              the cache is complete, before the upload starts. The file
+              contains the zarr conversion parameters as JSON.
+            - If the lock file is present and its JSON parameters match the
+              current call, cache creation is skipped and the upload resumes
+              directly (useful for retrying interrupted uploads).
+            - If the lock file is present but its JSON parameters differ from
+              the current call, a ``ValueError`` is raised.
+            - If the directory is non-empty but the lock file is absent, a
+              ``ValueError`` is raised listing the files found.
+        **kwargs
+            Additional arguments passed to huggingface_hub.upload_large_folder().
 
         Returns
         -------
@@ -149,7 +170,6 @@ class HubDatasetMixin:
         >>> # Upload with BIDS-like structure
         >>> url = dataset.push_to_hub(
         ...     repo_id="myusername/nmt-dataset",
-        ...     commit_message="Upload NMT EEG dataset"
         ... )
         """
         if huggingface_hub is False or zarr is False:
@@ -159,13 +179,12 @@ class HubDatasetMixin:
             )
 
         # Create API instance
-        _ = huggingface_hub.HfApi(token=token)
+        hf_api = huggingface_hub.HfApi(token=token)
 
         # Create repository if it doesn't exist
         try:
-            huggingface_hub.create_repo(
+            hf_api.create_repo(
                 repo_id=repo_id,
-                token=token,
                 private=private,
                 repo_type="dataset",
                 exist_ok=True,
@@ -173,74 +192,68 @@ class HubDatasetMixin:
         except Exception as e:
             raise RuntimeError(f"Failed to create repository: {e}")
 
-        # Create a temporary directory for upload
-        with tempfile.TemporaryDirectory() as tmpdir:
-            tmp_path = Path(tmpdir)
-
-            # Create BIDS-like sourcedata structure
-            log.info("Creating BIDS-like sourcedata structure...")
-            bids_layout = hub_format.BIDSSourcedataLayout(
-                tmp_path, pipeline_name=pipeline_name
-            )
-            sourcedata_dir = bids_layout.create_structure()
-
-            # Save dataset_description.json
-            bids_layout.save_dataset_description()
-
-            # Save participants.tsv
-            descriptions = [ds.description for ds in self.datasets]
-            bids_layout.save_participants(descriptions)
-
-            # Save BIDS sidecar files for each recording
-            self._save_bids_sidecar_files(bids_layout)
-
-            # Convert dataset to Zarr format inside sourcedata
-            log.info("Converting dataset to Zarr format...")
-            dataset_path = sourcedata_dir / "dataset.zarr"
-
-            self._convert_to_zarr_inline(
-                dataset_path,
-                compression,
-                compression_level,
-            )
-
-            # Save dataset metadata (README.md)
-            self._save_dataset_card(tmp_path)
-
-            # Save format info
-            format_info_path = tmp_path / "format_info.json"
-            with open(format_info_path, "w", encoding="utf-8") as f:
-                format_info = self._get_format_info_inline()
-                json.dump(
-                    {
-                        "format": "zarr",
-                        "pipeline_name": pipeline_name,
-                        "compression": compression,
-                        "compression_level": compression_level,
-                        "braindecode_version": braindecode.__version__,
-                        **format_info,
-                    },
-                    f,
-                    indent=2,
-                )
+        format_info = self._get_format_info_inline()
+        format_info_lock = {
+            "format": "zarr",
+            "pipeline_name": pipeline_name,
+            "compression": compression,
+            "compression_level": compression_level,
+            "chunk_size": chunk_size,
+            "braindecode_version": braindecode.__version__,
+            **format_info,
+        }
 
-            # Default commit message
-            if commit_message is None:
-                commit_message = (
-                    f"Upload EEG dataset in BIDS-like "
-                    f"Zarr format ({len(self.datasets)} recordings)"
-                )
+        # Determine upload directory and whether to build the cache
+        with contextlib.ExitStack() as stack:
+            if local_cache_dir is None:
+                tmpdir = stack.enter_context(tempfile.TemporaryDirectory())
+                tmp_path = Path(tmpdir)
+                build_cache = True
+            else:
+                tmp_path = Path(local_cache_dir)
+                lock_path = tmp_path / _LOCK_FILE
+                if lock_path.exists():
+                    with open(lock_path, "r", encoding="utf-8") as _f:
+                        _lock_params = json.load(_f)
+                    if _lock_params != format_info_lock:
+                        raise ValueError(
+                            f"Lock file found at '{lock_path}' but its "
+                            f"parameters {_lock_params} differ from the "
+                            f"current call parameters {format_info_lock}. "
+                            "Provide an empty directory or match the "
+                            "original parameters."
+                        )
+                    log.info(
+                        f"Lock file found at '{lock_path}', skipping cache "
+                        "creation and resuming upload."
+                    )
+                    build_cache = False
+                else:
+                    if tmp_path.exists():
+                        existing = list(tmp_path.iterdir())
+                        if existing:
+                            entries = ", ".join(p.name for p in existing)
+                            raise ValueError(
+                                f"local_cache_dir '{tmp_path}' is not empty and "
+                                f"has no lock file. Found: {entries}. Provide an "
+                                "empty directory or one previously prepared by "
+                                "push_to_hub()."
+                            )
+                    else:
+                        tmp_path.mkdir(parents=True)
+                    build_cache = True
+
+            if build_cache:
+                self._build_local_cache(tmp_path, format_info_lock)
 
             # Upload folder to Hub
             log.info(f"Uploading to Hugging Face Hub ({repo_id})...")
             try:
-                url = huggingface_hub.upload_folder(
+                url = hf_api.upload_large_folder(
                     repo_id=repo_id,
                     folder_path=str(tmp_path),
                     repo_type="dataset",
-                    commit_message=commit_message,
-                    token=token,
-                    create_pr=create_pr,
+                    **kwargs,
                 )
                 log.info(f"Dataset uploaded successfully to {repo_id}")
                 log.info(f"URL: https://huggingface.co/datasets/{repo_id}")
@@ -248,6 +261,55 @@ class HubDatasetMixin:
             except Exception as e:
                 raise RuntimeError(f"Failed to upload dataset: {e}")
 
+    def _build_local_cache(
+        self,
+        tmp_path,
+        format_info_lock,
+    ):
+        """Build the local cache directory with the dataset in Zarr format and BIDS-like structure.
+        This folder will be uploaded to the Hub"""
+        compression = format_info_lock["compression"]
+        compression_level = format_info_lock["compression_level"]
+        pipeline_name = format_info_lock["pipeline_name"]
+        chunk_size = format_info_lock["chunk_size"]
+
+        # Create BIDS-like sourcedata structure
+        log.info("Creating BIDS-like sourcedata structure...")
+        bids_layout = hub_format.BIDSSourcedataLayout(
+            tmp_path, pipeline_name=pipeline_name
+        )
+        sourcedata_dir = bids_layout.create_structure()
+
+        # Save dataset_description.json
+        bids_layout.save_dataset_description()
+
+        # Save participants.tsv
+        descriptions = [ds.description for ds in self.datasets]
+        bids_layout.save_participants(descriptions)
+
+        # Save BIDS sidecar files for each recording
+        self._save_bids_sidecar_files(bids_layout)
+
+        # Convert dataset to Zarr format inside sourcedata
+        log.info("Converting dataset to Zarr format...")
+        dataset_path = sourcedata_dir / "dataset.zarr"
+
+        self._convert_to_zarr_inline(
+            dataset_path,
+            compression,
+            compression_level,
+            chunk_size,
+        )
+
+        # Save dataset metadata (README.md)
+        self._save_dataset_card(tmp_path)
+
+        # Save format info
+        # This marks the cache as complete
+        format_info_path = tmp_path / _LOCK_FILE
+        with open(format_info_path, "w", encoding="utf-8") as f:
+            json.dump(format_info_lock, f, indent=2)
+
     def _save_dataset_card(self, path: Path, bids_inspired: bool = True) -> None:
         """Generate and save a dataset card (README.md) with metadata.
 
@@ -465,7 +527,7 @@ class HubDatasetMixin:
             )
 
             # Load format info
-            format_info_path = Path(dataset_dir) / "format_info.json"
+            format_info_path = Path(dataset_dir) / _LOCK_FILE
             if format_info_path.exists():
                 with open(format_info_path, "r") as f:
                     format_info = json.load(f)
@@ -595,6 +657,7 @@ class HubDatasetMixin:
         output_path: Path,
         compression: str,
         compression_level: int,
+        chunk_size: int = 10000,
     ) -> None:
         """Convert dataset to Zarr format (inline implementation)."""
 
@@ -682,6 +745,7 @@ class HubDatasetMixin:
                     targets_from,
                     last_target_only,
                     compressor,
+                    chunk_size,
                 )
 
             elif dataset_type == "RawDataset":
@@ -693,7 +757,13 @@ class HubDatasetMixin:
 
                 # Save using inlined function
                 _save_raw_to_zarr(
-                    grp, raw, description, info_dict, target_name, compressor
+                    grp,
+                    raw,
+                    description,
+                    info_dict,
+                    target_name,
+                    compressor,
+                    chunk_size,
                 )
 
     def _get_format_info_inline(self):

{braindecode-1.3.2.dev182321725 → braindecode-1.3.2.dev182822510}/braindecode/datasets/bids/hub_io.py

@@ -70,7 +70,15 @@ def _save_windows_to_zarr(
 
 
 def _save_eegwindows_to_zarr(
-    grp, raw, metadata, description, info, targets_from, last_target_only, compressor
+    grp,
+    raw,
+    metadata,
+    description,
+    info,
+    targets_from,
+    last_target_only,
+    compressor,
+    chunk_size,
 ):
     """Save EEG continuous raw data to Zarr group (low-level function)."""
     continuous_data = raw.get_data()
@@ -80,7 +88,7 @@ def _save_eegwindows_to_zarr(
     grp.create_array(
         "data",
         data=continuous_float,
-        chunks=(continuous_float.shape[0], min(10000, continuous_float.shape[1])),
+        chunks=(continuous_float.shape[0], min(chunk_size, continuous_float.shape[1])),
         compressors=compressors_list,
     )
 
@@ -143,7 +151,7 @@ def _load_eegwindows_from_zarr(grp, preload):
     return data, metadata, description, info_dict, targets_from, last_target_only
 
 
-def _save_raw_to_zarr(grp, raw, description, info, target_name, compressor):
+def _save_raw_to_zarr(grp, raw, description, info, target_name, compressor, chunk_size):
     """Save RawDataset continuous raw data to Zarr group (low-level function)."""
     continuous_data = raw.get_data()
     continuous_float = continuous_data.astype(np.float32)
@@ -152,7 +160,7 @@ def _save_raw_to_zarr(grp, raw, description, info, target_name, compressor):
     grp.create_array(
         "data",
         data=continuous_float,
-        chunks=(continuous_float.shape[0], min(10000, continuous_float.shape[1])),
+        chunks=(continuous_float.shape[0], min(chunk_size, continuous_float.shape[1])),
         compressors=compressors_list,
     )
 

{braindecode-1.3.2.dev182321725 → braindecode-1.3.2.dev182822510}/braindecode/datasets/mne.py

@@ -100,6 +100,11 @@ def create_from_mne_epochs(
     window_size_samples: int,
     window_stride_samples: int,
     drop_last_window: bool,
+    descriptions: list[dict | pd.Series] | None = None,
+    mapping: dict[str | int, int] | None = None,
+    preload: bool = False,
+    picks: str | list | slice | None = None,
+    drop_bad_windows: bool = True,
 ) -> BaseConcatDataset:
     """Create WindowsDatasets from mne.Epochs.
 
@@ -114,6 +119,25 @@ def create_from_mne_epochs(
     drop_last_window : bool
         whether or not have a last overlapping window, when
         windows do not equally divide the continuous signal
+    descriptions : array-like | None
+        list of dicts or pandas.Series with additional information about
+        the epochs. If None, no description is added. Length must match
+        ``list_of_epochs``. Each description is applied to all windows
+        generated from the corresponding Epochs object.
+    mapping : dict(str | int: int) | None
+        Mapping from event description to target value. Keys can be
+        integers (matching ``epochs.events[:, 2]``) or their string
+        representations. If None, targets are set to the raw integer
+        event codes. If a mapping is provided and an event code is not
+        found in the mapping, the original event code is kept as-is.
+    preload : bool
+        if True, preload the data of the Epochs objects.
+    picks : str | list | slice | None
+        channels to include. If None, all available channels are used.
+    drop_bad_windows : bool
+        If True, call `.drop_bad()` on the resulting mne.Epochs object.
+        This allows identifying windows that fall outside of the valid
+        signal range. Defaults to True.
 
     Returns
     -------
@@ -126,8 +150,15 @@ def create_from_mne_epochs(
 
     _check_windowing_arguments(0, 0, window_size_samples, window_stride_samples)
 
+    if descriptions is not None:
+        if len(descriptions) != len(list_of_epochs):
+            raise ValueError(
+                f"length of 'list_of_epochs' ({len(list_of_epochs)}) and "
+                f"'descriptions' ({len(descriptions)}) has to match"
+            )
+
     list_of_windows_ds = []
-    for epochs in list_of_epochs:
+    for i, epochs in enumerate(list_of_epochs):
         event_descriptions = epochs.events[:, 2]
         original_trial_starts = epochs.events[:, 0]
         stop = len(epochs.times) - window_size_samples
@@ -142,6 +173,15 @@ def create_from_mne_epochs(
         fake_events = [[start, window_size_samples, -1] for start in starts]
 
         for trial_i, trial in enumerate(epochs):
+            # resolve target using mapping if provided
+            event_code = event_descriptions[trial_i]
+            if mapping is not None:
+                target = mapping.get(
+                    event_code, mapping.get(str(event_code), event_code)
+                )
+            else:
+                target = event_code
+
             metadata = pd.DataFrame(
                 {
                     "i_window_in_trial": np.arange(len(fake_events)),
@@ -149,7 +189,7 @@ def create_from_mne_epochs(
                     "i_stop_in_trial": starts
                     + original_trial_starts[trial_i]
                     + window_size_samples,
-                    "target": len(fake_events) * [event_descriptions[trial_i]],
+                    "target": len(fake_events) * [target],
                 }
             )
             # window size - 1, since tmax is inclusive
@@ -160,11 +200,15 @@ def create_from_mne_epochs(
                 tmin=0,
                 tmax=(window_size_samples - 1) / epochs.info["sfreq"],
                 metadata=metadata,
+                preload=preload,
+                picks=picks,
             )
 
-            mne_epochs.drop_bad(reject=None, flat=None)
+            if drop_bad_windows:
+                mne_epochs.drop_bad(reject=None, flat=None)
 
-            windows_ds = WindowsDataset(mne_epochs)
+            desc = descriptions[i] if descriptions is not None else None
+            windows_ds = WindowsDataset(mne_epochs, description=desc)
             list_of_windows_ds.append(windows_ds)
 
     return BaseConcatDataset(list_of_windows_ds)

{braindecode-1.3.2.dev182321725 → braindecode-1.3.2.dev182822510}/braindecode/datautil/hub_formats.py

@@ -36,6 +36,7 @@ def convert_to_zarr(
     output_path: Union[str, Path],
     compression: str = "blosc",
     compression_level: int = 5,
+    chunk_size: int = 10000,
     overwrite: bool = False,
 ) -> Path:
     """Convert BaseConcatDataset to Zarr format.
@@ -58,6 +59,10 @@ def convert_to_zarr(
         blosc uses zstd codec by default, providing best balance of speed and compression.
     compression_level : int, default=5
         Compression level (0-9). Level 5 provides optimal balance based on benchmarks.
+    chunk_size : int, default=10000
+        Number of samples per chunk in Zarr. Larger chunk size will create fewer
+        but larger files. Only used for RawDataset and EEGWindowsDataset (continuous data).
+        With WindowsDataset (pre-cut epochs), each window is saved as a separate chunk.
     overwrite : bool, default=False
         Whether to overwrite existing directory.
 
@@ -96,7 +101,9 @@ def convert_to_zarr(
         shutil.rmtree(output_path)
 
     # Delegate to HubDatasetMixin method
-    dataset._convert_to_zarr_inline(output_path, compression, compression_level)
+    dataset._convert_to_zarr_inline(
+        output_path, compression, compression_level, chunk_size
+    )
 
     return output_path