Braindecode 1.3.2.dev169626090__tar.gz → 1.3.2.dev170355810__tar.gz

{braindecode-1.3.2.dev169626090/braindecode.egg-info → braindecode-1.3.2.dev170355810}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braindecode
-Version: 1.3.2.dev169626090
+Version: 1.3.2.dev170355810
 Summary: Deep learning software to decode EEG, ECG or MEG signals
 Author-email: Robin Tibor Schirrmeister <robintibor@gmail.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Alexandre Gramfort <agramfort@meta.com>
 Maintainer-email: Alexandre Gramfort <agramfort@meta.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Robin Tibor Schirrmeister <robintibor@gmail.com>

{braindecode-1.3.2.dev169626090 → braindecode-1.3.2.dev170355810}/braindecode/datasets/bids/hub.py

@@ -103,7 +103,7 @@ class HubDatasetMixin:
         compression: str = "blosc",
         compression_level: int = 5,
         pipeline_name: str = "braindecode",
-        chunk_size: int = 10000,
+        chunk_size: int = 5_000_000,
         local_cache_dir: str | Path | None = None,
         **kwargs,
     ) -> str:
@@ -129,10 +129,13 @@ class HubDatasetMixin:
             Compression level (0-9). Level 5 provides optimal balance.
         pipeline_name : str, default="braindecode"
             Name of the processing pipeline for BIDS sourcedata.
-        chunk_size : int, default=10000
-            Number of samples per chunk in Zarr. Larger chunk size will create fewer
-            but larger files. Only used for RawDataset and EEGWindowsDataset (continuous data).
-            With WindowsDataset (pre-cut epochs), each window is saved as a separate chunk.
+        chunk_size : int, default=5_000_000
+            Number of samples per chunk in Zarr along the time/window dimension.
+            Larger chunk sizes create fewer but larger chunks/files. This parameter
+            is used for both continuous data (e.g., RawDataset, EEGWindowsDataset)
+            and pre-cut windows (WindowsDataset). For WindowsDataset, multiple
+            windows may be stored in a single chunk depending on their duration
+            and the chosen ``chunk_size``.
         local_cache_dir : str | Path | None
             Local directory to use for temporary files during upload. If None, uses
             the system temp directory and cleans it up after upload. If provided,
@@ -657,7 +660,7 @@ class HubDatasetMixin:
         output_path: Path,
         compression: str,
         compression_level: int,
-        chunk_size: int = 10000,
+        chunk_size: int = 5_000_000,
     ) -> None:
         """Convert dataset to Zarr format (inline implementation)."""
 
@@ -723,7 +726,14 @@ class HubDatasetMixin:
 
                 # Save using inlined function
                 _save_windows_to_zarr(
-                    grp, data, metadata, description, info_dict, compressor, target_name
+                    grp,
+                    data,
+                    metadata,
+                    description,
+                    info_dict,
+                    compressor,
+                    target_name,
+                    chunk_size,
                 )
 
             elif dataset_type == "EEGWindowsDataset":

{braindecode-1.3.2.dev169626090 → braindecode-1.3.2.dev170355810}/braindecode/datasets/bids/hub_io.py

@@ -45,16 +45,25 @@ def _restore_nan_from_json(obj):
 
 
 def _save_windows_to_zarr(
-    grp, data, metadata, description, info, compressor, target_name
+    grp, data, metadata, description, info, compressor, target_name, chunk_size
 ):
     """Save windowed data to Zarr group (low-level function)."""
     data_array = data.astype(np.float32)
     compressors_list = [compressor] if compressor is not None else None
 
+    max_windows_per_chunk = max(
+        1, chunk_size // (data_array.shape[1] * data_array.shape[2])
+    )
+    n_windows_per_chunk = min(data_array.shape[0], max_windows_per_chunk)
+
     grp.create_array(
         "data",
         data=data_array,
-        chunks=(1, data_array.shape[1], data_array.shape[2]),
+        chunks=(
+            n_windows_per_chunk,
+            data_array.shape[1],
+            data_array.shape[2],
+        ),
         compressors=compressors_list,
     )
 
@@ -85,10 +94,13 @@ def _save_eegwindows_to_zarr(
     continuous_float = continuous_data.astype(np.float32)
     compressors_list = [compressor] if compressor is not None else None
 
+    max_samples_per_chunk = max(1, chunk_size // continuous_float.shape[0])
+    n_samples_per_chunk = min(continuous_float.shape[1], max_samples_per_chunk)
+
     grp.create_array(
         "data",
         data=continuous_float,
-        chunks=(continuous_float.shape[0], min(chunk_size, continuous_float.shape[1])),
+        chunks=(continuous_float.shape[0], n_samples_per_chunk),
         compressors=compressors_list,
     )
 
@@ -157,10 +169,13 @@ def _save_raw_to_zarr(grp, raw, description, info, target_name, compressor, chun
     continuous_float = continuous_data.astype(np.float32)
     compressors_list = [compressor] if compressor is not None else None
 
+    max_samples_per_chunk = max(1, chunk_size // continuous_float.shape[0])
+    n_samples_per_chunk = min(continuous_float.shape[1], max_samples_per_chunk)
+
     grp.create_array(
         "data",
         data=continuous_float,
-        chunks=(continuous_float.shape[0], min(chunk_size, continuous_float.shape[1])),
+        chunks=(continuous_float.shape[0], n_samples_per_chunk),
         compressors=compressors_list,
     )
 

{braindecode-1.3.2.dev169626090 → braindecode-1.3.2.dev170355810}/braindecode/datautil/hub_formats.py

@@ -36,7 +36,7 @@ def convert_to_zarr(
     output_path: Union[str, Path],
     compression: str = "blosc",
     compression_level: int = 5,
-    chunk_size: int = 10000,
+    chunk_size: int = 5_000_000,
     overwrite: bool = False,
 ) -> Path:
     """Convert BaseConcatDataset to Zarr format.
@@ -59,10 +59,13 @@ def convert_to_zarr(
         blosc uses zstd codec by default, providing best balance of speed and compression.
     compression_level : int, default=5
         Compression level (0-9). Level 5 provides optimal balance based on benchmarks.
-    chunk_size : int, default=10000
-        Number of samples per chunk in Zarr. Larger chunk size will create fewer
-        but larger files. Only used for RawDataset and EEGWindowsDataset (continuous data).
-        With WindowsDataset (pre-cut epochs), each window is saved as a separate chunk.
+    chunk_size : int, default=5_000_000
+        Target chunk size for Zarr storage.
+        For continuous data (e.g., RawDataset, EEGWindowsDataset), this is interpreted
+        as the approximate number of samples per chunk; larger values create fewer but
+        larger chunks on disk.
+        For pre-cut windowed data (WindowsDataset), this controls how many windows are
+        grouped together in each chunk (i.e., multiple windows can be stored per chunk).
     overwrite : bool, default=False
         Whether to overwrite existing directory.
 
@@ -74,8 +77,11 @@ def convert_to_zarr(
     Notes
     -----
     The chunking strategy is optimized for random access:
-    - Windowed data: Each window is a separate chunk (1, n_channels, n_times)
-    - Raw data: Chunks of (n_channels, 10000) samples
+    - Windowed data: Windows are stored in chunks of shape
+      (n_windows_in_chunk, n_channels, n_times), where n_windows_in_chunk is
+      determined by ``chunk_size``.
+    - Raw / continuous data: Time is split into chunks of approximately
+      ``chunk_size`` samples (per channel), e.g., (n_channels, n_times_per_chunk).
 
     Examples
     --------

braindecode-1.3.2.dev170355810/braindecode/version.py

@@ -0,0 +1 @@
+__version__ = "1.3.2.dev170355810"

{braindecode-1.3.2.dev169626090 → braindecode-1.3.2.dev170355810/braindecode.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: braindecode
-Version: 1.3.2.dev169626090
+Version: 1.3.2.dev170355810
 Summary: Deep learning software to decode EEG, ECG or MEG signals
 Author-email: Robin Tibor Schirrmeister <robintibor@gmail.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Alexandre Gramfort <agramfort@meta.com>
 Maintainer-email: Alexandre Gramfort <agramfort@meta.com>, Bruno Aristimunha Pinto <b.aristimunha@gmail.com>, Robin Tibor Schirrmeister <robintibor@gmail.com>

{braindecode-1.3.2.dev169626090 → braindecode-1.3.2.dev170355810}/docs/whats_new.rst

@@ -78,7 +78,7 @@ Bugs
 - Fix Zenodo citation: update to global concept DOI and add BibTeX/APA citation formats
   in ``docs/cite.rst``, ``README.rst``, ``CITATION.cff``, and ``docs/conf.py``
   (:gh:`937` by `Bruno Aristimunha`_)
-- Push large datasets to HuggingFace Hub using :func:`huggingface_hub.upload_large_folder` to avoid limitations, and allow resuming downloads (:gh:`945` by `Pierre Guetschel`_)
+- Push large datasets to HuggingFace Hub using :func:`huggingface_hub.upload_large_folder` to avoid limitations, and allow resuming downloads (:gh:`945` and :gh:`953` by `Pierre Guetschel`_)
 
 Code health
 ============

braindecode-1.3.2.dev169626090/braindecode/version.py

@@ -1 +0,0 @@
-__version__ = "1.3.2.dev169626090"