junifer 0.0.8.dev326__tar.gz → 0.0.8.dev335__tar.gz

{junifer-0.0.8.dev326/junifer.egg-info → junifer-0.0.8.dev335}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: junifer
-Version: 0.0.8.dev326
+Version: 0.0.8.dev335
 Summary: JUelich NeuroImaging FEature extractoR
 Author-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>
 Maintainer-email: Fede Raimondo <f.raimondo@fz-juelich.de>, Synchon Mandal <s.mandal@fz-juelich.de>

junifer-0.0.8.dev335/docs/changes/newsfragments/530.change

@@ -0,0 +1 @@
+Use single precision in ``antsApplyTransforms`` (``--float 1``), halving its memory and giving float32 instead of float64 data, with differences below 0.002% of the data range by `Fede Raimondo`_

junifer-0.0.8.dev335/docs/changes/newsfragments/530.enh

@@ -0,0 +1 @@
+Warp the volumes of 4D data in chunks of ``preprocessing.warping.chunksize`` volumes (default 100) in :class:`.SpaceWarper`, as the memory used by ANTs and FSL grows with the number of volumes (e.g., about 53 GB to warp 1200 volumes to native space with ANTs) by `Fede Raimondo`_

junifer-0.0.8.dev335/docs/changes/newsfragments/530.enh.1

@@ -0,0 +1 @@
+Write the intermediate images (e.g., the warped, smoothed or filtered data) uncompressed in the temporary directory of the element, as they are read right away and deleted after the element is processed, so there is no need to spend time compressing and decompressing them, and run the FSL commands with ``FSLOUTPUTTYPE=NIFTI`` to write them uncompressed by `Fede Raimondo`_

junifer-0.0.8.dev335/docs/changes/newsfragments/531.enh

@@ -0,0 +1 @@
+Raise an error if a marker changes its input data (which is shared with the other markers of a pipeline), checking the values and their contents (arrays, data frames and images) by `Fede Raimondo`_

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/docs/using/configuring.rst

@@ -63,3 +63,9 @@ Available Configurations
      - ``preprocessing.dump.granularity``
      - "full" or "final"
      - Dump all pre-processing steps or just the final pre-processed data
+   * - ``JUNIFER_PREPROCESSING_WARPING_CHUNKSIZE``
+     - ``preprocessing.warping.chunksize``
+     - int
+     - | Number of volumes of 4D data to warp at once with
+       | :class:`.SpaceWarper` (default 100). The memory used by ANTs and FSL
+       | grows with the number of volumes, so smaller chunks use less memory

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.0.8.dev326'
-__version_tuple__ = version_tuple = (0, 0, 8, 'dev326')
+__version__ = version = '0.0.8.dev335'
+__version_tuple__ = version_tuple = (0, 0, 8, 'dev335')
 
-__commit_id__ = commit_id = 'g1bcfcf0b5'
+__commit_id__ = commit_id = 'g611612030'

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/api/res/fsl/run_fsl_docker.sh

@@ -34,6 +34,10 @@ echo "Docker args: ${docker_args[*]}" >&2
 echo "Corrected args for FSL: ${corrected_args[*]}" >&2
 
 cwd=$(pwd)
+# Pass the output type, as FSL writes it instead of the type of the extension
+if [ -n "${FSLOUTPUTTYPE}" ]; then
+    docker_args+=("-e FSLOUTPUTTYPE=${FSLOUTPUTTYPE}")
+fi
 cmd="docker run --rm ${docker_args[*]} -v ${cwd}:${cwd} -w ${cwd} brainlife/fsl ${corrected_args[*]}"
 echo "Running command: ${cmd}" >&2
 ${cmd}

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/data/maps/_ants_maps_warper.py

@@ -100,16 +100,17 @@ class ANTsMapsWarper:
             logger.debug("Using ANTs for maps transformation")
 
             # Save existing maps image to a tempfile
-            prewarp_maps_path = element_tempdir / "prewarp_maps.nii.gz"
+            prewarp_maps_path = element_tempdir / "prewarp_maps.nii"
             nib.save(maps_img, prewarp_maps_path)
 
             # Create a tempfile for warped output
-            warped_maps_path = element_tempdir / "maps_warped.nii.gz"
+            warped_maps_path = element_tempdir / "maps_warped.nii"
             # Set antsApplyTransforms command
             apply_transforms_cmd = [
                 "antsApplyTransforms",
                 "-d 3",
                 "-e 3",
+                "--float 1",
                 "-n LanczosWindowedSinc",
                 f"-i {prewarp_maps_path.resolve()}",
                 # use resampled reference
@@ -133,20 +134,21 @@ class ANTsMapsWarper:
                 extra_input=None,
             )
             # Save template to a tempfile
-            template_space_img_path = element_tempdir / f"{dst}_T1w.nii.gz"
+            template_space_img_path = element_tempdir / f"{dst}_T1w.nii"
             nib.save(template_space_img, template_space_img_path)
 
             # Save existing maps image to a tempfile
-            prewarp_maps_path = element_tempdir / "prewarp_maps.nii.gz"
+            prewarp_maps_path = element_tempdir / "prewarp_maps.nii"
             nib.save(maps_img, prewarp_maps_path)
 
             # Create a tempfile for warped output
-            warped_maps_path = element_tempdir / "maps_warped.nii.gz"
+            warped_maps_path = element_tempdir / "maps_warped.nii"
             # Set antsApplyTransforms command
             apply_transforms_cmd = [
                 "antsApplyTransforms",
                 "-d 3",
                 "-e 3",
+                "--float 1",
                 "-n LanczosWindowedSinc",
                 f"-i {prewarp_maps_path.resolve()}",
                 f"-r {template_space_img_path.resolve()}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/data/maps/_fsl_maps_warper.py

@@ -66,13 +66,14 @@ class FSLMapsWarper:
         )
 
         # Save existing maps image to a tempfile
-        prewarp_maps_path = element_tempdir / "prewarp_maps.nii.gz"
+        prewarp_maps_path = element_tempdir / "prewarp_maps.nii"
         nib.save(maps_img, prewarp_maps_path)
 
         # Create a tempfile for warped output
-        warped_maps_path = element_tempdir / "maps_warped.nii.gz"
+        warped_maps_path = element_tempdir / "maps_warped.nii"
         # Set applywarp command
         applywarp_cmd = [
+            "FSLOUTPUTTYPE=NIFTI",
             "applywarp",
             "--interp=spline",
             f"-i {prewarp_maps_path.resolve()}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/data/masks/_ants_mask_warper.py

@@ -121,16 +121,17 @@ class ANTsMaskWarper:
             logger.debug("Using ANTs for mask transformation")
 
             # Save existing mask image to a tempfile
-            prewarp_mask_path = element_tempdir / "prewarp_mask.nii.gz"
+            prewarp_mask_path = element_tempdir / "prewarp_mask.nii"
             nib.save(mask_img, prewarp_mask_path)
 
             # Create a tempfile for warped output
-            warped_mask_path = element_tempdir / "mask_warped.nii.gz"
+            warped_mask_path = element_tempdir / "mask_warped.nii"
             # Set antsApplyTransforms command
             apply_transforms_cmd = [
                 "antsApplyTransforms",
                 "-d 3",
                 "-e 3",
+                "--float 1",
                 "-n 'GenericLabel[NearestNeighbor]'",
                 f"-i {prewarp_mask_path.resolve()}",
                 # use resampled reference
@@ -154,20 +155,21 @@ class ANTsMaskWarper:
                 extra_input=None,
             )
             # Save template to a tempfile
-            template_space_img_path = element_tempdir / f"{dst}_T1w.nii.gz"
+            template_space_img_path = element_tempdir / f"{dst}_T1w.nii"
             nib.save(template_space_img, template_space_img_path)
 
             # Save existing mask image to a tempfile
-            prewarp_mask_path = element_tempdir / "prewarp_mask.nii.gz"
+            prewarp_mask_path = element_tempdir / "prewarp_mask.nii"
             nib.save(mask_img, prewarp_mask_path)
 
             # Create a tempfile for warped output
-            warped_mask_path = element_tempdir / "mask_warped.nii.gz"
+            warped_mask_path = element_tempdir / "mask_warped.nii"
             # Set antsApplyTransforms command
             apply_transforms_cmd = [
                 "antsApplyTransforms",
                 "-d 3",
                 "-e 3",
+                "--float 1",
                 f"-n {_get_interpolation_method(mask_img)}",
                 f"-i {prewarp_mask_path.resolve()}",
                 f"-r {template_space_img_path.resolve()}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/data/masks/_fsl_mask_warper.py

@@ -87,13 +87,14 @@ class FSLMaskWarper:
         )
 
         # Save existing mask image to a tempfile
-        prewarp_mask_path = element_tempdir / "prewarp_mask.nii.gz"
+        prewarp_mask_path = element_tempdir / "prewarp_mask.nii"
         nib.save(mask_img, prewarp_mask_path)
 
         # Create a tempfile for warped output
-        warped_mask_path = element_tempdir / "mask_warped.nii.gz"
+        warped_mask_path = element_tempdir / "mask_warped.nii"
         # Set applywarp command
         applywarp_cmd = [
+            "FSLOUTPUTTYPE=NIFTI",
             "applywarp",
             f"--interp={_get_interpolation_method(mask_img)}",
             f"-i {prewarp_mask_path.resolve()}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/data/parcellations/_ants_parcellation_warper.py

@@ -101,19 +101,20 @@ class ANTsParcellationWarper:
 
             # Save existing parcellation image to a tempfile
             prewarp_parcellation_path = (
-                element_tempdir / "prewarp_parcellation.nii.gz"
+                element_tempdir / "prewarp_parcellation.nii"
             )
             nib.save(parcellation_img, prewarp_parcellation_path)
 
             # Create a tempfile for warped output
             warped_parcellation_path = (
-                element_tempdir / "parcellation_warped.nii.gz"
+                element_tempdir / "parcellation_warped.nii"
             )
             # Set antsApplyTransforms command
             apply_transforms_cmd = [
                 "antsApplyTransforms",
                 "-d 3",
                 "-e 3",
+                "--float 1",
                 "-n 'GenericLabel[NearestNeighbor]'",
                 f"-i {prewarp_parcellation_path.resolve()}",
                 # use resampled reference
@@ -139,24 +140,25 @@ class ANTsParcellationWarper:
                 extra_input=None,
             )
             # Save template to a tempfile
-            template_space_img_path = element_tempdir / f"{dst}_T1w.nii.gz"
+            template_space_img_path = element_tempdir / f"{dst}_T1w.nii"
             nib.save(template_space_img, template_space_img_path)
 
             # Save existing parcellation image to a tempfile
             prewarp_parcellation_path = (
-                element_tempdir / "prewarp_parcellation.nii.gz"
+                element_tempdir / "prewarp_parcellation.nii"
             )
             nib.save(parcellation_img, prewarp_parcellation_path)
 
             # Create a tempfile for warped output
             warped_parcellation_path = (
-                element_tempdir / "parcellation_warped.nii.gz"
+                element_tempdir / "parcellation_warped.nii"
             )
             # Set antsApplyTransforms command
             apply_transforms_cmd = [
                 "antsApplyTransforms",
                 "-d 3",
                 "-e 3",
+                "--float 1",
                 "-n 'GenericLabel[NearestNeighbor]'",
                 f"-i {prewarp_parcellation_path.resolve()}",
                 f"-r {template_space_img_path.resolve()}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/data/parcellations/_fsl_parcellation_warper.py

@@ -67,16 +67,15 @@ class FSLParcellationWarper:
 
         # Save existing parcellation image to a tempfile
         prewarp_parcellation_path = (
-            element_tempdir / "prewarp_parcellation.nii.gz"
+            element_tempdir / "prewarp_parcellation.nii"
         )
         nib.save(parcellation_img, prewarp_parcellation_path)
 
         # Create a tempfile for warped output
-        warped_parcellation_path = (
-            element_tempdir / "parcellation_warped.nii.gz"
-        )
+        warped_parcellation_path = element_tempdir / "parcellation_warped.nii"
         # Set applywarp command
         applywarp_cmd = [
+            "FSLOUTPUTTYPE=NIFTI",
             "applywarp",
             "--interp=nn",
             f"-i {prewarp_parcellation_path.resolve()}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/markers/falff/_junifer_falff.py

@@ -137,8 +137,8 @@ class JuniferALFF(metaclass=Singleton):
         element_tempdir = WorkDirManager().get_element_tempdir(
             prefix="junifer_lff"
         )
-        output_alff_path = element_tempdir / "output_alff.nii.gz"
-        output_falff_path = element_tempdir / "output_falff.nii.gz"
+        output_alff_path = element_tempdir / "output_alff.nii"
+        output_falff_path = element_tempdir / "output_falff.nii"
         # Save computed data to file
         nib.save(alff_data, output_alff_path)
         nib.save(falff_data, output_falff_path)

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/markers/reho/_junifer_reho.py

@@ -204,7 +204,7 @@ class JuniferReHo(metaclass=Singleton):
         element_tempdir = WorkDirManager().get_element_tempdir(
             prefix="junifer_reho"
         )
-        output_path = element_tempdir / "output.nii.gz"
+        output_path = element_tempdir / "output.nii"
         # Save computed data to file
         nib.save(output_data, output_path)
 

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/markers/tests/test_markers_base.py

@@ -347,3 +347,30 @@ def test_base_marker_input_not_changed(tmp_path: Path) -> None:
 
     out = _fit(read, tmp_path)
     assert "feat" in out["BOLD"]
+
+
+def test_base_marker_input_not_loaded(tmp_path: Path) -> None:
+    """Test checking the input does not load the data of the images.
+
+    Parameters
+    ----------
+    tmp_path : pathlib.Path
+        The path to the test directory.
+
+    """
+    input = _input(tmp_path)
+
+    class Marker(BaseMarker):
+        _MARKER_INOUT_MAPPINGS = {  # noqa: RUF012
+            DataType.BOLD: {"feat": StorageType.Vector},
+        }
+
+        def compute(self, input, extra_input=None):
+            return {"feat": {"data": np.zeros((1, 1)), "col_names": ["x"]}}
+
+    Marker(on=["BOLD"], name="marker").fit_transform(input=input)
+    # The file-backed image without its data loaded is still not loaded
+    mask = input["BOLD"]["mask"]
+    assert not mask.in_memory
+    assert getattr(mask, "_data_cache", None) is None
+    assert getattr(mask, "_fdata_cache", None) is None

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/markers/utils.py

@@ -15,7 +15,6 @@ from typing import Any
 import numpy as np
 import pandas as pd
 from nibabel.spatialimages import SpatialImage
-from nilearn.image import get_data
 from scipy.stats import zscore
 
 from ..utils import raise_error
@@ -162,10 +161,40 @@ def _checksum(array: np.ndarray) -> tuple:
     """
     crc = None
     if not array.dtype.hasobject:
+        # The transpose of an array in Fortran order (e.g., the data of a
+        # NIfTI image) is in C order, so it is not copied
+        if array.flags.f_contiguous and not array.flags.c_contiguous:
+            array = array.T
         crc = zlib.crc32(memoryview(np.ascontiguousarray(array)).cast("B"))
     return (array.shape, array.dtype.str, crc)
 
 
+def _image_data(img: SpatialImage) -> tuple:
+    """Get the data of an image in memory.
+
+    The data read from the file of an image is not loaded, as it cannot be
+    changed (unless it is cached) and it would be kept in memory.
+
+    Parameters
+    ----------
+    img : nibabel.spatialimages.SpatialImage
+        The image.
+
+    Returns
+    -------
+    tuple
+        The data of the image if it is in memory (the data of an image
+        created from an array, or the data cached by nilearn, e.g., with
+        ``nilearn.image.get_data``) and the data cached by nibabel's
+        ``get_fdata``, or None.
+
+    """
+    data = img.dataobj
+    if not isinstance(data, np.ndarray):
+        data = getattr(img, "_data_cache", None)
+    return data, getattr(img, "_fdata_cache", None)
+
+
 def _checksums(value: Any) -> tuple:
     """Get the checksums of an array, data frame, image or set.
 
@@ -179,9 +208,8 @@ def _checksums(value: Any) -> tuple:
     tuple
         The checksums, empty for other values. For data frames and series:
         the values and index, and the column labels or name. For images: the
-        affine, the data (loaded with nilearn, which caches it in the image,
-        as most markers access it this way) and the data cached by nibabel's
-        ``get_fdata`` (None if not cached). For sets: their contents.
+        affine and the data in memory (see :func:`_image_data`, None if not in
+        memory). For sets: their contents.
 
     """
     if isinstance(value, np.ndarray):
@@ -197,11 +225,12 @@ def _checksums(value: Any) -> tuple:
             labels,
         )
     if isinstance(value, SpatialImage):
-        fdata = getattr(value, "_fdata_cache", None)
         return (
             _checksum(value.affine),
-            _checksum(get_data(value)),
-            None if fdata is None else _checksum(fdata),
+            *(
+                None if data is None else _checksum(data)
+                for data in _image_data(value)
+            ),
         )
     if isinstance(value, set | frozenset):
         return (frozenset(value),)
@@ -257,21 +286,26 @@ def _is_changed(old: tuple, new: tuple) -> bool:
     if new_value is not old_value:
         return True
     if isinstance(new_value, SpatialImage):
-        old_fdata, new_fdata = old_sums[2], new_sums[2]
-        old_sums, new_sums = old_sums[:2], new_sums[:2]
-        if new_fdata is not None:
-            if old_fdata is not None and old_fdata[1] == new_fdata[1]:
-                # Same dtype: the cached data must be the same
-                if old_fdata != new_fdata:
+        # The affine must be the same
+        if old_sums[0] != new_sums[0]:
+            return True
+        for old_data, new_data, data in zip(
+            old_sums[1:], new_sums[1:], _image_data(new_value), strict=True
+        ):
+            if new_data is None:
+                # Not in memory (or no longer cached)
+                continue
+            if old_data is not None and old_data[1] == new_data[1]:
+                # Same dtype: the data in memory must be the same
+                if old_data != new_data:
                     return True
             else:
-                # The data was cached by get_fdata during the computation,
-                # possibly with another dtype: compare it with the data read
-                # as get_fdata does
-                fdata = new_value._fdata_cache
-                expected = np.asanyarray(new_value.dataobj, dtype=fdata.dtype)
-                if not np.array_equal(fdata, expected, equal_nan=True):
+                # The data was cached during the computation, possibly with
+                # another dtype: compare it with the data read from the file
+                expected = np.asanyarray(new_value.dataobj, dtype=data.dtype)
+                if not np.array_equal(data, expected, equal_nan=True):
                     return True
+        return False
     return old_sums != new_sums
 
 

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/preprocess/_temporal_filter.py

@@ -151,7 +151,7 @@ class TemporalFilter(BasePreprocessor):
                 extra_input=extra_input,
             )
             # Save generated mask for use later
-            generated_mask_img_path = element_tempdir / "generated_mask.nii.gz"
+            generated_mask_img_path = element_tempdir / "generated_mask.nii"
             nib.save(mask_img, generated_mask_img_path)
 
             # Save BOLD mask and link it to the BOLD data type dict;
@@ -193,7 +193,7 @@ class TemporalFilter(BasePreprocessor):
         # Fix t_r as nilearn messes it up
         cleaned_img.header["pixdim"][4] = t_r
         # Save filtered data
-        filtered_data_path = element_tempdir / "filtered_data.nii.gz"
+        filtered_data_path = element_tempdir / "filtered_data.nii"
         nib.save(cleaned_img, filtered_data_path)
 
         logger.debug("Updating `BOLD`")

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/preprocess/_temporal_slicer.py

@@ -151,7 +151,7 @@ class TemporalSlicer(BasePreprocessor):
         # Fix t_r as nilearn messes it up
         sliced_img.header["pixdim"][4] = t_r
         # Save sliced data
-        sliced_img_path = element_tempdir / "sliced_data.nii.gz"
+        sliced_img_path = element_tempdir / "sliced_data.nii"
         nib.save(sliced_img, sliced_img_path)
 
         logger.debug("Updating `BOLD`")

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/preprocess/confounds/fmriprep_confound_remover.py

@@ -620,7 +620,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
                 extra_input=extra_input,
             )
             # Save generated mask for use later
-            generated_mask_img_path = element_tempdir / "generated_mask.nii.gz"
+            generated_mask_img_path = element_tempdir / "generated_mask.nii"
             nib.save(mask_img, generated_mask_img_path)
 
             # Save BOLD mask and link it to the BOLD data type dict;
@@ -686,7 +686,7 @@ class fMRIPrepConfoundRemover(BasePreprocessor):
         # Fix t_r as nilearn messes it up
         cleaned_img.header["pixdim"][4] = t_r
         # Save deconfounded data
-        deconfounded_img_path = element_tempdir / "deconfounded_data.nii.gz"
+        deconfounded_img_path = element_tempdir / "deconfounded_data.nii"
         nib.save(cleaned_img, deconfounded_img_path)
 
         logger.debug("Updating `BOLD`")

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/preprocess/smoothing/_fsl_smoothing.py

@@ -79,9 +79,10 @@ class FSLSmoothing:
         element_tempdir = WorkDirManager().get_element_tempdir(
             prefix="fsl_susan"
         )
-        susan_out_path = element_tempdir / "smoothed_data.nii.gz"
+        susan_out_path = element_tempdir / "smoothed_data.nii"
         # Set susan command
         susan_cmd = [
+            "FSLOUTPUTTYPE=NIFTI",
             "susan",
             f"{input['path'].resolve()}",
             f"{brightness_threshold}",

{junifer-0.0.8.dev326 → junifer-0.0.8.dev335}/junifer/preprocess/smoothing/_nilearn_smoothing.py

@@ -74,7 +74,7 @@ class NilearnSmoothing:
         smoothed_img = nimg.smooth_img(imgs=input["data"], fwhm=fwhm)
 
         # Save smoothed output
-        smoothed_img_path = element_tempdir / "smoothed_data.nii.gz"
+        smoothed_img_path = element_tempdir / "smoothed_data.nii"
         nib.save(smoothed_img, smoothed_img_path)
 
         logger.debug("Updating smoothed data")