rapidtide 3.0.10__py3-none-any.whl → 3.1__py3-none-any.whl

rapidtide/workflows/happy.py

@@ -22,9 +22,10 @@ import sys
 import time
 import warnings
 from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import numpy as np
-from scipy.stats import pearsonr
+from numpy.typing import NDArray
 
 import rapidtide.correlate as tide_corr
 import rapidtide.filter as tide_filt
@@ -50,23 +51,104 @@ try:
 except ImportError:
     mklexists = False
 
-try:
-    import rapidtide.dlfilter as tide_dlfilt
-
-    dlfilterexists = True
-    print("dlfilter exists")
-except ImportError:
-    dlfilterexists = False
-    print("dlfilter does not exist")
 
-
-def happy_main(argparsingfunc):
+def happy_main(argparsingfunc: Any) -> None:
+    """
+    Process fMRI data to remove cardiac noise using temporal and/or spatial regression.
+
+    This function performs cardiac noise regression on fMRI data using either
+    temporal or spatial regression methods. It calculates cardiac noise signals
+    from the cardiac phase information and removes them from the fMRI data.
+
+    Parameters
+    ----------
+    args : argparse.Namespace
+        Command line arguments containing processing options
+    infodict : dict
+        Dictionary containing processing information including shared memory settings
+    fmri_data : numpy.ndarray
+        4D fMRI data array (x, y, z, time)
+    mask : numpy.ndarray
+        3D mask array indicating valid voxels
+    projmask_byslice : numpy.ndarray
+        2D mask array for each slice indicating valid projection locations
+    cardphasevals : numpy.ndarray
+        2D array of cardiac phase values for each slice and timepoint
+    rawapp_byslice : numpy.ndarray
+        3D array of raw cardiac signals for each slice and timepoint
+    outphases : numpy.ndarray
+        Array of cardiac phases
+    timepoints : int
+        Number of timepoints in the fMRI data
+    xsize : int
+        X dimension of the fMRI data
+    ysize : int
+        Y dimension of the fMRI data
+    numslices : int
+        Number of slices in the fMRI data
+    outputroot : str
+        Root name for output files
+    tide_util : module
+        Utility module for memory management and other operations
+    tide_io : module
+        I/O module for reading/writing data
+    tide_linfitfiltpass : module
+        Linear fitting and filtering module
+    tide_filt : module
+        Filtering module
+    platform : module
+        Platform information module
+    Path : class
+        Path class from pathlib module
+
+    Returns
+    -------
+    None
+        Function performs in-place processing and saves results to files
+
+    Notes
+    -----
+    This function performs multiple steps:
+    1. Calculates cardiac noise signals from cardiac phase information
+    2. Applies temporal or spatial regression to remove cardiac noise
+    3. Saves filtered data and regression coefficients
+    4. Handles shared memory cleanup when needed
+
+    Examples
+    --------
+    >>> process_fmri_with_cardiac_regression(
+    ...     args, infodict, fmri_data, mask, projmask_byslice,
+    ...     cardphasevals, rawapp_byslice, outphases,
+    ...     timepoints, xsize, ysize, numslices,
+    ...     outputroot, tide_util, tide_io, tide_linfitfiltpass,
+    ...     tide_filt, platform, Path
+    ... )
+    """
     timings = [["Start", time.time(), None, None]]
 
     # get the command line parameters
     args = argparsingfunc
     infodict = vars(args)
 
+    if args.usepytorch:
+        try:
+            import rapidtide.dlfiltertorch as tide_dlfilt
+
+            dlfilterexists = True
+            print("pytorch dlfilter initialized")
+        except ImportError:
+            dlfilterexists = False
+            print("pytorch dlfilter could not be initialized")
+    else:
+        try:
+            import rapidtide.dlfilter as tide_dlfilt
+
+            dlfilterexists = True
+            print("tensorflow dlfilter initialized")
+        except ImportError:
+            dlfilterexists = False
+            print("tensorflow dlfilter could not be initialized")
+
     fmrifilename = args.fmrifilename
     slicetimename = args.slicetimename
     outputroot = args.outputroot
@@ -93,7 +175,7 @@ def happy_main(argparsingfunc):
         logger_filename=f"{outputroot}_log.txt",
         timing_filename=f"{outputroot}_runtimings.tsv",
         memory_filename=f"{outputroot}_memusage.tsv",
-        verbose=args.verbose,
+        isverbose=args.verbose,
         debug=args.debug,
     )
 
@@ -114,6 +196,8 @@ def happy_main(argparsingfunc):
         "***********************************************************************************************************************************")
     print("")"""
 
+    infodict["pid"] = os.getpid()
+
     infodict["fmrifilename"] = fmrifilename
     infodict["slicetimename"] = slicetimename
     infodict["outputroot"] = outputroot
@@ -140,6 +224,12 @@ def happy_main(argparsingfunc):
         "CommandLineArgs": args.commandline,
     }
 
+    # decide if we need to use shared memory
+    if args.nprocs > 1:
+        infodict["sharedmem"] = True
+    else:
+        infodict["sharedmem"] = False
+
     # save the information file
     if args.saveinfoasjson:
         tide_io.writedicttojson(infodict, outputroot + "_info.json")
@@ -195,14 +285,41 @@ def happy_main(argparsingfunc):
 
     # make and save a mask of the voxels to process based on image intensity
     tide_util.logmem("before mask creation")
-    mask = np.uint16(tide_mask.makeepimask(input_data.nim).dataobj.reshape(numspatiallocs))
+    if args.processmask is not None:
+        mask = tide_mask.readamask(
+            args.processmask,
+            input_data.copyheader(numtimepoints=1),
+            xsize,
+            thresh=0.001,
+            maskname="process",
+            debug=args.debug,
+        )
+        mask = np.uint16(np.where(mask > 0, 1, 0).reshape(numspatiallocs))
+    else:
+        mask = np.uint16(tide_mask.makeepimask(input_data.nim).dataobj.reshape(numspatiallocs))
+
     theheader = input_data.copyheader(numtimepoints=1)
     timings.append(["Mask created", time.time(), None, None])
     if args.outputlevel > 0:
-        maskfilename = outputroot + "_desc-processvoxels_mask"
         bidsdict = bidsbasedict.copy()
-        tide_io.writedicttojson(bidsdict, maskfilename + ".json")
-        tide_io.savetonifti(mask.reshape((xsize, ysize, numslices)), theheader, maskfilename)
+        maplist = [
+            (
+                mask.reshape((xsize, ysize, numslices)),
+                "processvoxels",
+                "mask",
+                None,
+                "fMRI voxels processed by happy",
+            ),
+        ]
+        tide_io.savemaplist(
+            outputroot,
+            maplist,
+            None,
+            (xsize, ysize, numslices),
+            theheader,
+            bidsdict,
+            debug=args.debug,
+        )
     timings.append(["Mask saved", time.time(), None, None])
     mask_byslice = mask.reshape((xsize * ysize, numslices))
 
@@ -267,25 +384,49 @@ def happy_main(argparsingfunc):
         )
         # save motionr2 map
         theheader = input_data.copyheader(numtimepoints=1)
-        motionr2filename = outputroot + "_desc-motionr2_map"
         bidsdict = bidsbasedict.copy()
-        tide_io.writedicttojson(bidsdict, motionr2filename + ".json")
         outarray = np.zeros((xsize, ysize, numslices), dtype=float)
         outarray.reshape(numspatiallocs)[validprojvoxels] = confoundr2
-        tide_io.savetonifti(
-            outarray.reshape((xsize, ysize, numslices)), theheader, motionr2filename
+        maplist = [
+            (
+                outarray.reshape((xsize, ysize, numslices)),
+                "motionr2",
+                "map",
+                None,
+                "R2 of motion regression",
+            ),
+        ]
+        tide_io.savemaplist(
+            outputroot,
+            maplist,
+            None,
+            (xsize, ysize, numslices),
+            theheader,
+            bidsdict,
+            debug=args.debug,
         )
         if args.savemotionglmfilt:
             print("prior to save - fmri_data has shape", fmri_data.shape)
-            motionfilteredfilename = outputroot + "_desc-motionfiltered_bold"
             bidsdict = bidsbasedict.copy()
-            bidsdict["Units"] = "second"
-            tide_io.writedicttojson(bidsdict, motionfilteredfilename + ".json")
-            tide_io.savetonifti(
-                fmri_data.reshape((xsize, ysize, numslices, timepoints)),
+            maplist = [
+                (
+                    fmri_data.reshape((xsize, ysize, numslices, timepoints)),
+                    "motionfiltered",
+                    "bold",
+                    None,
+                    "fMRI data after motion regression",
+                ),
+            ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices, timepoints),
                 theheader,
-                motionfilteredfilename,
+                bidsdict,
+                debug=args.debug,
             )
+
             timings.append(["Motion filtered data saved", time.time(), numspatiallocs, "voxels"])
 
     # get slice times
@@ -324,16 +465,39 @@ def happy_main(argparsingfunc):
 
     # save means, medians, and mads
     theheader = input_data.copyheader(numtimepoints=1)
-    meansfilename = outputroot + "_desc-means_map"
-    mediansfilename = outputroot + "_desc-medians_map"
-    madsfilename = outputroot + "_desc-mads_map"
     bidsdict = bidsbasedict.copy()
-    tide_io.writedicttojson(bidsdict, meansfilename + ".json")
-    tide_io.writedicttojson(bidsdict, mediansfilename + ".json")
-    tide_io.writedicttojson(bidsdict, madsfilename + ".json")
-    tide_io.savetonifti(means.reshape((xsize, ysize, numslices)), theheader, meansfilename)
-    tide_io.savetonifti(medians.reshape((xsize, ysize, numslices)), theheader, mediansfilename)
-    tide_io.savetonifti(mads.reshape((xsize, ysize, numslices)), theheader, madsfilename)
+    maplist = [
+        (
+            means.reshape((xsize, ysize, numslices)),
+            "means",
+            "map",
+            None,
+            "fMRI timecourse mean over time",
+        ),
+        (
+            medians.reshape((xsize, ysize, numslices)),
+            "medians",
+            "map",
+            None,
+            "fMRI timecourse median over time",
+        ),
+        (
+            mads.reshape((xsize, ysize, numslices)),
+            "mads",
+            "map",
+            None,
+            "fMRI timecourse MAD over time",
+        ),
+    ]
+    tide_io.savemaplist(
+        outputroot,
+        maplist,
+        None,
+        (xsize, ysize, numslices),
+        theheader,
+        bidsdict,
+        debug=args.debug,
+    )
 
     # read in estimation mask if present. Otherwise, otherwise use intensity mask.
     infodict["estweightsname"] = args.estweightsname
@@ -371,10 +535,12 @@ def happy_main(argparsingfunc):
         if numpasses > 1:
             print()
             print()
-            print("starting pass", thispass + 1, "of", numpasses)
-            passstring = " - pass " + str(thispass + 1)
+            print(f"starting pass {thispass + 1} of {numpasses}")
+            passstring = f" - pass {thispass + 1}"
+            passnamefrag = f"pass{thispass + 1}"
         else:
             passstring = ""
+            passnamefrag = ""
         # now get an estimate of the cardiac signal
         print("estimating cardiac signal from fmri data")
         tide_util.logmem("before cardiacfromimage")
@@ -402,6 +568,7 @@ def happy_main(argparsingfunc):
             madnorm=args.domadnorm,
             nprocs=args.nprocs,
             notchpct=args.notchpct,
+            notchrolloff=args.notchrolloff,
             fliparteries=args.fliparteries,
             arteriesonly=args.arteriesonly,
             usemask=args.usemaskcardfromfmri,
@@ -1405,9 +1572,24 @@ def happy_main(argparsingfunc):
             )
 
             theheader = input_data.copyheader(numtimepoints=1)
-            wrightfilename = f"{outputroot}_desc-wrightcorrspass{thispass + 1}_map"
-            tide_io.writedicttojson(bidsbasedict, wrightfilename + ".json")
-            tide_io.savetonifti(wrightcorrs, theheader, wrightfilename)
+            maplist = [
+                (
+                    wrightcorrs,
+                    f"wrightcorrspass{passnamefrag}",
+                    "map",
+                    None,
+                    "fMRI timecourse MAD over time",
+                ),
+            ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices),
+                theheader,
+                bidsdict,
+                debug=args.debug,
+            )
             timings.append(
                 [
                     "Wright mask generation completed" + passstring,
@@ -1535,23 +1717,49 @@ def happy_main(argparsingfunc):
             numtimepoints=args.destpoints, tr=-np.pi, toffset=2.0 * np.pi / args.destpoints
         )
         if thispass == numpasses - 1:
-            appfilename = outputroot + "_desc-app_info"
-            normappfilename = outputroot + "_desc-normapp_info"
-            cinefilename = outputroot + "_desc-cine_info"
-            rawappfilename = outputroot + "_desc-rawapp_info"
-            bidsdict = bidsbasedict.copy()
-            bidsdict["Units"] = "second"
-            tide_io.writedicttojson(bidsdict, appfilename + ".json")
-            tide_io.writedicttojson(bidsdict, normappfilename + ".json")
-            tide_io.writedicttojson(bidsdict, cinefilename + ".json")
-            if args.outputlevel > 0:
-                tide_io.writedicttojson(bidsdict, rawappfilename + ".json")
-            tide_io.savetonifti(app, theheader, appfilename)
-            tide_io.savetonifti(normapp, theheader, normappfilename)
-            tide_io.savetonifti(cine, theheader, cinefilename)
+            maplist = [
+                (
+                    app,
+                    "app",
+                    "info",
+                    None,
+                    "Cardiac pulsatility waveform with minimum over time removed",
+                ),
+                (
+                    normapp,
+                    "normapp",
+                    "info",
+                    "percent",
+                    "Cardiac pulsatility waveform in percentage change relative to mean",
+                ),
+                (
+                    cine,
+                    "cine",
+                    "info",
+                    None,
+                    "fMRI signal averaged over a single cardiac cycle",
+                ),
+            ]
             if args.outputlevel > 0:
-                tide_io.savetonifti(rawapp, theheader, rawappfilename)
-
+                maplist += [
+                    (
+                        rawapp,
+                        "rawapp",
+                        "info",
+                        None,
+                        "Cardiac pulsatility waveform",
+                    ),
+                ]
+            # write the 4D phase projection maps
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices, args.destpoints),
+                theheader,
+                bidsdict,
+                debug=args.debug,
+            )
         timings.append(["Phase projected data saved" + passstring, time.time(), None, None])
 
         if args.doaliasedcorrelation and thispass == numpasses - 1:
@@ -1560,21 +1768,57 @@ def happy_main(argparsingfunc):
                 tr=(thealiasedcorrx[1] - thealiasedcorrx[0]),
                 toffset=0.0,
             )
-            corrfuncfilename = outputroot + "_desc-corrfunc_info"
-            wavedelayfilename = outputroot + "_desc-wavedelay_map"
-            wavedelayCOMfilename = outputroot + "_desc-wavedelayCOM_map"
-            waveampfilename = outputroot + "_desc-waveamp_map"
-            bidsdict = bidsbasedict.copy()
-            tide_io.writedicttojson(bidsdict, waveampfilename + ".json")
-            bidsdict["Units"] = "second"
-            tide_io.writedicttojson(bidsdict, corrfuncfilename + ".json")
-            tide_io.writedicttojson(bidsdict, wavedelayfilename + ".json")
-            tide_io.writedicttojson(bidsdict, wavedelayCOMfilename + ".json")
-            tide_io.savetonifti(thecorrfunc, theheader, corrfuncfilename)
+            maplist = [
+                (
+                    thecorrfunc,
+                    "corrfunc",
+                    "info",
+                    None,
+                    "Aliased correlation function",
+                ),
+            ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices, aliasedcorrelationpts),
+                theheader,
+                bidsdict,
+                debug=args.debug,
+            )
             theheader["dim"][4] = 1
-            tide_io.savetonifti(wavedelay, theheader, wavedelayfilename)
-            tide_io.savetonifti(wavedelayCOM, theheader, wavedelayCOMfilename)
-            tide_io.savetonifti(waveamp, theheader, waveampfilename)
+            maplist = [
+                (
+                    wavedelay,
+                    "wavedelay",
+                    "map",
+                    "seconds",
+                    "Peak delay of aliased correlation function",
+                ),
+                (
+                    wavedelayCOM,
+                    "wavedelayCOM",
+                    "map",
+                    "seconds",
+                    "Center of mass of aliased correlation function",
+                ),
+                (
+                    waveamp,
+                    "waveamp",
+                    "map",
+                    None,
+                    "Peak amplitude of aliased correlation function",
+                ),
+            ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices),
+                theheader,
+                bidsdict,
+                debug=args.debug,
+            )
 
         # make and save a voxel intensity histogram
         if args.unnormvesselmap:
@@ -1612,24 +1856,34 @@ def happy_main(argparsingfunc):
         maskedapp2d[np.where(vesselmask.reshape(numspatiallocs) == 0)[0], :] = 0.0
         if args.outputlevel > 1:
             if thispass == numpasses - 1:
-                maskedappfilename = outputroot + "_desc-maskedapp_info"
                 bidsdict = bidsbasedict.copy()
-                bidsdict["Units"] = "second"
-                tide_io.writedicttojson(bidsdict, maskedappfilename + ".json")
-                tide_io.savetonifti(
-                    maskedapp2d.reshape((xsize, ysize, numslices, args.destpoints)),
+                maplist = [
+                    (
+                        maskedapp2d.reshape((xsize, ysize, numslices, args.destpoints)),
+                        "maskedapp",
+                        "info",
+                        None,
+                        "Masked analytic phase projection",
+                    ),
+                ]
+                tide_io.savemaplist(
+                    outputroot,
+                    maplist,
+                    None,
+                    (xsize, ysize, numslices, args.destpoints),
                     theheader,
-                    maskedappfilename,
+                    bidsdict,
+                    debug=args.debug,
+                )
+                timings.append(
+                    [
+                        "Vessel masked phase projected data saved" + passstring,
+                        time.time(),
+                        None,
+                        None,
+                    ]
                 )
         del maskedapp2d
-        timings.append(
-            [
-                "Vessel masked phase projected data saved" + passstring,
-                time.time(),
-                None,
-                None,
-            ]
-        )
 
         # save multiple versions of the hard vessel mask
         if args.unnormvesselmap:
@@ -1645,57 +1899,316 @@ def happy_main(argparsingfunc):
         veins = np.where(appflips_byslice.reshape((xsize, ysize, numslices)) > 0, vesselmask, 0)
         theheader = input_data.copyheader(numtimepoints=1)
         if thispass == numpasses - 1:
-            vesselmaskfilename = outputroot + "_desc-vessels_mask"
-            minphasefilename = outputroot + "_desc-minphase_map"
-            maxphasefilename = outputroot + "_desc-maxphase_map"
-            arterymapfilename = outputroot + "_desc-arteries_map"
-            veinmapfilename = outputroot + "_desc-veins_map"
-            bidsdict = bidsbasedict.copy()
-            tide_io.writedicttojson(bidsdict, vesselmaskfilename + ".json")
-            tide_io.savetonifti(vesselmask, theheader, vesselmaskfilename)
+            maplist = [
+                (
+                    vesselmask,
+                    "vessels",
+                    "mask",
+                    None,
+                    "Vessel mask",
+                ),
+            ]
             if args.outputlevel > 0:
-                tide_io.writedicttojson(bidsdict, arterymapfilename + ".json")
-                tide_io.writedicttojson(bidsdict, veinmapfilename + ".json")
-                bidsdict["Units"] = "radians"
-                tide_io.writedicttojson(bidsdict, minphasefilename + ".json")
-                tide_io.writedicttojson(bidsdict, maxphasefilename + ".json")
-                tide_io.savetonifti(minphase, theheader, minphasefilename)
-                tide_io.savetonifti(maxphase, theheader, maxphasefilename)
-                tide_io.savetonifti(arteries, theheader, arterymapfilename)
-                tide_io.savetonifti(veins, theheader, veinmapfilename)
+                maplist += [
+                    (
+                        minphase,
+                        "minphase",
+                        "map",
+                        "radians",
+                        "Cardiac phase where minimum pulsatility signal occurs",
+                    ),
+                    (
+                        maxphase,
+                        "maxphase",
+                        "map",
+                        "radians",
+                        "Cardiac phase where maximum pulsatility signal occurs",
+                    ),
+                    (
+                        arteries,
+                        "arteries",
+                        "map",
+                        None,
+                        "Arterial voxels (maybe)",
+                    ),
+                    (
+                        veins,
+                        "veins",
+                        "map",
+                        None,
+                        "Venous voxels (maybe)",
+                    ),
+                ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices),
+                theheader,
+                bidsdict,
+                debug=args.debug,
+            )
         timings.append(["Masks saved" + passstring, time.time(), None, None])
 
         # save the mask we used for this pass
-        tide_io.savetonifti(
-            estweights_byslice.reshape((xsize, ysize, numslices)),
+        tide_io.savemaplist(
+            outputroot,
+            [
+                (
+                    estweights_byslice.reshape((xsize, ysize, numslices)),
+                    f"estweights{passnamefrag}",
+                    "map",
+                    None,
+                    f"Estweights{passstring}",
+                ),
+            ],
+            None,
+            (xsize, ysize, numslices),
             theheader,
-            f"{outputroot}_desc-estweightspass{thispass}_map",
+            bidsdict,
+            debug=args.debug,
         )
 
+        # estimate pulsatility
+        (
+            card_min,
+            card_max,
+            card_mean,
+            card_std,
+            card_median,
+            card_mad,
+            card_skew,
+            card_kurtosis,
+        ) = tide_stats.fmristats(normapp.reshape(numspatiallocs, -1))
+        maplist = [
+            (
+                card_min,
+                "appmin",
+                "map",
+                None,
+                "Minimum value of analytic phase projection across all phases",
+            ),
+            (
+                card_max,
+                "appmax",
+                "map",
+                None,
+                "Maximum value of analytic phase projection across all phases",
+            ),
+            (
+                card_mean,
+                "appmean",
+                "map",
+                None,
+                "Mean value of analytic phase projection across all phases",
+            ),
+            (
+                card_std,
+                "appstd",
+                "map",
+                None,
+                "Standard deviation of analytic phase projection across all phases",
+            ),
+            (
+                card_median,
+                "appmedian",
+                "map",
+                None,
+                "Median of analytic phase projection across all phases",
+            ),
+            (
+                card_mad,
+                "appMAD",
+                "map",
+                None,
+                "Median average deviate of analytic phase projection across all phases",
+            ),
+            (
+                card_skew,
+                "appskew",
+                "map",
+                None,
+                "Skewness of analytic phase projection across all phases",
+            ),
+            (
+                card_kurtosis,
+                "appkurtosis",
+                "map",
+                None,
+                "Kurtosis of analytic phase projection across all phases",
+            ),
+        ]
+        # write the 3D maps
+        tide_io.savemaplist(
+            outputroot,
+            maplist,
+            None,
+            (xsize, ysize, numslices),
+            theheader,
+            bidsdict,
+            debug=args.debug,
+        )
+
+        pulsatilitymap = 100.0 * (np.max(normapp, axis=3) - np.min(normapp, axis=3))
+        rawrobustmax = tide_stats.getfracval(pulsatilitymap, 0.98, nozero=True)
+        rawmedian = tide_stats.getfracval(pulsatilitymap, 0.50, nozero=True)
+        infodict["pulsatilitythresh"] = rawmedian * 3.0
+        pulsatilitymap = np.where(pulsatilitymap < rawrobustmax, pulsatilitymap, rawrobustmax)
+        pulsatilitymask = np.where(pulsatilitymap > infodict["pulsatilitythresh"], 1.0, 0.0)
+
         # now get ready to start again with a new mask
         if args.doaliasedcorrelation and thispass > 0:
             estweights_byslice = waveamp_byslice * vesselmask.reshape((xsize * ysize, numslices))
         else:
-            estweights_byslice = vesselmask.reshape((xsize * ysize, numslices)) + 0
-
-    # save a vessel image
-    if args.unnormvesselmap:
-        vesselmap = np.max(app, axis=3)
+            if args.useoriginalvesselmethod:
+                estweights_byslice = vesselmask.reshape((xsize * ysize, numslices)) + 0
+            else:
+                estweights_byslice = pulsatilitymask.reshape((xsize * ysize, numslices)) + 0
+
+    # calculate the lf and hf pulsatility maps
+    lfnormapp = normapp * 0.0
+    hfnormapp = normapp * 0.0
+    for thephase in range(args.destpoints):
+        lfnormapp[:, :, :, thephase] = (
+            tide_filt.ssmooth(
+                xdim, ydim, slicethickness, args.pulsatilitysigma, normapp[:, :, :, thephase]
+            )
+            * pulsatilitymask
+        )
+        hfnormapp[:, :, :, thephase] = normapp[:, :, :, thephase] - lfnormapp[:, :, :, thephase]
+    lfnormapp -= np.min(lfnormapp, axis=3)[:, :, :, None]
+    hfnormapp -= np.min(hfnormapp, axis=3)[:, :, :, None]
+    lfpulsatilitymap = (
+        tide_filt.ssmooth(xdim, ydim, slicethickness, args.pulsatilitysigma, pulsatilitymap)
+        * pulsatilitymask
+    )
+    hfpulsatilitymap = pulsatilitymap - lfpulsatilitymap
+    lfpulsatilitymap2 = 100.0 * np.max(lfnormapp, axis=3)
+    lfrobustmax = tide_stats.getfracval(lfpulsatilitymap2, 0.98, nozero=True)
+    lfpulsatilitymap2 = np.where(lfpulsatilitymap2 < lfrobustmax, lfpulsatilitymap2, lfrobustmax)
+    hfpulsatilitymap2 = 100.0 * np.max(hfnormapp, axis=3)
+    hfrobustmax = tide_stats.getfracval(hfpulsatilitymap2, 0.98, nozero=True)
+    hfpulsatilitymap2 = np.where(hfpulsatilitymap2 < hfrobustmax, hfpulsatilitymap2, hfrobustmax)
+
+    # make a vessel map
+    if args.useoriginalvesselmethod:
+        if args.unnormvesselmap:
+            vesselmap = np.max(app, axis=3)
+        else:
+            # vesselmap = np.max(normapp, axis=3)
+            vesselmap = np.where(hfpulsatilitymap > args.pulsatilitythreshold, 1.0, 0.0)
     else:
-        vesselmap = np.max(normapp, axis=3)
-    vesselmapfilename = outputroot + "_desc-vessels_map"
-    arterymapfilename = outputroot + "_desc-arteries_map"
-    veinmapfilename = outputroot + "_desc-veins_map"
-    tide_io.savetonifti(vesselmap, theheader, vesselmapfilename)
-    tide_io.savetonifti(
-        np.where(appflips_byslice.reshape((xsize, ysize, numslices)) < 0, vesselmap, 0.0),
+        vesselmap = pulsatilitymask
+    veinmap = np.where(appflips_byslice.reshape((xsize, ysize, numslices)) > 0, vesselmap, 0.0)
+    arterymap = np.where(appflips_byslice.reshape((xsize, ysize, numslices)) < 0, vesselmap, 0.0)
+
+    # specify the 3D maps
+    maplist = [
+        (
+            pulsatilitymap,
+            "pulsatility",
+            "map",
+            "percent",
+            "Cardiac pulsatility in percentage change relative to mean",
+        ),
+        (
+            pulsatilitymask,
+            "pulsatility",
+            "mask",
+            None,
+            "Valid cardiac pulsatility voxels",
+        ),
+        (
+            lfpulsatilitymap,
+            "lfpulsatility",
+            "map",
+            "percent",
+            "Low spatial frequency cardiac pulsatility in percentage change relative to mean",
+        ),
+        (
+            hfpulsatilitymap,
+            "hfpulsatility",
+            "map",
+            "percent",
+            "High spatial frequency cardiac pulsatility in percentage change relative to mean",
+        ),
+        (
+            lfpulsatilitymap2,
+            "lfpulsatility2",
+            "map",
+            "percent",
+            "Low spatial frequency cardiac pulsatility in percentage change relative to mean",
+        ),
+        (
+            hfpulsatilitymap2,
+            "hfpulsatility2",
+            "map",
+            "percent",
+            "High spatial frequency cardiac pulsatility in percentage change relative to mean",
+        ),
+        (
+            vesselmap,
+            "vessels",
+            "map",
+            None,
+            "Vessel voxels",
+        ),
+        (
+            arterymap,
+            "arteries",
+            "map",
+            None,
+            "Arterial voxels (maybe)",
+        ),
+        (
+            veinmap,
+            "veins",
+            "map",
+            None,
+            "Venous voxels (maybe)",
+        ),
+    ]
+    # write the 3D maps
+    tide_io.savemaplist(
+        outputroot,
+        maplist,
+        None,
+        (xsize, ysize, numslices),
         theheader,
-        arterymapfilename,
+        bidsdict,
+        debug=args.debug,
+    )
+
+    # specify the 4D maps
+    theheader = input_data.copyheader(
+        numtimepoints=args.destpoints, tr=-np.pi, toffset=2.0 * np.pi / args.destpoints
     )
-    tide_io.savetonifti(
-        np.where(appflips_byslice.reshape((xsize, ysize, numslices)) > 0, vesselmap, 0.0),
+    maplist = [
+        (
+            lfnormapp,
+            "lfnormapp",
+            "info",
+            "percent",
+            "Low spatial frequency cardiac pulsatility waveform in percentage change relative to mean",
+        ),
+        (
+            hfnormapp,
+            "hfnormapp",
+            "info",
+            "percent",
+            "High spatial frequency cardiac pulsatility waveform in percentage change relative to mean",
+        ),
+    ]
+
+    # write the 4D maps
+    tide_io.savemaplist(
+        outputroot,
+        maplist,
+        None,
+        (xsize, ysize, numslices, args.destpoints),
         theheader,
-        veinmapfilename,
+        bidsdict,
+        debug=args.debug,
     )
 
     # now generate aliased cardiac signals and regress them out of the data
@@ -1721,34 +2234,85 @@ def happy_main(argparsingfunc):
         theheader = input_data.copyheader()
         timings.append(["Cardiac signal generated", time.time(), None, None])
         if args.savecardiacnoise:
-            cardiacnoisefilename = outputroot + "_desc-cardiacnoise_info"
-            phaseindexfilename = outputroot + "_desc-phaseindices_info"
-            tide_io.savetonifti(
-                cardiacnoise.reshape((xsize, ysize, numslices, timepoints)),
-                theheader,
-                cardiacnoisefilename,
-            )
-            tide_io.savetonifti(
-                phaseindices.reshape((xsize, ysize, numslices, timepoints)),
+            maplist = [
+                (
+                    cardiacnoise.reshape((xsize, ysize, numslices, timepoints)),
+                    "cardiacnoise",
+                    "info",
+                    None,
+                    "Calculated cardiac noise EVs",
+                ),
+                (
+                    phaseindices.reshape((xsize, ysize, numslices, timepoints)),
+                    "phaseindices",
+                    "info",
+                    None,
+                    "Phase indices",
+                ),
+            ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices, timepoints),
                 theheader,
-                phaseindexfilename,
+                bidsdict,
+                debug=args.debug,
             )
             timings.append(["Cardiac signal saved", time.time(), None, None])
 
         # now remove them
         tide_util.logmem("before cardiac removal")
         print("Removing cardiac signal with GLM")
-        filtereddata = 0.0 * fmri_data
         validlocs = np.where(mask > 0)[0]
         numvalidspatiallocs = len(validlocs)
+        if args.focaldebug:
+            print(f"{numvalidspatiallocs=}, {fmri_data.shape=}")
+        filtereddata, filtereddata_shm = tide_util.allocarray(
+            fmri_data.shape,
+            "float64",
+            shared=infodict["sharedmem"],
+            name=f"filtereddata_{infodict['pid']}",
+        )
+        datatoremove, datatoremove_shm = tide_util.allocarray(
+            fmri_data.shape,
+            "float64",
+            shared=infodict["sharedmem"],
+            name=f"datatoremove_{infodict['pid']}",
+        )
         threshval = 0.0
         if args.dospatialregression:
-            meanvals = np.zeros(timepoints, dtype=np.float64)
-            rvals = np.zeros(timepoints, dtype=np.float64)
-            r2vals = np.zeros(timepoints, dtype=np.float64)
-            fitcoffs = np.zeros(timepoints, dtype=np.float64)
-            fitNorm = np.zeros(timepoints, dtype=np.float64)
-            datatoremove = 0.0 * fmri_data
+            regressiontype = "spatial"
+            meanvals, meanvals_shm = tide_util.allocarray(
+                (timepoints),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"meanvals_{infodict['pid']}",
+            )
+            rvals, rvals_shm = tide_util.allocarray(
+                (timepoints),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"rvals_{infodict['pid']}",
+            )
+            r2vals, r2vals_shm = tide_util.allocarray(
+                (timepoints),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"r2vals_{infodict['pid']}",
+            )
+            fitcoffs, fitcoffs_shm = tide_util.allocarray(
+                (timepoints),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"fitcoffs_{infodict['pid']}",
+            )
+            fitNorm, fitNorm_shm = tide_util.allocarray(
+                (timepoints),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"fitNorm_{infodict['pid']}",
+            )
             print("Running spatial regression on", timepoints, "timepoints")
             tide_util.disablemkl(args.nprocs)
             tide_linfitfiltpass.linfitfiltpass(
@@ -1768,8 +2332,10 @@ def happy_main(argparsingfunc):
                 nprocs=args.nprocs,
             )
             tide_util.enablemkl(args.mklthreads)
-            print(datatoremove.shape, cardiacnoise.shape, fitcoffs.shape)
-            # datatoremove[validlocs, :] = np.multiply(cardiacnoise[validlocs, :], fitcoffs[:, None])
+            datatoremove[validlocs, :] = np.multiply(cardiacnoise[validlocs, :], fitcoffs[None, :])
+            print(f"{datatoremove.shape=}, {np.min(datatoremove)=}, {np.max(datatoremove)=}")
+            print(f"{cardiacnoise.shape=}, {np.min(cardiacnoise)=}, {np.max(cardiacnoise)=}")
+            print(f"{fitcoffs.shape=}, {np.min(fitcoffs)=}, {np.max(fitcoffs)=}")
             filtereddata = fmri_data - datatoremove
             timings.append(
                 [
@@ -1779,38 +2345,39 @@ def happy_main(argparsingfunc):
                     "timepoints",
                 ]
             )
-            tide_io.writevec(fitcoffs, outputroot + "_fitcoff.txt")
-            tide_io.writevec(meanvals, outputroot + "_fitmean.txt")
-            tide_io.writevec(rvals, outputroot + "_fitR.txt")
-            theheader = input_data.copyheader()
-            cardfiltresultfilename = outputroot + "_desc-cardfiltResult_bold"
-            cardfiltremovedfilename = outputroot + "_desc-cardfiltRemoved_bold"
-            tide_io.savetonifti(
-                filtereddata.reshape((xsize, ysize, numslices, timepoints)),
-                theheader,
-                cardfiltresultfilename,
+
+        if args.dotemporalregression:
+            regressiontype = "temporal"
+            meanvals, meanvals_shm = tide_util.allocarray(
+                (numspatiallocs),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"meanvals_{infodict['pid']}",
             )
-            tide_io.savetonifti(
-                datatoremove.reshape((xsize, ysize, numslices, timepoints)),
-                theheader,
-                cardfiltremovedfilename,
+            rvals, rvals_shm = tide_util.allocarray(
+                (numspatiallocs),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"rvals_{infodict['pid']}",
             )
-            timings.append(
-                [
-                    "Cardiac signal spatial regression files written",
-                    time.time(),
-                    None,
-                    None,
-                ]
+            r2vals, r2vals_shm = tide_util.allocarray(
+                (numspatiallocs),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"r2vals_{infodict['pid']}",
+            )
+            fitcoffs, fitcoffs_shm = tide_util.allocarray(
+                (numspatiallocs),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"fitcoffs_{infodict['pid']}",
+            )
+            fitNorm, fitNorm_shm = tide_util.allocarray(
+                (numspatiallocs),
+                "float64",
+                shared=infodict["sharedmem"],
+                name=f"fitNorm_{infodict['pid']}",
             )
-
-        if args.dotemporalregression:
-            meanvals = np.zeros(numspatiallocs, dtype=np.float64)
-            rvals = np.zeros(numspatiallocs, dtype=np.float64)
-            r2vals = np.zeros(numspatiallocs, dtype=np.float64)
-            fitcoffs = np.zeros(numspatiallocs, dtype=np.float64)
-            fitNorm = np.zeros(numspatiallocs, dtype=np.float64)
-            datatoremove = 0.0 * fmri_data
             print("Running temporal regression on", numvalidspatiallocs, "voxels")
             tide_util.disablemkl(args.nprocs)
             tide_linfitfiltpass.linfitfiltpass(
@@ -1827,11 +2394,15 @@ def happy_main(argparsingfunc):
                 filtereddata[validlocs, :],
                 procbyvoxel=True,
                 nprocs=args.nprocs,
+                debug=args.focaldebug,
             )
             tide_util.enablemkl(args.mklthreads)
             datatoremove[validlocs, :] = np.multiply(
                 cardiacnoise[validlocs, :], fitcoffs[validlocs, None]
             )
+            print(f"{datatoremove.shape=}, {np.min(datatoremove)=}, {np.max(datatoremove)=}")
+            print(f"{cardiacnoise.shape=}, {np.min(cardiacnoise)=}, {np.max(cardiacnoise)=}")
+            print(f"{fitcoffs.shape=}, {np.min(fitcoffs)=}, {np.max(fitcoffs)=}")
             filtereddata[validlocs, :] = fmri_data[validlocs, :] - datatoremove[validlocs, :]
             timings.append(
                 [
@@ -1842,44 +2413,85 @@ def happy_main(argparsingfunc):
                 ]
             )
             theheader = input_data.copyheader(numtimepoints=1)
-            cardfiltcoeffsfilename = outputroot + "_desc-cardfiltCoeffs_map"
-            cardfiltmeanfilename = outputroot + "_desc-cardfiltMean_map"
-            cardfiltRfilename = outputroot + "_desc-cardfiltR_map"
-            tide_io.savetonifti(
-                fitcoffs.reshape((xsize, ysize, numslices)),
-                theheader,
-                cardfiltcoeffsfilename,
-            )
-            tide_io.savetonifti(
-                meanvals.reshape((xsize, ysize, numslices)),
+            maplist = [
+                (
+                    fitcoffs.reshape((xsize, ysize, numslices)),
+                    "cardfiltCoeffs",
+                    "map",
+                    None,
+                    "Coefficients for temporal cardiac noise regression",
+                ),
+                (
+                    meanvals.reshape((xsize, ysize, numslices)),
+                    "cardfiltMean",
+                    "map",
+                    None,
+                    "Mean values after temporal cardiac noise regression",
+                ),
+                (
+                    rvals.reshape((xsize, ysize, numslices)),
+                    "cardfiltR",
+                    "map",
+                    None,
+                    "R values for temporal cardiac noise regression",
+                ),
+            ]
+            tide_io.savemaplist(
+                outputroot,
+                maplist,
+                None,
+                (xsize, ysize, numslices),
                 theheader,
-                cardfiltmeanfilename,
-            )
-            tide_io.savetonifti(
-                rvals.reshape((xsize, ysize, numslices)), theheader, cardfiltRfilename
+                bidsdict,
+                debug=args.debug,
             )
 
-            theheader = input_data.copyheader()
-            cardfiltresultfilename = outputroot + "_desc-cardfiltResult_bold"
-            cardfiltremovedfilename = outputroot + "_desc-cardfiltRemoved_bold"
-            tide_io.savetonifti(
+        # now write out the filtered data
+        theheader = input_data.copyheader()
+        maplist = [
+            (
                 filtereddata.reshape((xsize, ysize, numslices, timepoints)),
-                theheader,
-                cardfiltresultfilename,
-            )
-            tide_io.savetonifti(
+                "cardfiltResult",
+                "bold",
+                None,
+                f"Cardiac filtered BOLD data after {regressiontype} regression",
+            ),
+            (
                 datatoremove.reshape((xsize, ysize, numslices, timepoints)),
-                theheader,
-                cardfiltremovedfilename,
-            )
-            timings.append(
-                [
-                    "Cardiac signal temporal regression files written",
-                    time.time(),
-                    None,
-                    None,
-                ]
-            )
+                "cardfiltRemoved",
+                "bold",
+                None,
+                f"Cardiac noise removed with {regressiontype} regression",
+            ),
+        ]
+        tide_io.savemaplist(
+            outputroot,
+            maplist,
+            None,
+            (xsize, ysize, numslices, timepoints),
+            theheader,
+            bidsdict,
+            debug=args.debug,
+        )
+        timings.append(
+            [
+                f"Cardiac signal {regressiontype} regression files written",
+                time.time(),
+                None,
+                None,
+            ]
+        )
+
+    # clean up shared memory, if using
+    if args.dotemporalregression or args.dospatialregression:
+        if infodict["sharedmem"]:
+            tide_util.cleanup_shm(filtereddata_shm)
+            tide_util.cleanup_shm(datatoremove_shm)
+            tide_util.cleanup_shm(meanvals_shm)
+            tide_util.cleanup_shm(rvals_shm)
+            tide_util.cleanup_shm(r2vals_shm)
+            tide_util.cleanup_shm(fitcoffs_shm)
+            tide_util.cleanup_shm(fitNorm_shm)
 
     timings.append(["Done", time.time(), None, None])