rapidtide 3.0a13__py3-none-any.whl → 3.0a15__py3-none-any.whl

rapidtide/workflows/rapidtide.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 #
-#   Copyright 2016-2024 Blaise Frederick
+#   Copyright 2016-2025 Blaise Frederick
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -51,7 +51,9 @@ import rapidtide.resample as tide_resample
 import rapidtide.simfuncfit as tide_simfuncfit
 import rapidtide.stats as tide_stats
 import rapidtide.util as tide_util
+import rapidtide.voxelData as tide_voxelData
 import rapidtide.wiener as tide_wiener
+import rapidtide.workflows.cleanregressor as tide_cleanregressor
 import rapidtide.workflows.regressfrommaps as tide_regressfrommaps
 from rapidtide.tests.utils import mse
 
@@ -64,32 +66,11 @@ try:
 except ImportError:
     mklexists = False
 
-try:
-    from memory_profiler import profile
-
-    memprofilerexists = True
-except ImportError:
-    memprofilerexists = False
-
 LGR = logging.getLogger("GENERAL")
 ErrorLGR = logging.getLogger("ERROR")
 TimingLGR = logging.getLogger("TIMING")
 
 
-def conditionalprofile():
-    def resdec(f):
-        if memprofilerexists:
-            return profile(f)
-        return f
-
-    return resdec
-
-
-@conditionalprofile()
-def memcheckpoint(message):
-    LGR.info(message)
-
-
 def getglobalsignal(
     indata, optiondict, includemask=None, excludemask=None, pcacomponents=0.8, debug=False
 ):
@@ -159,14 +140,6 @@ def getglobalsignal(
     return tide_math.stdnormalize(globalmean), themask
 
 
-def addmemprofiling(thefunc, memprofile, themessage):
-    tide_util.logmem(themessage)
-    if memprofile:
-        return profile(thefunc, precision=2)
-    else:
-        return thefunc
-
-
 def checkforzeromean(thedataset):
     themean = np.mean(thedataset, axis=1)
     thestd = np.std(thedataset, axis=1)
@@ -216,21 +189,6 @@ def echocancel(thetimecourse, echooffset, thetimestep, outputname, padtimepoints
     return outputtimecourse, echofit, echoR2
 
 
-def disablemkl(numprocs, debug=False):
-    if mklexists:
-        if numprocs > 1:
-            if debug:
-                print("disablemkl: setting threads to 1")
-            mkl.set_num_threads(1)
-
-
-def enablemkl(numthreads, debug=False):
-    if mklexists:
-        if debug:
-            print(f"enablemkl: setting threads to {numthreads}")
-        mkl.set_num_threads(numthreads)
-
-
 def rapidtide_main(argparsingfunc):
     threaddebug = False
     optiondict, theprefilter = argparsingfunc
@@ -249,7 +207,7 @@ def rapidtide_main(argparsingfunc):
     optiondict["Description"] = (
         "A detailed dump of all internal variables in the program.  Useful for debugging and data provenance."
     )
-    fmrifilename = optiondict["in_file"]
+    inputdatafilename = optiondict["in_file"]
     outputname = optiondict["outputname"]
     regressorfilename = optiondict["regressorfile"]
 
@@ -392,7 +350,7 @@ def rapidtide_main(argparsingfunc):
         mklmaxthreads = mkl.get_max_threads()
         if not (1 <= optiondict["mklthreads"] <= mklmaxthreads):
             optiondict["mklthreads"] = mklmaxthreads
-        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
         LGR.info(f"using {optiondict['mklthreads']} MKL threads")
 
     # Generate MemoryLGR output file with column names
@@ -401,77 +359,30 @@ def rapidtide_main(argparsingfunc):
     ####################################################
     #  Read data
     ####################################################
-    # open the fmri datafile
-    tide_util.logmem("before reading in fmri data")
-    if tide_io.checkiftext(fmrifilename):
-        LGR.debug("input file is text - all I/O will be to text files")
-        optiondict["textio"] = True
-        if optiondict["gausssigma"] > 0.0:
-            optiondict["gausssigma"] = 0.0
-            LGR.info("gaussian spatial filter disabled for text input files")
-    else:
+    # read the fmri datafile
+    tide_util.logmem("before reading in input data")
+    theinputdata = tide_voxelData.VoxelData(inputdatafilename, timestep=optiondict["realtr"])
+    if optiondict["debug"]:
+        theinputdata.summarize()
+    nim = theinputdata.nim
+    nim_hdr = theinputdata.nim_hdr
+    nim_affine = theinputdata.nim_affine
+    xsize, ysize, numslices, timepoints = theinputdata.getdims()
+    thesizes = theinputdata.thesizes
+    xdim, ydim, slicethickness, fmritr = theinputdata.getsizes()
+    numspatiallocs = theinputdata.numspatiallocs
+    nativespaceshape = theinputdata.nativespaceshape
+    cifti_hdr = theinputdata.cifti_hdr
+    fmritr = theinputdata.timestep
+    if theinputdata.filetype == "cifti":
+        fileiscifti = True
         optiondict["textio"] = False
-
-    if optiondict["textio"]:
-        nim_data = tide_io.readvecs(fmrifilename)
-        nim_hdr = None
-        nim_affine = None
-        theshape = np.shape(nim_data)
-        xsize = theshape[0]
-        ysize = 1
-        numslices = 1
+    elif theinputdata.filetype == "text":
         fileiscifti = False
-        timepoints = theshape[1]
-        thesizes = [0, int(xsize), 1, 1, int(timepoints)]
-        numspatiallocs = int(xsize)
-        nativespaceshape = xsize
-        cifti_hdr = None
-    else:
-        fileiscifti = tide_io.checkifcifti(fmrifilename)
-        if fileiscifti:
-            LGR.debug("input file is CIFTI")
-            (
-                dummy,
-                cifti_hdr,
-                nim_data,
-                nim_hdr,
-                thedims,
-                thesizes,
-                dummy,
-            ) = tide_io.readfromcifti(fmrifilename)
-            nim_affine = None
-            optiondict["isgrayordinate"] = True
-            timepoints = nim_data.shape[1]
-            numspatiallocs = nim_data.shape[0]
-            LGR.debug(f"cifti file has {timepoints} timepoints, {numspatiallocs} numspatiallocs")
-            nativespaceshape = (1, 1, 1, 1, numspatiallocs)
-        else:
-            LGR.debug("input file is NIFTI")
-            nim, nim_data, nim_hdr, thedims, thesizes = tide_io.readfromnifti(fmrifilename)
-            nim_affine = nim.affine
-            optiondict["isgrayordinate"] = False
-            xsize, ysize, numslices, timepoints = tide_io.parseniftidims(thedims)
-            numspatiallocs = int(xsize) * int(ysize) * int(numslices)
-            cifti_hdr = None
-            nativespaceshape = (xsize, ysize, numslices)
-        xdim, ydim, slicethickness, dummy = tide_io.parseniftisizes(thesizes)
-
-    # correct some fields if necessary
-    if fileiscifti:
-        fmritr = 0.72  # this is wrong and is a hack until I can parse CIFTI XML
+        optiondict["textio"] = True
     else:
-        if optiondict["textio"]:
-            if optiondict["realtr"] <= 0.0:
-                raise ValueError(
-                    "for text file data input, you must use the -t option to set the timestep"
-                )
-        else:
-            if nim_hdr.get_xyzt_units()[1] == "msec":
-                fmritr = thesizes[4] / 1000.0
-            else:
-                fmritr = thesizes[4]
-    if optiondict["realtr"] > 0.0:
-        fmritr = optiondict["realtr"]
+        fileiscifti = False
+        optiondict["textio"] = False
 
     # check to see if we need to adjust the oversample factor
     if optiondict["oversampfactor"] < 0:
@@ -498,6 +409,8 @@ def rapidtide_main(argparsingfunc):
         raise ValueError(
             f"magnitude of lagmax exceeds {(validend - validstart + 1) * fmritr / 2.0} - invalid"
         )
+    # trim the fmri data to the limits
+    theinputdata.setvalidtimes(validstart, validend)
 
     # determine the valid timepoints
     validtimepoints = validend - validstart + 1
@@ -551,70 +464,33 @@ def rapidtide_main(argparsingfunc):
     whitemask = anatomicmasks[2]
 
     # do spatial filtering if requested
-    if fileiscifti:
-        optiondict["gausssigma"] = 0.0
-    if optiondict["gausssigma"] < 0.0 and not optiondict["textio"]:
-        # set gausssigma automatically
-        optiondict["gausssigma"] = np.mean([xdim, ydim, slicethickness]) / 2.0
+    optiondict["gausssigma"] = theinputdata.smooth(
+        optiondict["gausssigma"],
+        brainmask=brainmask,
+        graymask=graymask,
+        whitemask=whitemask,
+        premask=optiondict["premask"],
+        premasktissueonly=optiondict["premasktissueonly"],
+        showprogressbar=optiondict["showprogressbar"],
+    )
     if optiondict["gausssigma"] > 0.0:
-        # premask data if requested
-        if optiondict["premask"]:
-            if optiondict["premasktissueonly"]:
-                if (graymask is not None) and (whitemask is not None):
-                    multmask = graymask + whitemask
-                else:
-                    raise ValueError(
-                        "ERROR: graymask and whitemask must be defined to use premasktissueonly - exiting"
-                    )
-            else:
-                if brainmask is not None:
-                    multmask = brainmask
-                else:
-                    raise ValueError("ERROR: brainmask must be defined to use premask - exiting")
-            LGR.info(f"premasking timepoints {validstart} to {validend}")
-            for i in tqdm(
-                range(validstart, validend + 1),
-                desc="Timepoint",
-                unit="timepoints",
-                disable=(not optiondict["showprogressbar"]),
-            ):
-                nim_data[:, :, :, i] *= multmask
-
-        # this is where you'd slice time correct
-        # but we don't anymore
-
-        # now apply the filter
-        LGR.info(
-            f"applying gaussian spatial filter to timepoints {validstart} "
-            f"to {validend} with sigma={optiondict['gausssigma']}"
-        )
-        for i in tqdm(
-            range(validstart, validend + 1),
-            desc="Timepoint",
-            unit="timepoints",
-            disable=(not optiondict["showprogressbar"]),
-        ):
-            nim_data[:, :, :, i] = tide_filt.ssmooth(
-                xdim,
-                ydim,
-                slicethickness,
-                optiondict["gausssigma"],
-                nim_data[:, :, :, i],
-            )
         TimingLGR.info("End 3D smoothing")
 
     # reshape the data and trim to a time range, if specified.  Check for special case of no trimming to save RAM
-    fmri_data = nim_data.reshape((numspatiallocs, timepoints))[:, validstart : validend + 1]
+    fmri_data = theinputdata.voxelbytime()
+    print(f"{fmri_data.shape=}")
 
     # detect zero mean data
-    optiondict["dataiszeromean"] = checkforzeromean(fmri_data)
+    if not optiondict["dataiszeromean"]:
+        # check anyway
+        optiondict["dataiszeromean"] = checkforzeromean(fmri_data)
+
     if optiondict["dataiszeromean"]:
         LGR.warning(
             "WARNING: dataset is zero mean - forcing variance masking and no refine prenormalization. "
             "Consider specifying a global mean and correlation mask."
         )
         optiondict["refineprenorm"] = "None"
-        optiondict["globalmaskmethod"] = "variance"
 
     # reformat the brain mask, if it exists
     if brainmask is None:
@@ -716,18 +592,18 @@ def rapidtide_main(argparsingfunc):
         corrmask[np.where(datarange == 0)] = 0.0
     else:
         # check to see if the data has been demeaned
-        meanim = np.mean(fmri_data, axis=1)
-        stdim = np.std(fmri_data, axis=1)
-        if fileiscifti:
-            corrmask = np.uint(nim_data[:, 0] * 0 + 1)
+        if fileiscifti or optiondict["textio"]:
+            corrmask = np.uint(theinputdata.voxelbytime()[:, 0] * 0 + 1)
         else:
-            if (np.mean(stdim) < np.mean(meanim)) and not optiondict["nirs"]:
+            if not optiondict["dataiszeromean"]:
                 LGR.verbose("generating correlation mask from mean image")
                 corrmask = np.uint16(tide_mask.makeepimask(nim).dataobj.reshape(numspatiallocs))
             else:
                 LGR.verbose("generating correlation mask from std image")
                 corrmask = np.uint16(
-                    tide_stats.makemask(stdim, threshpct=optiondict["corrmaskthreshpct"])
+                    tide_stats.makemask(
+                        np.std(fmri_data, axis=1), threshpct=optiondict["corrmaskthreshpct"]
+                    )
                 )
     if internalbrainmask is not None:
         corrmask = internalbrainmask
@@ -777,7 +653,8 @@ def rapidtide_main(argparsingfunc):
         np.savetxt(f"{outputname}_validvoxels.txt", validvoxels)
     numvalidspatiallocs = np.shape(validvoxels)[0]
     LGR.debug(f"validvoxels shape = {numvalidspatiallocs}")
-    fmri_data_valid = fmri_data[validvoxels, :] + 0.0
+    theinputdata.setvalidvoxels(validvoxels)
+    fmri_data_valid = theinputdata.getvalidvoxels() + 0.0
     LGR.verbose(
         f"original size = {np.shape(fmri_data)}, trimmed size = {np.shape(fmri_data_valid)}"
     )
@@ -822,10 +699,7 @@ def rapidtide_main(argparsingfunc):
     if optiondict["sharedmem"]:
         LGR.info("moving fmri data to shared memory")
         TimingLGR.verbose("Start moving fmri_data to shared memory")
-        numpy2shared_func = addmemprofiling(
-            tide_util.numpy2shared, optiondict["memprofile"], "before fmri data move"
-        )
-        fmri_data_valid, fmri_data_valid_shm = numpy2shared_func(
+        fmri_data_valid, fmri_data_valid_shm = tide_util.numpy2shared(
             fmri_data_valid, rt_floatset, name=f"fmri_data_valid_{optiondict['pid']}"
         )
         TimingLGR.verbose("End moving fmri_data to shared memory")
@@ -857,7 +731,7 @@ def rapidtide_main(argparsingfunc):
     else:
         doconfounds = False
 
-    # now actually do the filtering
+    # now actually do the confound filtering
     if domotion or doconfounds:
         LGR.info("Doing confound filtering")
         TimingLGR.verbose("Confound filtering start")
@@ -883,7 +757,14 @@ def rapidtide_main(argparsingfunc):
             append=False,
         )
 
-        disablemkl(optiondict["nprocs_confoundregress"], debug=threaddebug)
+        if optiondict["focaldebug"]:
+            print(f"{mergedregressors.shape=}")
+            print(f"{mergedregressorlabels}")
+            print(f"{fmri_data_valid.shape=}")
+            print(f"{fmritr=}")
+            print(f"{validstart=}")
+            print(f"{validend=}")
+        tide_util.disablemkl(optiondict["nprocs_confoundregress"], debug=threaddebug)
         (
             mergedregressors,
             mergedregressorlabels,
@@ -900,7 +781,7 @@ def rapidtide_main(argparsingfunc):
             orthogonalize=optiondict["orthogonalize"],
             showprogressbar=optiondict["showprogressbar"],
         )
-        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
         if confoundr2 is None:
             print("There are no nonzero confound regressors - exiting")
             sys.exit()
@@ -959,8 +840,6 @@ def rapidtide_main(argparsingfunc):
             },
             append=False,
         )
-        if optiondict["memprofile"]:
-            memcheckpoint("...done")
         tide_util.logmem("after confound sLFO filter")
 
         if optiondict["saveconfoundfiltered"]:
@@ -1006,11 +885,11 @@ def rapidtide_main(argparsingfunc):
     # get rid of memory we aren't using
     tide_util.logmem("before purging full sized fmri data")
     meanvalue = np.mean(
-        nim_data.reshape((numspatiallocs, timepoints))[:, validstart : validend + 1],
+        theinputdata.voxelbytime(),
         axis=1,
     )
     stddevvalue = np.std(
-        nim_data.reshape((numspatiallocs, timepoints))[:, validstart : validend + 1],
+        theinputdata.voxelbytime(),
         axis=1,
     )
     covvalue = np.where(meanvalue > 0.0, stddevvalue / meanvalue, 0.0)
@@ -1033,8 +912,7 @@ def rapidtide_main(argparsingfunc):
     )
 
     # get rid of more memory we aren't using
-    del fmri_data
-    del nim_data
+    theinputdata.unload()
     uncollected = gc.collect()
     if uncollected != 0:
         print(f"garbage collected - unable to collect {uncollected} objects")
@@ -1147,59 +1025,6 @@ def rapidtide_main(argparsingfunc):
     LGR.verbose(f"the timepoint spacing is {1.0 / inputfreq}")
     LGR.verbose(f"the input timecourse start time is {inputstarttime}")
 
-    # if there is an externally specified noise regressor, read it in here
-    if optiondict["noisetimecoursespec"] is not None:
-        noisetimecoursespec = optiondict["noisetimecoursespec"]
-        LGR.info(f"using externally supplied noise regressor {noisetimecoursespec}")
-        (
-            filenoisefreq,
-            filenoisestarttime,
-            dummy,
-            noisevec,
-            dummy,
-            dummy,
-        ) = tide_io.readvectorsfromtextfile(optiondict["noisetimecoursespec"], onecol=True)
-        if optiondict["noiseinvert"]:
-            noisevec = noisevec * -1.0
-        noisefreq = optiondict["noisefreq"]
-        noisestarttime = optiondict["noisestarttime"]
-        if noisefreq is None:
-            if filenoisefreq is not None:
-                noisefreq = filenoisefreq
-            else:
-                noisefreq = 1.0 / fmritr
-            LGR.warning(f"no regressor frequency specified - defaulting to {noisefreq} (1/tr)")
-        if noisestarttime is None:
-            if filenoisestarttime is not None:
-                noisestarttime = filenoisestarttime
-            else:
-                LGR.warning("no regressor start time specified - defaulting to 0.0")
-                noisestarttime = 0.0
-        noiseperiod = 1.0 / noisefreq
-        numnoise = len(noisevec)
-        optiondict["noisefreq"] = noisefreq
-        optiondict["noisestarttime"] = noisestarttime
-        LGR.debug(
-            "Noise timecourse start time, end time, and step: {:.3f}, {:.3f}, {:.3f}".format(
-                -noisestarttime, noisestarttime + numnoise * noiseperiod, noiseperiod
-            )
-        )
-        noise_x = np.arange(0.0, numnoise) * noiseperiod - noisestarttime
-        noise_y = noisevec[0:numnoise] - np.mean(noisevec[0:numnoise])
-        # write out the noise regressor as read
-        tide_io.writebidstsv(
-            f"{outputname}_desc-initialnoiseregressor_timeseries",
-            noise_y,
-            noisefreq,
-            starttime=-noisestarttime,
-            columns=["prefilt"],
-            append=False,
-        )
-        LGR.verbose("noise vector")
-        LGR.verbose(f"length: {len(noisevec)}")
-        LGR.verbose(f"noise freq: {noisefreq}")
-        LGR.verbose(f"noise start time: {noisestarttime:.3f}")
-
     # generate the time axes
     fmrifreq = 1.0 / fmritr
     optiondict["fmrifreq"] = fmrifreq
@@ -1276,8 +1101,6 @@ def rapidtide_main(argparsingfunc):
         reference_y = -np.gradient(reference_y_classfilter)
     else:
         reference_y = reference_y_classfilter
-    if optiondict["noisetimecoursespec"] is not None:
-        noise_y = theprefilter.apply(noisefreq, noise_y)
 
     # write out the reference regressor used
     tide_io.writebidstsv(
@@ -1304,26 +1127,6 @@ def rapidtide_main(argparsingfunc):
             debug=optiondict["debug"],
         )
         reference_y = rt_floatset(reference_y_filt.real)
-        if optiondict["noisetimecoursespec"] is not None:
-            noise_y_filt = tide_filt.dolptrapfftfilt(
-                noisefreq,
-                0.25 * fmrifreq,
-                0.5 * fmrifreq,
-                noise_y,
-                padlen=int(noisefreq * optiondict["padseconds"]),
-                debug=optiondict["debug"],
-            )
-            noise_y = rt_floatset(noise_y_filt.real)
-
-            # write out the noise regressor after filtering
-            tide_io.writebidstsv(
-                f"{outputname}_desc-initialnoiseregressor_timeseries",
-                noise_y,
-                noisefreq,
-                starttime=-noisestarttime,
-                columns=["postfilt"],
-                append=True,
-            )
 
     warnings.filterwarnings("ignore", "Casting*")
 
@@ -1357,22 +1160,6 @@ def rapidtide_main(argparsingfunc):
             order=optiondict["detrendorder"],
             demean=optiondict["dodemean"],
         )
-        if optiondict["noisetimecoursespec"] is not None:
-            if optiondict["detrendorder"] > 0:
-                resampnoise_y = tide_fit.detrend(
-                    tide_resample.doresample(
-                        noise_x,
-                        noise_y,
-                        os_fmri_x,
-                        padlen=int(oversampfreq * optiondict["padseconds"]),
-                        padtype="zero",
-                        method=optiondict["interptype"],
-                        debug=optiondict["debug"],
-                    ),
-                    order=optiondict["detrendorder"],
-                    demean=optiondict["dodemean"],
-                )
-
     else:
         resampnonosref_y = tide_resample.doresample(
             reference_x,
@@ -1388,15 +1175,6 @@ def rapidtide_main(argparsingfunc):
             padlen=int(oversampfreq * optiondict["padseconds"]),
             method=optiondict["interptype"],
         )
-        if optiondict["noisetimecoursespec"] is not None:
-            resampnoise_y = tide_resample.doresample(
-                noise_x,
-                noise_y,
-                os_fmri_x,
-                padlen=int(oversampfreq * optiondict["padseconds"]),
-                padtype="zero",
-                method=optiondict["interptype"],
-            )
 
     LGR.debug(
         f"{len(os_fmri_x)} "
@@ -1458,22 +1236,6 @@ def rapidtide_main(argparsingfunc):
     else:
         tmaskos_y = None
 
-    if optiondict["noisetimecoursespec"] is not None:
-        tide_io.writebidstsv(
-            f"{outputname}_desc-noiseregressor_timeseries",
-            tide_math.stdnormalize(resampnonosref_y),
-            1.0 / fmritr,
-            columns=["resampled"],
-            append=False,
-        )
-        tide_io.writebidstsv(
-            f"{outputname}_desc-oversamplednoiseregressor_timeseries",
-            tide_math.stdnormalize(resampref_y),
-            oversampfreq,
-            columns=["oversampled"],
-            append=False,
-        )
-
     (
         optiondict["kurtosis_reference_pass1"],
         optiondict["kurtosisz_reference_pass1"],
@@ -1664,7 +1426,7 @@ def rapidtide_main(argparsingfunc):
     print(f"allocated {thesize:.3f} {theunit} {ramlocation} for correlation")
     tide_util.logmem("after correlation array allocation")
 
-    # if there is a fixed delay map, read it in
+    # if there is an initial delay map, read it in
     if optiondict["initialdelayvalue"] is not None:
         try:
             theinitialdelay = float(optiondict["initialdelayvalue"])
@@ -1699,9 +1461,10 @@ def rapidtide_main(argparsingfunc):
         + 30.0
         + np.abs(optiondict["offsettime"])
     )
-    LGR.info(f"setting up fast resampling with padtime = {optiondict['fastresamplerpadtime']}")
     numpadtrs = int(optiondict["fastresamplerpadtime"] // fmritr)
     optiondict["fastresamplerpadtime"] = fmritr * numpadtrs
+    LGR.info(f"setting up fast resampling with padtime = {optiondict['fastresamplerpadtime']}")
+
     genlagtc = tide_resample.FastResampler(
         reference_x, reference_y, padtime=optiondict["fastresamplerpadtime"]
     )
@@ -1729,13 +1492,14 @@ def rapidtide_main(argparsingfunc):
         2 * numpadtrs + np.shape(initial_fmri_x)[0],
     )
 
+    # prepare for regressor refinement, if we're doing it
     if (
         optiondict["passes"] > 1
         or optiondict["globalpreselect"]
         or optiondict["dofinalrefine"]
         or optiondict["convergencethresh"] is not None
     ):
-        # we will be doing regressor refinement, so set that up
+        # we will be doing regressor refinement, so configure the refiner
         theRegressorRefiner = tide_regrefiner.RegressorRefiner(
             internalvalidfmrishape,
             internalvalidpaddedfmrishape,
@@ -1776,8 +1540,6 @@ def rapidtide_main(argparsingfunc):
             debug=optiondict["debug"],
         )
 
-    outfmriarray = np.zeros(internalfmrishape, dtype=rt_floattype)
-
     # cycle over all voxels
     refine = True
     LGR.verbose(f"refine is set to {refine}")
@@ -1787,7 +1549,7 @@ def rapidtide_main(argparsingfunc):
     LGR.verbose(f"edgebufferfrac set to {optiondict['edgebufferfrac']}")
 
     # initialize the correlation fitter
-    thefitter = tide_classes.SimilarityFunctionFitter(
+    theFitter = tide_classes.SimilarityFunctionFitter(
         lagmod=optiondict["lagmod"],
         lthreshval=optiondict["lthreshval"],
         uthreshval=optiondict["uthreshval"],
@@ -1808,11 +1570,6 @@ def rapidtide_main(argparsingfunc):
     if optiondict["echocancel"]:
         LGR.info("\n\nEcho cancellation")
         TimingLGR.info("Echo cancellation start")
-        calcsimilaritypass_func = addmemprofiling(
-            tide_calcsimfunc.correlationpass,
-            optiondict["memprofile"],
-            "before correlationpass",
-        )
 
         referencetc = tide_math.corrnormalize(
             resampref_y[osvalidsimcalcstart : osvalidsimcalcend + 1],
@@ -1820,12 +1577,12 @@ def rapidtide_main(argparsingfunc):
             windowfunc=optiondict["windowfunc"],
         )
 
-        disablemkl(optiondict["nprocs_calcsimilarity"], debug=threaddebug)
+        tide_util.disablemkl(optiondict["nprocs_calcsimilarity"], debug=threaddebug)
         (
             voxelsprocessed_echo,
             theglobalmaxlist,
             trimmedcorrscale,
-        ) = calcsimilaritypass_func(
+        ) = tide_calcsimfunc.correlationpass(
             fmri_data_valid[:, validsimcalcstart : validsimcalcend + 1],
             referencetc,
             theCorrelator,
@@ -1844,7 +1601,7 @@ def rapidtide_main(argparsingfunc):
             rt_floatset=rt_floatset,
             rt_floattype=rt_floattype,
         )
-        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
         for i in range(len(theglobalmaxlist)):
             theglobalmaxlist[i] = corrscale[theglobalmaxlist[i]] - optiondict["simcalcoffset"]
@@ -1913,237 +1670,51 @@ def rapidtide_main(argparsingfunc):
         )
 
         # Step -1 - check the regressor for periodic components in the passband
-        dolagmod = True
-        doreferencenotch = True
-        if optiondict["respdelete"]:
-            resptracker = tide_classes.FrequencyTracker(nperseg=64)
-            thetimes, thefreqs = resptracker.track(resampref_y, 1.0 / oversamptr)
-            tide_io.writevec(thefreqs, f"{outputname}_peakfreaks_pass{thepass}.txt")
-            resampref_y = resptracker.clean(resampref_y, 1.0 / oversamptr, thetimes, thefreqs)
-            tide_io.writevec(resampref_y, f"{outputname}_respfilt_pass{thepass}.txt")
-            referencetc = tide_math.corrnormalize(
-                resampref_y[osvalidsimcalcstart : osvalidsimcalcend + 1],
-                detrendorder=optiondict["detrendorder"],
-                windowfunc=optiondict["windowfunc"],
-            )
-        if optiondict["noisetimecoursespec"] is not None:
-            # align the noise signal with referencetc
-            (
-                shiftednoise,
-                optiondict[f"noisedelay_pass{thepass}"],
-                optiondict[f"noisecorr_pass{thepass}"],
-                dummy,
-            ) = tide_corr.aligntcwithref(
-                resampref_y,
-                resampnoise_y,
-                oversampfreq,
-                zerooutbadfit=False,
-                verbose=True,
-            )
-            LGR.info(
-                "Maximum correlation amplitude with noise regressor is "
-                + str(optiondict[f"noisecorr_pass{thepass}"])
-                + " at "
-                + str(optiondict[f"noisedelay_pass{thepass}"])
-            )
-
-            # regress out
-            resampref_y, datatoremove, R, dummy = tide_fit.linfitfilt(
-                resampref_y, shiftednoise, debug=True
-            )
-
-            # save
-            tide_io.writebidstsv(
-                f"{outputname}_desc-regressornoiseremoval_timeseries",
-                shiftednoise,
-                1.0 / oversamptr,
-                starttime=0.0,
-                columns=[f"shiftednoise_pass{thepass}"],
-                append=(thepass > 1),
-            )
-            tide_io.writebidstsv(
-                f"{outputname}_desc-regressornoiseremoval_timeseries",
-                datatoremove,
-                1.0 / oversamptr,
-                starttime=0.0,
-                columns=[f"removed_pass{thepass}"],
-                append=True,
-            )
-            tide_io.writebidstsv(
-                f"{outputname}_desc-regressornoiseremoval_timeseries",
-                resampref_y,
-                1.0 / oversamptr,
-                starttime=0.0,
-                columns=[f"filtered_pass{thepass}"],
-                append=True,
-            )
-
-        if optiondict["check_autocorrelation"]:
-            LGR.info("checking reference regressor autocorrelation properties")
-            optiondict["lagmod"] = 1000.0
-            lagindpad = np.max((lagmininpts, lagmaxinpts))
-            acmininpts = lagindpad
-            acmaxinpts = lagindpad
-            theCorrelator.setreftc(referencetc)
-            theCorrelator.setlimits(acmininpts, acmaxinpts)
-            # theCorrelator.setlimits(lagmininpts, lagmaxinpts)
-            print("check_autocorrelation:", acmininpts, acmaxinpts, lagmininpts, lagmaxinpts)
-            thexcorr, accheckcorrscale, dummy = theCorrelator.run(
-                resampref_y[osvalidsimcalcstart : osvalidsimcalcend + 1]
-            )
-            thefitter.setcorrtimeaxis(accheckcorrscale)
-            (
-                dummy,
-                dummy,
-                dummy,
-                acwidth,
-                dummy,
-                dummy,
-                dummy,
-                dummy,
-            ) = tide_simfuncfit.onesimfuncfit(
-                thexcorr,
-                thefitter,
-                despeckle_thresh=optiondict["despeckle_thresh"],
-                lthreshval=optiondict["lthreshval"],
-                fixdelay=optiondict["fixdelay"],
-                rt_floatset=rt_floatset,
-                rt_floattype=rt_floattype,
-            )
-            tide_io.writebidstsv(
-                f"{outputname}_desc-autocorr_timeseries",
-                thexcorr,
-                1.0 / (accheckcorrscale[1] - accheckcorrscale[0]),
-                starttime=accheckcorrscale[0],
-                extraheaderinfo={
-                    "Description": "Autocorrelation of the probe regressor for each pass"
-                },
-                columns=[f"pass{thepass}"],
-                append=(thepass > 1),
-            )
-            thelagthresh = np.max((abs(optiondict["lagmin"]), abs(optiondict["lagmax"])))
-            theampthresh = 0.1
-            LGR.info(
-                f"searching for sidelobes with amplitude > {theampthresh} "
-                f"with abs(lag) < {thelagthresh} s"
-            )
-            sidelobetime, sidelobeamp = tide_corr.check_autocorrelation(
-                accheckcorrscale,
-                thexcorr,
-                acampthresh=theampthresh,
-                aclagthresh=thelagthresh,
-                detrendorder=optiondict["detrendorder"],
-            )
-            optiondict["acwidth"] = acwidth + 0.0
-            optiondict["absmaxsigma"] = acwidth * 10.0
-            passsuffix = "_pass" + str(thepass)
-            if sidelobetime is not None:
-                optiondict["acsidelobelag" + passsuffix] = sidelobetime
-                optiondict["despeckle_thresh"] = np.max(
-                    [optiondict["despeckle_thresh"], sidelobetime / 2.0]
-                )
-                optiondict["acsidelobeamp" + passsuffix] = sidelobeamp
-                LGR.warning(
-                    f"\n\nWARNING: check_autocorrelation found bad sidelobe at {sidelobetime} "
-                    f"seconds ({1.0 / sidelobetime} Hz)..."
-                )
-                # bidsify
-                """tide_io.writebidstsv(
-                    f"{outputname}_desc-movingregressor_timeseries",
-                    tide_math.stdnormalize(resampnonosref_y),
-                    1.0 / fmritr,
-                    columns=["pass1"],
-                    append=False,
-                )"""
-                tide_io.writenpvecs(
-                    np.array([sidelobetime]),
-                    f"{outputname}_autocorr_sidelobetime" + passsuffix + ".txt",
-                )
-                if optiondict["fix_autocorrelation"]:
-                    LGR.info("Removing sidelobe")
-                    if dolagmod:
-                        LGR.info("subjecting lag times to modulus")
-                        optiondict["lagmod"] = sidelobetime / 2.0
-                    if doreferencenotch:
-                        LGR.info("removing spectral component at sidelobe frequency")
-                        acstopfreq = 1.0 / sidelobetime
-                        acfixfilter = tide_filt.NoncausalFilter(
-                            debug=optiondict["debug"],
-                        )
-                        acfixfilter.settype("arb_stop")
-                        acfixfilter.setfreqs(
-                            acstopfreq * 0.9,
-                            acstopfreq * 0.95,
-                            acstopfreq * 1.05,
-                            acstopfreq * 1.1,
-                        )
-                        cleaned_resampref_y = tide_math.corrnormalize(
-                            acfixfilter.apply(1.0 / oversamptr, resampref_y),
-                            windowfunc="None",
-                            detrendorder=optiondict["detrendorder"],
-                        )
-                        cleaned_referencetc = tide_math.corrnormalize(
-                            cleaned_resampref_y,
-                            detrendorder=optiondict["detrendorder"],
-                            windowfunc=optiondict["windowfunc"],
-                        )
-                        cleaned_nonosreferencetc = tide_math.stdnormalize(
-                            acfixfilter.apply(fmrifreq, resampnonosref_y)
-                        )
-                        tide_io.writebidstsv(
-                            f"{outputname}_desc-cleanedreferencefmrires_info",
-                            cleaned_nonosreferencetc,
-                            fmrifreq,
-                            columns=[f"pass{thepass}"],
-                            append=(thepass > 1),
-                        )
-                        tide_io.writebidstsv(
-                            f"{outputname}_desc-cleanedreference_info",
-                            cleaned_referencetc,
-                            1.0 / oversamptr,
-                            columns=[f"pass{thepass}"],
-                            append=(thepass > 1),
-                        )
-                        tide_io.writebidstsv(
-                            f"{outputname}_desc-cleanedresamprefy_info",
-                            cleaned_resampref_y,
-                            1.0 / oversamptr,
-                            columns=[f"pass{thepass}"],
-                            append=(thepass > 1),
-                        )
-                else:
-                    cleaned_resampref_y = 1.0 * tide_math.corrnormalize(
-                        resampref_y,
-                        windowfunc="None",
-                        detrendorder=optiondict["detrendorder"],
-                    )
-                    cleaned_referencetc = 1.0 * referencetc
-                    cleaned_nonosreferencetc = 1.0 * resampnonosref_y
-            else:
-                LGR.info("no sidelobes found in range")
-                cleaned_resampref_y = 1.0 * tide_math.corrnormalize(
-                    resampref_y,
-                    windowfunc="None",
-                    detrendorder=optiondict["detrendorder"],
-                )
-                cleaned_referencetc = 1.0 * referencetc
-                cleaned_nonosreferencetc = 1.0 * resampnonosref_y
-        else:
-            cleaned_resampref_y = 1.0 * tide_math.corrnormalize(
-                resampref_y, windowfunc="None", detrendorder=optiondict["detrendorder"]
-            )
-            cleaned_referencetc = 1.0 * referencetc
-            cleaned_nonosreferencetc = 1.0 * resampnonosref_y
+        passsuffix = "_pass" + str(thepass)
+        (
+            cleaned_resampref_y,
+            cleaned_referencetc,
+            cleaned_nonosreferencetc,
+            optiondict["despeckle_thresh"],
+            optiondict["acsidelobeamp" + passsuffix],
+            optiondict["acsidelobelag" + passsuffix],
+            optiondict["lagmod"],
+            optiondict["acwidth"],
+            optiondict["absmaxsigma"],
+        ) = tide_cleanregressor.cleanregressor(
+            outputname,
+            thepass,
+            referencetc,
+            resampref_y,
+            resampnonosref_y,
+            fmrifreq,
+            oversampfreq,
+            osvalidsimcalcstart,
+            osvalidsimcalcend,
+            lagmininpts,
+            lagmaxinpts,
+            theFitter,
+            theCorrelator,
+            optiondict["lagmin"],
+            optiondict["lagmax"],
+            LGR=LGR,
+            check_autocorrelation=optiondict["check_autocorrelation"],
+            fix_autocorrelation=optiondict["fix_autocorrelation"],
+            despeckle_thresh=optiondict["despeckle_thresh"],
+            lthreshval=optiondict["lthreshval"],
+            fixdelay=optiondict["fixdelay"],
+            detrendorder=optiondict["detrendorder"],
+            windowfunc=optiondict["windowfunc"],
+            respdelete=optiondict["respdelete"],
+            debug=optiondict["debug"],
+            rt_floattype=rt_floattype,
+            rt_floatset=rt_floatset,
+        )
 
         # Step 0 - estimate significance
         if optiondict["numestreps"] > 0:
             TimingLGR.info(f"Significance estimation start, pass {thepass}")
             LGR.info(f"\n\nSignificance estimation, pass {thepass}")
-            getNullDistributionData_func = addmemprofiling(
-                tide_nullsimfunc.getNullDistributionDatax,
-                optiondict["memprofile"],
-                "before getnulldistristributiondata",
-            )
             if optiondict["checkpoint"]:
                 # bidsify
                 """tide_io.writebidstsv(
@@ -2170,19 +1741,19 @@ def rapidtide_main(argparsingfunc):
             theMutualInformationator.setlimits(lagmininpts, lagmaxinpts)
             theMutualInformationator.setreftc(cleaned_resampref_y)
             dummy, trimmedcorrscale, dummy = theCorrelator.getfunction()
-            thefitter.setcorrtimeaxis(trimmedcorrscale)
+            theFitter.setcorrtimeaxis(trimmedcorrscale)
 
             # parallel path for mutual information
             if optiondict["similaritymetric"] == "mutualinfo":
                 theSimFunc = theMutualInformationator
             else:
                 theSimFunc = theCorrelator
-            disablemkl(optiondict["nprocs_getNullDist"], debug=threaddebug)
-            simdistdata = getNullDistributionData_func(
+            tide_util.disablemkl(optiondict["nprocs_getNullDist"], debug=threaddebug)
+            simdistdata = tide_nullsimfunc.getNullDistributionDatax(
                 cleaned_resampref_y,
                 oversampfreq,
                 theSimFunc,
-                thefitter,
+                theFitter,
                 numestreps=optiondict["numestreps"],
                 nprocs=optiondict["nprocs_getNullDist"],
                 alwaysmultiproc=optiondict["alwaysmultiproc"],
@@ -2193,7 +1764,7 @@ def rapidtide_main(argparsingfunc):
                 rt_floatset=np.float64,
                 rt_floattype="float64",
             )
-            enablemkl(optiondict["mklthreads"], debug=threaddebug)
+            tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
             tide_io.writebidstsv(
                 f"{outputname}_desc-simdistdata_info",
@@ -2284,11 +1855,20 @@ def rapidtide_main(argparsingfunc):
                     "message3": "repetitions",
                 },
             )
-
         # write out the current version of the run options
         optiondict["currentstage"] = f"precorrelation_pass{thepass}"
         tide_io.writedicttojson(optiondict, f"{outputname}_desc-runoptions_info.json")
 
+        # tide_delayestimate.estimateDelay(
+        #    fmri_data_valid[:, validsimcalcstart : validsimcalcend + 1],
+        #    initial_fmri_x[validsimcalcstart : validsimcalcend + 1],
+        #    os_fmri_x[osvalidsimcalcstart : osvalidsimcalcend + 1],
+        #    theMutualInformationator,
+        #    thepass,
+        # )
+        ########################
+        # Delay estimation start
+        ########################
         # Step 1 - Correlation step
         if optiondict["similaritymetric"] == "mutualinfo":
             similaritytype = "Mutual information"
@@ -2298,20 +1878,15 @@ def rapidtide_main(argparsingfunc):
             similaritytype = "MI enhanced correlation"
         LGR.info(f"\n\n{similaritytype} calculation, pass {thepass}")
         TimingLGR.info(f"{similaritytype} calculation start, pass {thepass}")
-        calcsimilaritypass_func = addmemprofiling(
-            tide_calcsimfunc.correlationpass,
-            optiondict["memprofile"],
-            "before correlationpass",
-        )
 
-        disablemkl(optiondict["nprocs_calcsimilarity"], debug=threaddebug)
+        tide_util.disablemkl(optiondict["nprocs_calcsimilarity"], debug=threaddebug)
         if optiondict["similaritymetric"] == "mutualinfo":
             theMutualInformationator.setlimits(lagmininpts, lagmaxinpts)
             (
                 voxelsprocessed_cp,
                 theglobalmaxlist,
                 trimmedcorrscale,
-            ) = calcsimilaritypass_func(
+            ) = tide_calcsimfunc.correlationpass(
                 fmri_data_valid[:, validsimcalcstart : validsimcalcend + 1],
                 cleaned_referencetc,
                 theMutualInformationator,
@@ -2335,7 +1910,7 @@ def rapidtide_main(argparsingfunc):
                 voxelsprocessed_cp,
                 theglobalmaxlist,
                 trimmedcorrscale,
-            ) = calcsimilaritypass_func(
+            ) = tide_calcsimfunc.correlationpass(
                 fmri_data_valid[:, validsimcalcstart : validsimcalcend + 1],
                 cleaned_referencetc,
                 theCorrelator,
@@ -2354,7 +1929,7 @@ def rapidtide_main(argparsingfunc):
                 rt_floatset=rt_floatset,
                 rt_floattype=rt_floattype,
             )
-        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
         for i in range(len(theglobalmaxlist)):
             theglobalmaxlist[i] = corrscale[theglobalmaxlist[i]] - optiondict["simcalcoffset"]
@@ -2396,14 +1971,9 @@ def rapidtide_main(argparsingfunc):
         if optiondict["similaritymetric"] == "hybrid":
             LGR.info(f"\n\nPeak prefit calculation, pass {thepass}")
             TimingLGR.info(f"Peak prefit calculation start, pass {thepass}")
-            peakevalpass_func = addmemprofiling(
-                tide_peakeval.peakevalpass,
-                optiondict["memprofile"],
-                "before peakevalpass",
-            )
 
-            disablemkl(optiondict["nprocs_peakeval"], debug=threaddebug)
-            voxelsprocessed_pe, thepeakdict = peakevalpass_func(
+            tide_util.disablemkl(optiondict["nprocs_peakeval"], debug=threaddebug)
+            voxelsprocessed_pe, thepeakdict = tide_peakeval.peakevalpass(
                 fmri_data_valid[:, validsimcalcstart : validsimcalcend + 1],
                 cleaned_referencetc,
                 initial_fmri_x[validsimcalcstart : validsimcalcend + 1],
@@ -2421,7 +1991,7 @@ def rapidtide_main(argparsingfunc):
                 rt_floatset=rt_floatset,
                 rt_floattype=rt_floattype,
             )
-            enablemkl(optiondict["mklthreads"], debug=threaddebug)
+            tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
             TimingLGR.info(
                 f"Peak prefit end, pass {thepass}",
@@ -2443,11 +2013,9 @@ def rapidtide_main(argparsingfunc):
         tide_io.writedicttojson(optiondict, f"{outputname}_desc-runoptions_info.json")
         LGR.info(f"\n\nTime lag estimation pass {thepass}")
         TimingLGR.info(f"Time lag estimation start, pass {thepass}")
-        fitcorr_func = addmemprofiling(
-            tide_simfuncfit.fitcorr, optiondict["memprofile"], "before fitcorr"
-        )
-        thefitter.setfunctype(optiondict["similaritymetric"])
-        thefitter.setcorrtimeaxis(trimmedcorrscale)
+
+        theFitter.setfunctype(optiondict["similaritymetric"])
+        theFitter.setcorrtimeaxis(trimmedcorrscale)
 
         # use initial lags if this is a hybrid fit
         if optiondict["similaritymetric"] == "hybrid" and thepeakdict is not None:
@@ -2455,10 +2023,10 @@ def rapidtide_main(argparsingfunc):
         else:
             initlags = None
 
-        disablemkl(optiondict["nprocs_fitcorr"], debug=threaddebug)
-        voxelsprocessed_fc = fitcorr_func(
+        tide_util.disablemkl(optiondict["nprocs_fitcorr"], debug=threaddebug)
+        voxelsprocessed_fc = tide_simfuncfit.fitcorr(
             trimmedcorrscale,
-            thefitter,
+            theFitter,
             corrout,
             fitmask,
             failreason,
@@ -2481,7 +2049,7 @@ def rapidtide_main(argparsingfunc):
             rt_floatset=rt_floatset,
             rt_floattype=rt_floattype,
         )
-        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
         TimingLGR.info(
             f"Time lag estimation end, pass {thepass}",
@@ -2521,10 +2089,10 @@ def rapidtide_main(argparsingfunc):
                     numdespeckled = len(np.where(initlags != -1000000.0)[0])
                     if lastnumdespeckled > numdespeckled > 0:
                         lastnumdespeckled = numdespeckled
-                        disablemkl(optiondict["nprocs_fitcorr"], debug=threaddebug)
-                        voxelsprocessed_thispass = fitcorr_func(
+                        tide_util.disablemkl(optiondict["nprocs_fitcorr"], debug=threaddebug)
+                        voxelsprocessed_thispass = tide_simfuncfit.fitcorr(
                             trimmedcorrscale,
-                            thefitter,
+                            theFitter,
                             corrout,
                             fitmask,
                             failreason,
@@ -2547,7 +2115,7 @@ def rapidtide_main(argparsingfunc):
                             rt_floatset=rt_floatset,
                             rt_floattype=rt_floattype,
                         )
-                        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+                        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
                         voxelsprocessed_fc_ds += voxelsprocessed_thispass
                         optiondict[
@@ -2740,6 +2308,9 @@ def rapidtide_main(argparsingfunc):
 
             # now shift the patches to align with the majority of the image
             tide_patch.interppatch(lagtimes, patchmap[validvoxels])
+        ########################
+        # Delay estimation end
+        ########################
 
         # Step 2d - make a rank order map
         timepercentile = (
@@ -2869,11 +2440,11 @@ def rapidtide_main(argparsingfunc):
 
             # align timecourses to prepare for refinement
             LGR.info("aligning timecourses")
-            disablemkl(optiondict["nprocs_refine"], debug=threaddebug)
+            tide_util.disablemkl(optiondict["nprocs_refine"], debug=threaddebug)
             voxelsprocessed_rra = theRegressorRefiner.alignvoxels(
                 fmri_data_valid, fmritr, lagtimes
             )
-            enablemkl(optiondict["mklthreads"], debug=threaddebug)
+            tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
             LGR.info(f"align complete: {voxelsprocessed_rra=}")
 
             # prenormalize
@@ -2908,57 +2479,6 @@ def rapidtide_main(argparsingfunc):
             )
             for key, value in outputdict.items():
                 optiondict[key] = value
-
-        """if optiondict["saveintermediatemaps"]:
-            if not optiondict["textio"]:
-                theheader = copy.deepcopy(nim_hdr)
-                if fileiscifti:
-                    timeindex = theheader["dim"][0] - 1
-                    spaceindex = theheader["dim"][0]
-                    theheader["dim"][timeindex] = 1
-                    theheader["dim"][spaceindex] = numspatiallocs
-                else:
-                    theheader["dim"][0] = 3
-                    theheader["dim"][4] = 1
-                    theheader["pixdim"][4] = 1.0
-            bidspasssuffix = f"_intermediatedata-pass{thepass}"
-            maplist = [
-                (fitmask, "corrfit", "mask", None, "Voxels where correlation value was fit"),
-                (
-                    failreason,
-                    "corrfitfailreason",
-                    "info",
-                    None,
-                    "Result codes for correlation fit",
-                ),
-            ]
-            if optiondict["savedespecklemasks"] and (optiondict["despeckle_passes"] > 0):
-                maplist.append(
-                    (
-                        despecklesavemask,
-                        "despecklemask",
-                        "map",
-                        None,
-                        "Voxels that underwent despeckling",
-                    )
-                )
-            if thepass < optiondict["passes"]:
-                maplist.append(
-                    (refinemask, "refinemask", "map", None, "Voxels used for regressor refinement")
-                )
-            tide_io.savemaplist(
-                f"{outputname}{bidspasssuffix}",
-                maplist,
-                validvoxels,
-                nativespaceshape,
-                theheader,
-                bidsbasedict,
-                textio=optiondict["textio"],
-                fileiscifti=fileiscifti,
-                rt_floattype=rt_floattype,
-                cifti_hdr=cifti_hdr,
-            )"""
-
         # We are done with refinement.
 
     if optiondict["convergencethresh"] is None:
@@ -3045,13 +2565,8 @@ def rapidtide_main(argparsingfunc):
         thesize, theunit = tide_util.format_bytes(optiondict["totalcoherencebytes"])
         print(f"allocated {thesize:.3f} {theunit} {ramlocation} for coherence calculation")
 
-        coherencepass_func = addmemprofiling(
-            tide_calccoherence.coherencepass,
-            optiondict["memprofile"],
-            "before coherencepass",
-        )
-        disablemkl(1, debug=threaddebug)
-        voxelsprocessed_coherence = coherencepass_func(
+        tide_util.disablemkl(1, debug=threaddebug)
+        voxelsprocessed_coherence = tide_calccoherence.coherencepass(
             fmri_data_valid,
             theCoherer,
             coherencefunc,
@@ -3065,7 +2580,7 @@ def rapidtide_main(argparsingfunc):
             rt_floatset=rt_floatset,
             rt_floattype=rt_floattype,
         )
-        enablemkl(optiondict["mklthreads"], debug=threaddebug)
+        tide_util.enablemkl(optiondict["mklthreads"], debug=threaddebug)
 
         # save the results of the calculations
         if not optiondict["textio"]:
@@ -3125,12 +2640,7 @@ def rapidtide_main(argparsingfunc):
         thesize, theunit = tide_util.format_bytes(optiondict["totalwienerbytes"])
         print(f"allocated {thesize:.3f} {theunit} {ramlocation} for wiener deconvolution")
 
-        wienerpass_func = addmemprofiling(
-            tide_wiener.wienerpass,
-            optiondict["memprofile"],
-            "before wienerpass",
-        )
-        voxelsprocessed_wiener = wienerpass_func(
+        voxelsprocessed_wiener = tide_wiener.wienerpass(
             numspatiallocs,
             fmri_data_valid,
             threshval,
@@ -3189,29 +2699,11 @@ def rapidtide_main(argparsingfunc):
                     f"reading in {optiondict['denoisesourcefile']} for sLFO filter, please wait"
                 )
                 sourcename = optiondict["denoisesourcefile"]
-            else:
-                LGR.info(f"rereading {fmrifilename} for sLFO filter, please wait")
-                sourcename = fmrifilename
-            if fileiscifti:
-                LGR.info("input file is CIFTI")
-                (
-                    dummy,
-                    cifti_hdr,
-                    nim_data,
-                    nim_hdr,
-                    thedims,
-                    thesizes,
-                    dummy,
-                ) = tide_io.readfromcifti(sourcename)
-            else:
-                if optiondict["textio"]:
-                    nim_data = tide_io.readvecs(sourcename)
-                else:
-                    nim, nim_data, nim_hdr, thedims, thesizes = tide_io.readfromnifti(sourcename)
+                theinputdata = tide_voxelData.VoxelData(sourcename, timestep=optiondict["realtr"])
+                theinputdata.setvalidtimes(validstart, validend)
+                theinputdata.setvalidvoxels(validvoxels)
 
-            fmri_data_valid = (
-                nim_data.reshape((numspatiallocs, timepoints))[:, validstart : validend + 1]
-            )[validvoxels, :] + 0.0
+            fmri_data_valid = theinputdata.getvalidvoxels() + 0.0
 
             if optiondict["docvrmap"]:
                 # percent normalize the fmri data
@@ -3234,18 +2726,13 @@ def rapidtide_main(argparsingfunc):
                 tide_util.cleanup_shm(fmri_data_valid_shm)
                 LGR.info("moving fmri data to shared memory")
                 TimingLGR.info("Start moving fmri_data to shared memory")
-                numpy2shared_func = addmemprofiling(
-                    tide_util.numpy2shared,
-                    optiondict["memprofile"],
-                    "before movetoshared (sLFO filter)",
-                )
-                fmri_data_valid, fmri_data_valid_shm = numpy2shared_func(
+                fmri_data_valid, fmri_data_valid_shm = tide_util.numpy2shared(
                     fmri_data_valid,
                     rt_floatset,
                     name=f"fmri_data_valid_regressionfilt_{optiondict['pid']}",
                 )
                 TimingLGR.info("End moving fmri_data to shared memory")
-            del nim_data
+            theinputdata.unload()
 
         # now allocate the arrays needed for sLFO filtering
         if optiondict["refinedelay"]:
@@ -3307,12 +2794,6 @@ def rapidtide_main(argparsingfunc):
         )
         thesize, theunit = tide_util.format_bytes(optiondict["totalsLFOfilterbytes"])
         print(f"allocated {thesize:.3f} {theunit} {ramlocation} for sLFO filter/delay refinement")
-
-        if optiondict["memprofile"]:
-            if optiondict["dolinfitfilt"]:
-                memcheckpoint("about to start sLFO noise removal...")
-            else:
-                memcheckpoint("about to start CVR magnitude estimation...")
         tide_util.logmem("before sLFO filter")
 
         if optiondict["dolinfitfilt"]:
@@ -3322,9 +2803,11 @@ def rapidtide_main(argparsingfunc):
             # set the threshval to zero
             mode = "cvrmap"
             optiondict["regressfiltthreshval"] = 0.0
+
         if optiondict["debug"]:
             # dump the fmri input file going to sLFO filter
             if not optiondict["textio"]:
+                outfmriarray = np.zeros(internalfmrishape, dtype=rt_floattype)
                 theheader = copy.deepcopy(nim_hdr)
                 if fileiscifti:
                     timeindex = theheader["dim"][0] - 1
@@ -3337,6 +2820,7 @@ def rapidtide_main(argparsingfunc):
             else:
                 theheader = None
                 cifti_hdr = None
+                outfmriarray = None
 
             maplist = [
                 (
@@ -3359,6 +2843,8 @@ def rapidtide_main(argparsingfunc):
                 rt_floattype=rt_floattype,
                 cifti_hdr=cifti_hdr,
             )
+        else:
+            outfmriarray = None
 
         # refine the delay value prior to calculating the sLFO filter
         if optiondict["refinedelay"]:
@@ -3429,7 +2915,9 @@ def rapidtide_main(argparsingfunc):
                         f"calculating delayoffsets for {filteredregressderivratios.shape[0]} voxels"
                     )
                 for i in range(filteredregressderivratios.shape[0]):
-                    delayoffset[i] = tide_refinedelay.ratiotodelay(filteredregressderivratios[i])
+                    delayoffset[i], closestoffset = tide_refinedelay.ratiotodelay(
+                        filteredregressderivratios[i]
+                    )
             else:
                 medfiltregressderivratios = np.zeros_like(regressderivratios)
                 filteredregressderivratios = np.zeros_like(regressderivratios)
@@ -3530,7 +3018,6 @@ def rapidtide_main(argparsingfunc):
                     mp_chunksize=optiondict["mp_chunksize"],
                     showprogressbar=optiondict["showprogressbar"],
                     alwaysmultiproc=optiondict["alwaysmultiproc"],
-                    memprofile=optiondict["memprofile"],
                     debug=optiondict["debug"],
                 )
             )
@@ -3565,10 +3052,10 @@ def rapidtide_main(argparsingfunc):
                     "message3": "voxels",
                 },
             )
-            if optiondict["memprofile"]:
-                memcheckpoint("...done")
             tide_util.logmem("after sLFO filter")
             LGR.info("")
+    else:
+        outfmriarray = None
     ####################################################
     #  sLFO filtering end
     ####################################################
@@ -3980,7 +3467,6 @@ def rapidtide_main(argparsingfunc):
                 mp_chunksize=optiondict["mp_chunksize"],
                 showprogressbar=optiondict["showprogressbar"],
                 alwaysmultiproc=optiondict["alwaysmultiproc"],
-                memprofile=optiondict["memprofile"],
                 debug=optiondict["debug"],
             )
 
@@ -4180,6 +3666,8 @@ def rapidtide_main(argparsingfunc):
         tide_util.cleanup_shm(outcorrarray_shm)
 
     # now save all the files that are of the same length as the input data file and masked
+    if outfmriarray is None:
+        outfmriarray = np.zeros(internalfmrishape, dtype=rt_floattype)
     if not optiondict["textio"]:
         theheader = copy.deepcopy(nim_hdr)
         if fileiscifti: