rapidtide 3.0a11__py3-none-any.whl → 3.0a13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. cloud/gmscalc-HCPYA +1 -1
  2. cloud/rapidtide-HCPYA +3 -3
  3. rapidtide/Colortables.py +10 -10
  4. rapidtide/DerivativeDelay.py +211 -0
  5. rapidtide/RegressorRefiner.py +464 -0
  6. rapidtide/__init__.py +2 -1
  7. rapidtide/_version.py +1 -1
  8. rapidtide/data/examples/src/test_mlregressallt.py +32 -17
  9. rapidtide/data/examples/src/testalign +1 -1
  10. rapidtide/data/examples/src/testboth +1 -1
  11. rapidtide/data/examples/src/testcifti +11 -0
  12. rapidtide/data/examples/src/testdelayvar +13 -0
  13. rapidtide/data/examples/src/testfmri +3 -124
  14. rapidtide/data/examples/src/testglmfilt +8 -6
  15. rapidtide/data/examples/src/testhappy +1 -1
  16. rapidtide/data/examples/src/testinitdelay +19 -0
  17. rapidtide/data/examples/src/testnewrefine +49 -0
  18. rapidtide/data/examples/src/testnoiseamp +2 -2
  19. rapidtide/data/examples/src/testrefineonly +22 -0
  20. rapidtide/data/examples/src/testretro +16 -7
  21. rapidtide/data/examples/src/testretrolagtcs +1 -1
  22. rapidtide/dlfilter.py +0 -1
  23. rapidtide/fit.py +41 -9
  24. rapidtide/happy_supportfuncs.py +5 -0
  25. rapidtide/io.py +13 -2
  26. rapidtide/{glmpass.py → linfitfiltpass.py} +29 -20
  27. rapidtide/refinedelay.py +133 -55
  28. rapidtide/refineregressor.py +38 -24
  29. rapidtide/resample.py +3 -0
  30. rapidtide/scripts/{retroglm.py → delayvar.py} +2 -2
  31. rapidtide/scripts/{glmfilt.py → linfitfilt.py} +2 -2
  32. rapidtide/scripts/retroregress.py +28 -0
  33. rapidtide/scripts/stupidramtricks.py +9 -7
  34. rapidtide/simfuncfit.py +1 -1
  35. rapidtide/tests/cleanposttest +21 -0
  36. rapidtide/tests/test_delayestimation.py +3 -3
  37. rapidtide/tests/test_fastresampler.py +1 -2
  38. rapidtide/tests/test_fullrunhappy_v1.py +1 -1
  39. rapidtide/tests/test_fullrunhappy_v2.py +1 -1
  40. rapidtide/tests/test_fullrunrapidtide_v1.py +2 -2
  41. rapidtide/tests/test_fullrunrapidtide_v3.py +1 -1
  42. rapidtide/tests/test_fullrunrapidtide_v5.py +1 -1
  43. rapidtide/tests/test_fullrunrapidtide_v6.py +11 -11
  44. rapidtide/tests/{test_glmpass.py → test_linfitfiltpass.py} +9 -9
  45. rapidtide/tests/test_motionregress.py +3 -3
  46. rapidtide/tests/test_refinedelay.py +12 -12
  47. rapidtide/tidepoolTemplate.py +1 -0
  48. rapidtide/tidepoolTemplate.ui +1 -0
  49. rapidtide/tidepoolTemplate_alt.py +5 -4
  50. rapidtide/tidepoolTemplate_alt.ui +3 -2
  51. rapidtide/tidepoolTemplate_alt_qt6.py +177 -49
  52. rapidtide/tidepoolTemplate_big.py +1 -0
  53. rapidtide/tidepoolTemplate_big.ui +1 -0
  54. rapidtide/tidepoolTemplate_big_qt6.py +197 -53
  55. rapidtide/tidepoolTemplate_qt6.py +151 -39
  56. rapidtide/workflows/delayvar.py +1048 -0
  57. rapidtide/workflows/happy.py +37 -11
  58. rapidtide/workflows/happy_parser.py +4 -4
  59. rapidtide/workflows/{glmfilt.py → linfitfilt.py} +4 -4
  60. rapidtide/workflows/parser_funcs.py +10 -2
  61. rapidtide/workflows/rapidtide.py +388 -452
  62. rapidtide/workflows/rapidtide_parser.py +129 -90
  63. rapidtide/workflows/{glmfrommaps.py → regressfrommaps.py} +28 -26
  64. rapidtide/workflows/retrolagtcs.py +12 -12
  65. rapidtide/workflows/{retroglm.py → retroregress.py} +243 -141
  66. rapidtide/workflows/tidepool.py +2 -2
  67. {rapidtide-3.0a11.dist-info → rapidtide-3.0a13.dist-info}/METADATA +3 -2
  68. {rapidtide-3.0a11.dist-info → rapidtide-3.0a13.dist-info}/RECORD +72 -63
  69. {rapidtide-3.0a11.dist-info → rapidtide-3.0a13.dist-info}/WHEEL +1 -1
  70. {rapidtide-3.0a11.dist-info → rapidtide-3.0a13.dist-info}/entry_points.txt +3 -2
  71. rapidtide/data/examples/src/testoutputsize +0 -45
  72. {rapidtide-3.0a11.dist-info → rapidtide-3.0a13.dist-info/licenses}/LICENSE +0 -0
  73. {rapidtide-3.0a11.dist-info → rapidtide-3.0a13.dist-info}/top_level.txt +0 -0
@@ -37,22 +37,22 @@ import rapidtide.calcsimfunc as tide_calcsimfunc
37
37
  import rapidtide.correlate as tide_corr
38
38
  import rapidtide.filter as tide_filt
39
39
  import rapidtide.fit as tide_fit
40
- import rapidtide.glmpass as tide_glmpass
41
40
  import rapidtide.helper_classes as tide_classes
42
41
  import rapidtide.io as tide_io
42
+ import rapidtide.linfitfiltpass as tide_linfitfiltpass
43
43
  import rapidtide.maskutil as tide_mask
44
44
  import rapidtide.miscmath as tide_math
45
45
  import rapidtide.multiproc as tide_multiproc
46
46
  import rapidtide.patchmatch as tide_patch
47
47
  import rapidtide.peakeval as tide_peakeval
48
48
  import rapidtide.refinedelay as tide_refinedelay
49
- import rapidtide.refineregressor as tide_refineregressor
49
+ import rapidtide.RegressorRefiner as tide_regrefiner
50
50
  import rapidtide.resample as tide_resample
51
51
  import rapidtide.simfuncfit as tide_simfuncfit
52
52
  import rapidtide.stats as tide_stats
53
53
  import rapidtide.util as tide_util
54
54
  import rapidtide.wiener as tide_wiener
55
- import rapidtide.workflows.glmfrommaps as tide_glmfrommaps
55
+ import rapidtide.workflows.regressfrommaps as tide_regressfrommaps
56
56
  from rapidtide.tests.utils import mse
57
57
 
58
58
  from .utils import setup_logger
@@ -382,10 +382,10 @@ def rapidtide_main(argparsingfunc):
382
382
  else:
383
383
  optiondict["nprocs_makelaggedtcs"] = optiondict["nprocs"]
384
384
 
385
- if optiondict["singleproc_glm"]:
386
- optiondict["nprocs_glm"] = 1
385
+ if optiondict["singleproc_regressionfilt"]:
386
+ optiondict["nprocs_regressionfilt"] = 1
387
387
  else:
388
- optiondict["nprocs_glm"] = optiondict["nprocs"]
388
+ optiondict["nprocs_regressionfilt"] = optiondict["nprocs"]
389
389
 
390
390
  # set the number of MKL threads to use
391
391
  if mklexists:
@@ -889,7 +889,7 @@ def rapidtide_main(argparsingfunc):
889
889
  mergedregressorlabels,
890
890
  fmri_data_valid,
891
891
  confoundr2,
892
- ) = tide_glmpass.confoundregress(
892
+ ) = tide_linfitfiltpass.confoundregress(
893
893
  mergedregressors,
894
894
  mergedregressorlabels,
895
895
  fmri_data_valid,
@@ -961,7 +961,7 @@ def rapidtide_main(argparsingfunc):
961
961
  )
962
962
  if optiondict["memprofile"]:
963
963
  memcheckpoint("...done")
964
- tide_util.logmem("after confound glm filter")
964
+ tide_util.logmem("after confound sLFO filter")
965
965
 
966
966
  if optiondict["saveconfoundfiltered"]:
967
967
  if not optiondict["textio"]:
@@ -1455,6 +1455,8 @@ def rapidtide_main(argparsingfunc):
1455
1455
  resampref_y *= tmaskos_y
1456
1456
  thefit, R2val = tide_fit.mlregress(tmaskos_y, resampref_y)
1457
1457
  resampref_y -= thefit[0, 1] * tmaskos_y
1458
+ else:
1459
+ tmaskos_y = None
1458
1460
 
1459
1461
  if optiondict["noisetimecoursespec"] is not None:
1460
1462
  tide_io.writebidstsv(
@@ -1730,38 +1732,49 @@ def rapidtide_main(argparsingfunc):
1730
1732
  if (
1731
1733
  optiondict["passes"] > 1
1732
1734
  or optiondict["globalpreselect"]
1735
+ or optiondict["dofinalrefine"]
1733
1736
  or optiondict["convergencethresh"] is not None
1734
1737
  ):
1735
- if optiondict["sharedmem"]:
1736
- shiftedtcs, shiftedtcs_shm = tide_util.allocshared(
1737
- internalvalidfmrishape, rt_floatset, name=f"shiftedtcs_{optiondict['pid']}"
1738
- )
1739
- weights, weights_shm = tide_util.allocshared(
1740
- internalvalidfmrishape, rt_floatset, name=f"weights_{optiondict['pid']}"
1741
- )
1742
- paddedshiftedtcs, paddedshiftedtcs_shm = tide_util.allocshared(
1743
- internalvalidpaddedfmrishape,
1744
- rt_floatset,
1745
- name=f"paddedshiftedtcs_{optiondict['pid']}",
1746
- )
1747
- paddedweights, paddedweights_shm = tide_util.allocshared(
1748
- internalvalidpaddedfmrishape,
1749
- rt_floatset,
1750
- name=f"paddedweights_{optiondict['pid']}",
1751
- )
1752
- ramlocation = "in shared memory"
1753
- else:
1754
- shiftedtcs = np.zeros(internalvalidfmrishape, dtype=rt_floattype)
1755
- weights = np.zeros(internalvalidfmrishape, dtype=rt_floattype)
1756
- paddedshiftedtcs = np.zeros(internalvalidpaddedfmrishape, dtype=rt_floattype)
1757
- paddedweights = np.zeros(internalvalidpaddedfmrishape, dtype=rt_floattype)
1758
- ramlocation = "locally"
1759
- optiondict["totalrefinementbytes"] = (
1760
- shiftedtcs.nbytes + weights.nbytes + paddedshiftedtcs.nbytes + paddedweights.nbytes
1738
+ # we will be doing regressor refinement, so set that up
1739
+ theRegressorRefiner = tide_regrefiner.RegressorRefiner(
1740
+ internalvalidfmrishape,
1741
+ internalvalidpaddedfmrishape,
1742
+ optiondict["pid"],
1743
+ optiondict["outputname"],
1744
+ initial_fmri_x,
1745
+ paddedinitial_fmri_x,
1746
+ os_fmri_x,
1747
+ sharedmem=optiondict["sharedmem"],
1748
+ offsettime=optiondict["offsettime"],
1749
+ ampthresh=optiondict["ampthresh"],
1750
+ lagminthresh=optiondict["lagminthresh"],
1751
+ lagmaxthresh=optiondict["lagmaxthresh"],
1752
+ sigmathresh=optiondict["sigmathresh"],
1753
+ cleanrefined=optiondict["cleanrefined"],
1754
+ bipolar=optiondict["bipolar"],
1755
+ fixdelay=optiondict["fixdelay"],
1756
+ LGR=LGR,
1757
+ nprocs=optiondict["nprocs_refine"],
1758
+ detrendorder=optiondict["detrendorder"],
1759
+ alwaysmultiproc=optiondict["alwaysmultiproc"],
1760
+ showprogressbar=optiondict["showprogressbar"],
1761
+ chunksize=optiondict["mp_chunksize"],
1762
+ padtrs=numpadtrs,
1763
+ refineprenorm=optiondict["refineprenorm"],
1764
+ refineweighting=optiondict["refineweighting"],
1765
+ refinetype=optiondict["refinetype"],
1766
+ pcacomponents=optiondict["pcacomponents"],
1767
+ windowfunc=optiondict["windowfunc"],
1768
+ passes=optiondict["passes"],
1769
+ maxpasses=optiondict["maxpasses"],
1770
+ convergencethresh=optiondict["convergencethresh"],
1771
+ interptype=optiondict["interptype"],
1772
+ usetmask=(optiondict["tincludemaskname"] is not None),
1773
+ tmask_y=tmask_y,
1774
+ tmaskos_y=tmaskos_y,
1775
+ fastresamplerpadtime=optiondict["fastresamplerpadtime"],
1776
+ debug=optiondict["debug"],
1761
1777
  )
1762
- thesize, theunit = tide_util.format_bytes(optiondict["totalrefinementbytes"])
1763
- print(f"allocated {thesize:.3f} {theunit} {ramlocation} for refinement")
1764
- tide_util.logmem("after refinement array allocation")
1765
1778
 
1766
1779
  outfmriarray = np.zeros(internalfmrishape, dtype=rt_floattype)
1767
1780
 
@@ -1935,7 +1948,7 @@ def rapidtide_main(argparsingfunc):
1935
1948
  )
1936
1949
 
1937
1950
  # regress out
1938
- resampref_y, datatoremove, R, dummy = tide_fit.glmfilt(
1951
+ resampref_y, datatoremove, R, dummy = tide_fit.linfitfilt(
1939
1952
  resampref_y, shiftednoise, debug=True
1940
1953
  )
1941
1954
 
@@ -2779,6 +2792,7 @@ def rapidtide_main(argparsingfunc):
2779
2792
  thepass < optiondict["passes"]
2780
2793
  or optiondict["convergencethresh"] is not None
2781
2794
  or optiondict["globalpreselect"]
2795
+ or optiondict["dofinalrefine"]
2782
2796
  ):
2783
2797
  LGR.info(f"\n\nRegressor refinement, pass {thepass}")
2784
2798
  TimingLGR.info(f"Regressor refinement start, pass {thepass}")
@@ -2840,254 +2854,51 @@ def rapidtide_main(argparsingfunc):
2840
2854
  "NB: cannot exclude despeckled voxels from refinement - including for this pass"
2841
2855
  )
2842
2856
  thisinternalrefineexcludemask_valid = internalrefineexcludemask_valid
2857
+ theRegressorRefiner.setmasks(
2858
+ internalrefineincludemask_valid, thisinternalrefineexcludemask_valid
2859
+ )
2843
2860
 
2844
2861
  # regenerate regressor for next pass
2845
2862
  # create the refinement mask
2846
2863
  LGR.info("making refine mask")
2847
- (
2848
- dummy,
2849
- refinemask,
2850
- locationfails,
2851
- ampfails,
2852
- lagfails,
2853
- sigmafails,
2854
- numinmask,
2855
- ) = tide_refineregressor.makerefinemask(
2856
- lagstrengths,
2857
- lagtimes,
2858
- lagsigma,
2859
- fitmask,
2860
- offsettime=optiondict["offsettime"],
2861
- ampthresh=optiondict["ampthresh"],
2862
- lagmaskside=optiondict["lagmaskside"],
2863
- lagminthresh=optiondict["lagminthresh"],
2864
- lagmaxthresh=optiondict["lagmaxthresh"],
2865
- sigmathresh=optiondict["sigmathresh"],
2866
- cleanrefined=optiondict["cleanrefined"],
2867
- bipolar=optiondict["bipolar"],
2868
- includemask=internalrefineincludemask_valid,
2869
- excludemask=thisinternalrefineexcludemask_valid,
2870
- )
2871
-
2872
- if numinmask == 0:
2873
- LGR.critical("No voxels in refine mask - adjust thresholds or external masks")
2864
+ createdmask = theRegressorRefiner.makemask(lagstrengths, lagtimes, lagsigma, fitmask)
2865
+ print(f"Refine mask has {theRegressorRefiner.refinemaskvoxels} voxels")
2866
+ if not createdmask:
2867
+ print("no voxels qualify for refinement - exiting")
2874
2868
  sys.exit()
2875
2869
 
2876
2870
  # align timecourses to prepare for refinement
2877
- alignvoxels_func = addmemprofiling(
2878
- tide_refineregressor.alignvoxels,
2879
- optiondict["memprofile"],
2880
- "before aligning voxel timecourses",
2881
- )
2882
2871
  LGR.info("aligning timecourses")
2883
2872
  disablemkl(optiondict["nprocs_refine"], debug=threaddebug)
2884
- voxelsprocessed_rra = alignvoxels_func(
2885
- fmri_data_valid,
2886
- fmritr,
2887
- shiftedtcs,
2888
- weights,
2889
- paddedshiftedtcs,
2890
- paddedweights,
2891
- lagtimes,
2892
- refinemask,
2893
- nprocs=optiondict["nprocs_refine"],
2894
- detrendorder=optiondict["detrendorder"],
2895
- offsettime=optiondict["offsettime"],
2896
- alwaysmultiproc=optiondict["alwaysmultiproc"],
2897
- showprogressbar=optiondict["showprogressbar"],
2898
- chunksize=optiondict["mp_chunksize"],
2899
- padtrs=numpadtrs,
2900
- rt_floatset=rt_floatset,
2901
- rt_floattype=rt_floattype,
2873
+ voxelsprocessed_rra = theRegressorRefiner.alignvoxels(
2874
+ fmri_data_valid, fmritr, lagtimes
2902
2875
  )
2903
2876
  enablemkl(optiondict["mklthreads"], debug=threaddebug)
2904
2877
  LGR.info(f"align complete: {voxelsprocessed_rra=}")
2905
2878
 
2879
+ # prenormalize
2906
2880
  LGR.info("prenormalizing timecourses")
2907
- tide_refineregressor.prenorm(
2908
- paddedshiftedtcs,
2909
- refinemask,
2910
- lagtimes,
2911
- optiondict["lagmaxthresh"],
2912
- lagstrengths,
2913
- R2,
2914
- optiondict["refineprenorm"],
2915
- optiondict["refineweighting"],
2916
- )
2881
+ theRegressorRefiner.prenormalize(lagtimes, lagstrengths, R2)
2917
2882
 
2883
+ # now doing the refinement
2918
2884
  (
2919
2885
  voxelsprocessed_rr,
2920
- paddedoutputdata,
2921
- ) = tide_refineregressor.dorefine(
2922
- paddedshiftedtcs,
2923
- refinemask,
2924
- weights,
2886
+ outputdict,
2887
+ previousnormoutputdata,
2888
+ resampref_y,
2889
+ resampnonosref_y,
2890
+ stoprefining,
2891
+ refinestopreason,
2892
+ genlagtc,
2893
+ ) = theRegressorRefiner.refine(
2925
2894
  theprefilter,
2926
2895
  fmritr,
2927
2896
  thepass,
2928
2897
  lagstrengths,
2929
2898
  lagtimes,
2930
- optiondict["refinetype"],
2931
- optiondict["fmrifreq"],
2932
- optiondict["outputname"],
2933
- detrendorder=optiondict["detrendorder"],
2934
- pcacomponents=optiondict["pcacomponents"],
2935
- dodispersioncalc=optiondict["dodispersioncalc"],
2936
- dispersioncalc_lower=optiondict["dispersioncalc_lower"],
2937
- dispersioncalc_upper=optiondict["dispersioncalc_upper"],
2938
- dispersioncalc_step=optiondict["dispersioncalc_step"],
2939
- windowfunc=optiondict["windowfunc"],
2940
- cleanrefined=optiondict["cleanrefined"],
2941
- bipolar=optiondict["bipolar"],
2942
- debug=optiondict["debug"],
2943
- rt_floatset=rt_floatset,
2944
- rt_floattype=rt_floattype,
2945
- )
2946
- optiondict["refinemasksize_pass" + str(thepass)] = voxelsprocessed_rr
2947
- optiondict["refinemaskpct_pass" + str(thepass)] = (
2948
- 100.0 * voxelsprocessed_rr / optiondict["corrmasksize"]
2899
+ previousnormoutputdata,
2900
+ optiondict["corrmasksize"],
2949
2901
  )
2950
- optiondict["refinelocationfails_pass" + str(thepass)] = locationfails
2951
- optiondict["refineampfails_pass" + str(thepass)] = ampfails
2952
- optiondict["refinelagfails_pass" + str(thepass)] = lagfails
2953
- optiondict["refinesigmafails_pass" + str(thepass)] = sigmafails
2954
- if voxelsprocessed_rr > 0:
2955
- paddednormoutputdata = tide_math.stdnormalize(
2956
- theprefilter.apply(fmrifreq, paddedoutputdata)
2957
- )
2958
- outputdata = paddedoutputdata[numpadtrs:-numpadtrs]
2959
- normoutputdata = tide_math.stdnormalize(theprefilter.apply(fmrifreq, outputdata))
2960
- normunfilteredoutputdata = tide_math.stdnormalize(outputdata)
2961
- tide_io.writebidstsv(
2962
- f"{outputname}_desc-refinedmovingregressor_timeseries",
2963
- normunfilteredoutputdata,
2964
- 1.0 / fmritr,
2965
- columns=["unfiltered_pass" + str(thepass)],
2966
- extraheaderinfo={
2967
- "Description": "The raw and filtered probe regressor produced by the refinement procedure, at the time resolution of the data"
2968
- },
2969
- append=(thepass > 1),
2970
- )
2971
- tide_io.writebidstsv(
2972
- f"{outputname}_desc-refinedmovingregressor_timeseries",
2973
- normoutputdata,
2974
- 1.0 / fmritr,
2975
- columns=["filtered_pass" + str(thepass)],
2976
- extraheaderinfo={
2977
- "Description": "The raw and filtered probe regressor produced by the refinement procedure, at the time resolution of the data"
2978
- },
2979
- append=True,
2980
- )
2981
-
2982
- # check for convergence
2983
- regressormse = mse(normoutputdata, previousnormoutputdata)
2984
- optiondict["regressormse_pass" + str(thepass).zfill(2)] = regressormse
2985
- LGR.info(f"regressor difference at end of pass {thepass:d} is {regressormse:.6f}")
2986
- if optiondict["convergencethresh"] is not None:
2987
- if thepass >= optiondict["maxpasses"]:
2988
- LGR.info("refinement ended (maxpasses reached)")
2989
- stoprefining = True
2990
- refinestopreason = "maxpassesreached"
2991
- elif regressormse < optiondict["convergencethresh"]:
2992
- LGR.info("refinement ended (refinement has converged")
2993
- stoprefining = True
2994
- refinestopreason = "convergence"
2995
- else:
2996
- stoprefining = False
2997
- elif thepass >= optiondict["passes"]:
2998
- stoprefining = True
2999
- refinestopreason = "passesreached"
3000
- else:
3001
- stoprefining = False
3002
-
3003
- if optiondict["detrendorder"] > 0:
3004
- resampnonosref_y = tide_fit.detrend(
3005
- tide_resample.doresample(
3006
- paddedinitial_fmri_x,
3007
- paddednormoutputdata,
3008
- initial_fmri_x,
3009
- method=optiondict["interptype"],
3010
- ),
3011
- order=optiondict["detrendorder"],
3012
- demean=optiondict["dodemean"],
3013
- )
3014
- resampref_y = tide_fit.detrend(
3015
- tide_resample.doresample(
3016
- paddedinitial_fmri_x,
3017
- paddednormoutputdata,
3018
- os_fmri_x,
3019
- method=optiondict["interptype"],
3020
- ),
3021
- order=optiondict["detrendorder"],
3022
- demean=optiondict["dodemean"],
3023
- )
3024
- else:
3025
- resampnonosref_y = tide_resample.doresample(
3026
- paddedinitial_fmri_x,
3027
- paddednormoutputdata,
3028
- initial_fmri_x,
3029
- method=optiondict["interptype"],
3030
- )
3031
- resampref_y = tide_resample.doresample(
3032
- paddedinitial_fmri_x,
3033
- paddednormoutputdata,
3034
- os_fmri_x,
3035
- method=optiondict["interptype"],
3036
- )
3037
- if optiondict["tincludemaskname"] is not None:
3038
- resampnonosref_y *= tmask_y
3039
- thefit, R2val = tide_fit.mlregress(tmask_y, resampnonosref_y)
3040
- resampnonosref_y -= thefit[0, 1] * tmask_y
3041
- resampref_y *= tmaskos_y
3042
- thefit, R2val = tide_fit.mlregress(tmaskos_y, resampref_y)
3043
- resampref_y -= thefit[0, 1] * tmaskos_y
3044
-
3045
- # reinitialize genlagtc for resampling
3046
- previousnormoutputdata = normoutputdata + 0.0
3047
- genlagtc = tide_resample.FastResampler(
3048
- paddedinitial_fmri_x,
3049
- paddednormoutputdata,
3050
- padtime=optiondict["fastresamplerpadtime"],
3051
- )
3052
- genlagtc.save(f"{outputname}_desc-lagtcgenerator_timeseries")
3053
- if optiondict["debug"]:
3054
- genlagtc.info()
3055
- (
3056
- optiondict[f"kurtosis_reference_pass{thepass + 1}"],
3057
- optiondict[f"kurtosisz_reference_pass{thepass + 1}"],
3058
- optiondict[f"kurtosisp_reference_pass{thepass + 1}"],
3059
- ) = tide_stats.kurtosisstats(resampref_y)
3060
- (
3061
- optiondict[f"skewness_reference_pass{thepass + 1}"],
3062
- optiondict[f"skewnessz_reference_pass{thepass + 1}"],
3063
- optiondict[f"skewnessp_reference_pass{thepass + 1}"],
3064
- ) = tide_stats.skewnessstats(resampref_y)
3065
- if not stoprefining:
3066
- tide_io.writebidstsv(
3067
- f"{outputname}_desc-movingregressor_timeseries",
3068
- tide_math.stdnormalize(resampnonosref_y),
3069
- 1.0 / fmritr,
3070
- columns=["pass" + str(thepass + 1)],
3071
- extraheaderinfo={
3072
- "Description": "The probe regressor used in each pass, at the time resolution of the data"
3073
- },
3074
- append=True,
3075
- )
3076
- tide_io.writebidstsv(
3077
- f"{outputname}_desc-oversampledmovingregressor_timeseries",
3078
- tide_math.stdnormalize(resampref_y),
3079
- oversampfreq,
3080
- columns=["pass" + str(thepass + 1)],
3081
- extraheaderinfo={
3082
- "Description": "The probe regressor used in each pass, at the time resolution used for calculating the similarity function"
3083
- },
3084
- append=True,
3085
- )
3086
- else:
3087
- LGR.warning(f"refinement failed - terminating at end of pass {thepass}")
3088
- stoprefining = True
3089
- refinestopreason = "emptymask"
3090
-
3091
2902
  TimingLGR.info(
3092
2903
  f"Regressor refinement end, pass {thepass}",
3093
2904
  {
@@ -3095,7 +2906,10 @@ def rapidtide_main(argparsingfunc):
3095
2906
  "message3": "voxels",
3096
2907
  },
3097
2908
  )
3098
- if optiondict["saveintermediatemaps"]:
2909
+ for key, value in outputdict.items():
2910
+ optiondict[key] = value
2911
+
2912
+ """if optiondict["saveintermediatemaps"]:
3099
2913
  if not optiondict["textio"]:
3100
2914
  theheader = copy.deepcopy(nim_hdr)
3101
2915
  if fileiscifti:
@@ -3143,9 +2957,10 @@ def rapidtide_main(argparsingfunc):
3143
2957
  fileiscifti=fileiscifti,
3144
2958
  rt_floattype=rt_floattype,
3145
2959
  cifti_hdr=cifti_hdr,
3146
- )
2960
+ )"""
2961
+
2962
+ # We are done with refinement.
3147
2963
 
3148
- # We are done with refinement.
3149
2964
  if optiondict["convergencethresh"] is None:
3150
2965
  optiondict["actual_passes"] = optiondict["passes"]
3151
2966
  else:
@@ -3340,20 +3155,20 @@ def rapidtide_main(argparsingfunc):
3340
3155
  tide_util.cleanup_shm(wpeak_shm)
3341
3156
 
3342
3157
  ####################################################
3343
- # GLM filtering start
3158
+ # Linear regression filtering start
3344
3159
  ####################################################
3345
- # Post refinement step 1 - GLM fitting, either to remove moving signal, or to calculate delayed CVR
3160
+ # Post refinement step 1 - regression fitting, either to remove moving signal, or to calculate delayed CVR
3346
3161
  # write out the current version of the run options
3347
- optiondict["currentstage"] = "preglm"
3162
+ optiondict["currentstage"] = "presLFOfit"
3348
3163
  tide_io.writedicttojson(optiondict, f"{outputname}_desc-runoptions_info.json")
3349
- if optiondict["doglmfilt"] or optiondict["docvrmap"] or optiondict["refinedelay"]:
3350
- if optiondict["doglmfilt"]:
3164
+ if optiondict["dolinfitfilt"] or optiondict["docvrmap"] or optiondict["refinedelay"]:
3165
+ if optiondict["dolinfitfilt"]:
3351
3166
  if optiondict["refinedelay"]:
3352
- TimingLGR.info("Setting up for delay refinement and GLM filtering")
3353
- LGR.info("\n\nDelay refinement and GLM filtering setup")
3167
+ TimingLGR.info("Setting up for delay refinement and sLFO filtering")
3168
+ LGR.info("\n\nDelay refinement and sLFO filtering setup")
3354
3169
  else:
3355
- TimingLGR.info("Setting up for GLM filtering")
3356
- LGR.info("\n\nGLM filtering setup")
3170
+ TimingLGR.info("Setting up for sLFO filtering")
3171
+ LGR.info("\n\nsLFO filtering setup")
3357
3172
  elif optiondict["docvrmap"]:
3358
3173
  if optiondict["refinedelay"]:
3359
3174
  TimingLGR.info("Setting up for delay refinement and CVR map generation")
@@ -3366,14 +3181,16 @@ def rapidtide_main(argparsingfunc):
3366
3181
  LGR.info("\n\nDelay refinement setup")
3367
3182
  if (
3368
3183
  (optiondict["gausssigma"] > 0.0)
3369
- or (optiondict["glmsourcefile"] is not None)
3184
+ or (optiondict["denoisesourcefile"] is not None)
3370
3185
  or optiondict["docvrmap"]
3371
3186
  ):
3372
- if optiondict["glmsourcefile"] is not None:
3373
- LGR.info(f"reading in {optiondict['glmsourcefile']} for GLM filter, please wait")
3374
- sourcename = optiondict["glmsourcefile"]
3187
+ if optiondict["denoisesourcefile"] is not None:
3188
+ LGR.info(
3189
+ f"reading in {optiondict['denoisesourcefile']} for sLFO filter, please wait"
3190
+ )
3191
+ sourcename = optiondict["denoisesourcefile"]
3375
3192
  else:
3376
- LGR.info(f"rereading {fmrifilename} for GLM filter, please wait")
3193
+ LGR.info(f"rereading {fmrifilename} for sLFO filter, please wait")
3377
3194
  sourcename = fmrifilename
3378
3195
  if fileiscifti:
3379
3196
  LGR.info("input file is CIFTI")
@@ -3420,26 +3237,30 @@ def rapidtide_main(argparsingfunc):
3420
3237
  numpy2shared_func = addmemprofiling(
3421
3238
  tide_util.numpy2shared,
3422
3239
  optiondict["memprofile"],
3423
- "before movetoshared (glm)",
3240
+ "before movetoshared (sLFO filter)",
3424
3241
  )
3425
3242
  fmri_data_valid, fmri_data_valid_shm = numpy2shared_func(
3426
- fmri_data_valid, rt_floatset, name=f"fmri_data_valid_glm_{optiondict['pid']}"
3243
+ fmri_data_valid,
3244
+ rt_floatset,
3245
+ name=f"fmri_data_valid_regressionfilt_{optiondict['pid']}",
3427
3246
  )
3428
3247
  TimingLGR.info("End moving fmri_data to shared memory")
3429
3248
  del nim_data
3430
3249
 
3431
- # now allocate the arrays needed for GLM filtering
3250
+ # now allocate the arrays needed for sLFO filtering
3432
3251
  if optiondict["refinedelay"]:
3433
- derivaxissize = np.max([2, optiondict["glmderivs"] + 1])
3252
+ derivaxissize = np.max(
3253
+ [optiondict["refineregressderivs"] + 1, optiondict["regressderivs"] + 1]
3254
+ )
3434
3255
  else:
3435
- derivaxissize = optiondict["glmderivs"] + 1
3256
+ derivaxissize = optiondict["regressderivs"] + 1
3436
3257
  internalvalidspaceshapederivs = (
3437
3258
  internalvalidspaceshape,
3438
3259
  derivaxissize,
3439
3260
  )
3440
3261
  if optiondict["sharedmem"]:
3441
- glmmean, glmmean_shm = tide_util.allocshared(
3442
- internalvalidspaceshape, rt_outfloatset, name=f"glmmean_{optiondict['pid']}"
3262
+ sLFOfitmean, sLFOfitmean_shm = tide_util.allocshared(
3263
+ internalvalidspaceshape, rt_outfloatset, name=f"sLFOfitmean_{optiondict['pid']}"
3443
3264
  )
3444
3265
  rvalue, rvalue_shm = tide_util.allocshared(
3445
3266
  internalvalidspaceshape, rt_outfloatset, name=f"rvalue_{optiondict['pid']}"
@@ -3464,7 +3285,7 @@ def rapidtide_main(argparsingfunc):
3464
3285
  )
3465
3286
  ramlocation = "in shared memory"
3466
3287
  else:
3467
- glmmean = np.zeros(internalvalidspaceshape, dtype=rt_outfloattype)
3288
+ sLFOfitmean = np.zeros(internalvalidspaceshape, dtype=rt_outfloattype)
3468
3289
  rvalue = np.zeros(internalvalidspaceshape, dtype=rt_outfloattype)
3469
3290
  r2value = np.zeros(internalvalidspaceshape, dtype=rt_outfloattype)
3470
3291
  fitNorm = np.zeros(internalvalidspaceshapederivs, dtype=rt_outfloattype)
@@ -3474,8 +3295,8 @@ def rapidtide_main(argparsingfunc):
3474
3295
  filtereddata = np.zeros(internalvalidfmrishape, dtype=rt_outfloattype)
3475
3296
  ramlocation = "locally"
3476
3297
 
3477
- optiondict["totalglmbytes"] = (
3478
- glmmean.nbytes
3298
+ optiondict["totalsLFOfilterbytes"] = (
3299
+ sLFOfitmean.nbytes
3479
3300
  + rvalue.nbytes
3480
3301
  + r2value.nbytes
3481
3302
  + fitNorm.nbytes
@@ -3484,25 +3305,25 @@ def rapidtide_main(argparsingfunc):
3484
3305
  + lagtc.nbytes
3485
3306
  + filtereddata.nbytes
3486
3307
  )
3487
- thesize, theunit = tide_util.format_bytes(optiondict["totalglmbytes"])
3488
- print(f"allocated {thesize:.3f} {theunit} {ramlocation} for glm/delay refinement")
3308
+ thesize, theunit = tide_util.format_bytes(optiondict["totalsLFOfilterbytes"])
3309
+ print(f"allocated {thesize:.3f} {theunit} {ramlocation} for sLFO filter/delay refinement")
3489
3310
 
3490
3311
  if optiondict["memprofile"]:
3491
- if optiondict["doglmfilt"]:
3492
- memcheckpoint("about to start glm noise removal...")
3312
+ if optiondict["dolinfitfilt"]:
3313
+ memcheckpoint("about to start sLFO noise removal...")
3493
3314
  else:
3494
3315
  memcheckpoint("about to start CVR magnitude estimation...")
3495
- tide_util.logmem("before glm")
3316
+ tide_util.logmem("before sLFO filter")
3496
3317
 
3497
- if optiondict["doglmfilt"]:
3318
+ if optiondict["dolinfitfilt"]:
3498
3319
  mode = "glm"
3499
- optiondict["glmthreshval"] = threshval
3320
+ optiondict["regressfiltthreshval"] = threshval
3500
3321
  else:
3501
3322
  # set the threshval to zero
3502
3323
  mode = "cvrmap"
3503
- optiondict["glmthreshval"] = 0.0
3324
+ optiondict["regressfiltthreshval"] = 0.0
3504
3325
  if optiondict["debug"]:
3505
- # dump the fmri input file going to glm
3326
+ # dump the fmri input file going to sLFO filter
3506
3327
  if not optiondict["textio"]:
3507
3328
  theheader = copy.deepcopy(nim_hdr)
3508
3329
  if fileiscifti:
@@ -3523,7 +3344,7 @@ def rapidtide_main(argparsingfunc):
3523
3344
  "datatofilter",
3524
3345
  "bold",
3525
3346
  None,
3526
- "fMRI data that will be subjected to GLM filtering",
3347
+ "fMRI data that will be subjected to sLFO filtering",
3527
3348
  ),
3528
3349
  ]
3529
3350
  tide_io.savemaplist(
@@ -3539,7 +3360,7 @@ def rapidtide_main(argparsingfunc):
3539
3360
  cifti_hdr=cifti_hdr,
3540
3361
  )
3541
3362
 
3542
- # refine the delay value prior to calculating the GLM
3363
+ # refine the delay value prior to calculating the sLFO filter
3543
3364
  if optiondict["refinedelay"]:
3544
3365
  TimingLGR.info("Delay refinement start")
3545
3366
  LGR.info("\n\nDelay refinement")
@@ -3547,7 +3368,7 @@ def rapidtide_main(argparsingfunc):
3547
3368
  # set gausssigma automatically
3548
3369
  optiondict["delayoffsetgausssigma"] = np.mean([xdim, ydim, slicethickness]) / 2.0
3549
3370
 
3550
- glmderivratios = tide_refinedelay.getderivratios(
3371
+ regressderivratios, regressrvalues = tide_refinedelay.getderivratios(
3551
3372
  fmri_data_valid,
3552
3373
  validvoxels,
3553
3374
  initial_fmri_x,
@@ -3557,62 +3378,102 @@ def rapidtide_main(argparsingfunc):
3557
3378
  mode,
3558
3379
  outputname,
3559
3380
  oversamptr,
3560
- glmmean,
3381
+ sLFOfitmean,
3561
3382
  rvalue,
3562
3383
  r2value,
3563
- fitNorm[:, :2],
3564
- fitcoeff[:, :2],
3384
+ fitNorm[:, : (optiondict["refineregressderivs"] + 1)],
3385
+ fitcoeff[:, : (optiondict["refineregressderivs"] + 1)],
3565
3386
  movingsignal,
3566
3387
  lagtc,
3567
3388
  filtereddata,
3568
3389
  LGR,
3569
3390
  TimingLGR,
3570
3391
  optiondict,
3392
+ regressderivs=optiondict["refineregressderivs"],
3571
3393
  debug=optiondict["debug"],
3572
3394
  )
3573
3395
 
3574
- medfiltglmderivratios, filteredglmderivratios, delayoffsetMAD = (
3575
- tide_refinedelay.filterderivratios(
3576
- glmderivratios,
3577
- nativespaceshape,
3578
- validvoxels,
3579
- (xdim, ydim, slicethickness),
3580
- gausssigma=optiondict["delayoffsetgausssigma"],
3581
- patchthresh=optiondict["delaypatchthresh"],
3582
- fileiscifti=fileiscifti,
3583
- textio=optiondict["textio"],
3584
- rt_floattype="float64",
3396
+ if optiondict["refineregressderivs"] == 1:
3397
+ medfiltregressderivratios, filteredregressderivratios, delayoffsetMAD = (
3398
+ tide_refinedelay.filterderivratios(
3399
+ regressderivratios,
3400
+ nativespaceshape,
3401
+ validvoxels,
3402
+ (xdim, ydim, slicethickness),
3403
+ gausssigma=optiondict["delayoffsetgausssigma"],
3404
+ patchthresh=optiondict["delaypatchthresh"],
3405
+ fileiscifti=fileiscifti,
3406
+ textio=optiondict["textio"],
3407
+ rt_floattype="float64",
3408
+ debug=optiondict["debug"],
3409
+ )
3410
+ )
3411
+ optiondict["delayoffsetMAD"] = delayoffsetMAD
3412
+
3413
+ # find the mapping of derivative ratios to delays
3414
+ tide_refinedelay.trainratiotooffset(
3415
+ genlagtc,
3416
+ initial_fmri_x,
3417
+ outputname,
3418
+ optiondict["outputlevel"],
3419
+ mindelay=optiondict["mindelay"],
3420
+ maxdelay=optiondict["maxdelay"],
3421
+ numpoints=optiondict["numpoints"],
3585
3422
  debug=optiondict["debug"],
3586
3423
  )
3587
- )
3588
- optiondict["delayoffsetMAD"] = delayoffsetMAD
3589
3424
 
3590
- # find the mapping of glm ratios to delays
3591
- tide_refinedelay.trainratiotooffset(
3592
- genlagtc,
3593
- initial_fmri_x,
3594
- outputname,
3595
- optiondict["outputlevel"],
3596
- mindelay=optiondict["mindelay"],
3597
- maxdelay=optiondict["maxdelay"],
3598
- numpoints=optiondict["numpoints"],
3599
- debug=optiondict["debug"],
3600
- )
3425
+ # now calculate the delay offsets
3426
+ delayoffset = np.zeros_like(filteredregressderivratios)
3427
+ if optiondict["debug"]:
3428
+ print(
3429
+ f"calculating delayoffsets for {filteredregressderivratios.shape[0]} voxels"
3430
+ )
3431
+ for i in range(filteredregressderivratios.shape[0]):
3432
+ delayoffset[i] = tide_refinedelay.ratiotodelay(filteredregressderivratios[i])
3433
+ else:
3434
+ medfiltregressderivratios = np.zeros_like(regressderivratios)
3435
+ filteredregressderivratios = np.zeros_like(regressderivratios)
3436
+ delayoffsetMAD = np.zeros(optiondict["refineregressderivs"], dtype=float)
3437
+ for i in range(optiondict["refineregressderivs"]):
3438
+ (
3439
+ medfiltregressderivratios[i, :],
3440
+ filteredregressderivratios[i, :],
3441
+ delayoffsetMAD[i],
3442
+ ) = tide_refinedelay.filterderivratios(
3443
+ regressderivratios[i, :],
3444
+ (xsize, ysize, numslices),
3445
+ validvoxels,
3446
+ (xdim, ydim, slicethickness),
3447
+ gausssigma=optiondict["delayoffsetgausssigma"],
3448
+ patchthresh=optiondict["delaypatchthresh"],
3449
+ fileiscifti=False,
3450
+ textio=False,
3451
+ rt_floattype=rt_floattype,
3452
+ debug=optiondict["debug"],
3453
+ )
3454
+ optiondict[f"delayoffsetMAD_{i + 1}"] = delayoffsetMAD[i]
3455
+
3456
+ # now calculate the delay offsets
3457
+ delayoffset = np.zeros_like(filteredregressderivratios[0, :])
3458
+ if optiondict["debug"]:
3459
+ print(
3460
+ f"calculating delayoffsets for {filteredregressderivratios.shape[1]} voxels"
3461
+ )
3462
+ for i in range(filteredregressderivratios.shape[1]):
3463
+ delayoffset[i] = tide_refinedelay.coffstodelay(
3464
+ filteredregressderivratios[:, i],
3465
+ mindelay=optiondict["mindelay"],
3466
+ maxdelay=optiondict["maxdelay"],
3467
+ )
3601
3468
 
3602
- # now calculate the delay offsets
3603
- delayoffset = filteredglmderivratios * 0.0
3604
- if optiondict["debug"]:
3605
- print(f"calculating delayoffsets for {filteredglmderivratios.shape[0]} voxels")
3606
- for i in range(filteredglmderivratios.shape[0]):
3607
- delayoffset[i] = tide_refinedelay.ratiotodelay(filteredglmderivratios[i])
3608
3469
  namesuffix = "_desc-delayoffset_hist"
3609
- if optiondict["doglmfilt"]:
3470
+ if optiondict["dolinfitfilt"]:
3610
3471
  tide_stats.makeandsavehistogram(
3611
3472
  delayoffset[np.where(fitmask > 0)],
3612
3473
  optiondict["histlen"],
3613
3474
  1,
3614
3475
  outputname + namesuffix,
3615
- displaytitle="Histogram of delay offsets calculated from GLM",
3476
+ displaytitle="Histogram of delay offsets calculated from regression coefficients",
3616
3477
  dictvarname="delayoffsethist",
3617
3478
  thedict=optiondict,
3618
3479
  )
@@ -3622,11 +3483,11 @@ def rapidtide_main(argparsingfunc):
3622
3483
  # Delay refinement end
3623
3484
  ####################################################
3624
3485
 
3625
- # now calculate the GLM or CVR map
3626
- if optiondict["doglmfilt"] or optiondict["docvrmap"]:
3627
- if optiondict["doglmfilt"]:
3628
- TimingLGR.info("GLM filtering start")
3629
- LGR.info("\n\nGLM filtering")
3486
+ # now calculate the sLFO filter or CVR map
3487
+ if optiondict["dolinfitfilt"] or optiondict["docvrmap"]:
3488
+ if optiondict["dolinfitfilt"]:
3489
+ TimingLGR.info("sLFO filtering start")
3490
+ LGR.info("\n\nsLFO filtering")
3630
3491
  else:
3631
3492
  TimingLGR.info("CVR map generation")
3632
3493
  LGR.info("\n\nCVR mapping")
@@ -3635,46 +3496,48 @@ def rapidtide_main(argparsingfunc):
3635
3496
  # initialrawvariance = tide_math.imagevariance(fmri_data_valid, None, 1.0 / fmritr)
3636
3497
  initialvariance = tide_math.imagevariance(fmri_data_valid, theprefilter, 1.0 / fmritr)
3637
3498
 
3638
- # now calculate the GLM
3499
+ # now calculate the sLFO filter
3639
3500
  if optiondict["refinedelay"] and optiondict["filterwithrefineddelay"]:
3640
3501
  lagstouse = lagtimesrefined
3641
3502
  else:
3642
3503
  lagstouse = lagtimes
3643
- voxelsprocessed_glm, regressorset, evset = tide_glmfrommaps.glmfrommaps(
3644
- fmri_data_valid,
3645
- validvoxels,
3646
- initial_fmri_x,
3647
- lagstouse,
3648
- fitmask,
3649
- genlagtc,
3650
- mode,
3651
- outputname,
3652
- oversamptr,
3653
- glmmean,
3654
- rvalue,
3655
- r2value,
3656
- fitNorm[:, : optiondict["glmderivs"] + 1],
3657
- fitcoeff[:, : optiondict["glmderivs"] + 1],
3658
- movingsignal,
3659
- lagtc,
3660
- filtereddata,
3661
- LGR,
3662
- TimingLGR,
3663
- optiondict["glmthreshval"],
3664
- optiondict["saveminimumglmfiles"],
3665
- nprocs_makelaggedtcs=optiondict["nprocs_makelaggedtcs"],
3666
- nprocs_glm=optiondict["nprocs_glm"],
3667
- glmderivs=optiondict["glmderivs"],
3668
- mp_chunksize=optiondict["mp_chunksize"],
3669
- showprogressbar=optiondict["showprogressbar"],
3670
- alwaysmultiproc=optiondict["alwaysmultiproc"],
3671
- memprofile=optiondict["memprofile"],
3672
- debug=optiondict["debug"],
3504
+ voxelsprocessed_regressionfilt, regressorset, evset = (
3505
+ tide_regressfrommaps.regressfrommaps(
3506
+ fmri_data_valid,
3507
+ validvoxels,
3508
+ initial_fmri_x,
3509
+ lagstouse,
3510
+ fitmask,
3511
+ genlagtc,
3512
+ mode,
3513
+ outputname,
3514
+ oversamptr,
3515
+ sLFOfitmean,
3516
+ rvalue,
3517
+ r2value,
3518
+ fitNorm[:, : optiondict["regressderivs"] + 1],
3519
+ fitcoeff[:, : optiondict["regressderivs"] + 1],
3520
+ movingsignal,
3521
+ lagtc,
3522
+ filtereddata,
3523
+ LGR,
3524
+ TimingLGR,
3525
+ optiondict["regressfiltthreshval"],
3526
+ optiondict["saveminimumsLFOfiltfiles"],
3527
+ nprocs_makelaggedtcs=optiondict["nprocs_makelaggedtcs"],
3528
+ nprocs_regressionfilt=optiondict["nprocs_regressionfilt"],
3529
+ regressderivs=optiondict["regressderivs"],
3530
+ mp_chunksize=optiondict["mp_chunksize"],
3531
+ showprogressbar=optiondict["showprogressbar"],
3532
+ alwaysmultiproc=optiondict["alwaysmultiproc"],
3533
+ memprofile=optiondict["memprofile"],
3534
+ debug=optiondict["debug"],
3535
+ )
3673
3536
  )
3674
3537
 
3675
3538
  evcolnames = ["base"]
3676
- if optiondict["glmderivs"] > 0:
3677
- for i in range(1, optiondict["glmderivs"] + 1):
3539
+ if optiondict["regressderivs"] > 0:
3540
+ for i in range(1, optiondict["regressderivs"] + 1):
3678
3541
  evcolnames.append(f"deriv_{str(i)}")
3679
3542
 
3680
3543
  tide_io.writebidstsv(
@@ -3682,7 +3545,7 @@ def rapidtide_main(argparsingfunc):
3682
3545
  np.transpose(evset),
3683
3546
  1.0 / fmritr,
3684
3547
  columns=evcolnames,
3685
- extraheaderinfo={"Description": "GLM regressor set"},
3548
+ extraheaderinfo={"Description": "sLFO filter regressor set"},
3686
3549
  append=False,
3687
3550
  )
3688
3551
 
@@ -3694,30 +3557,20 @@ def rapidtide_main(argparsingfunc):
3694
3557
  varchange = initialvariance * 0.0
3695
3558
  varchange[divlocs] = 100.0 * (finalvariance[divlocs] / initialvariance[divlocs] - 1.0)
3696
3559
 
3697
- """divlocs = np.where(finalrawvariance > 0.0)
3698
- rawvarchange = initialrawvariance * 0.0
3699
- rawvarchange[divlocs] = 100.0 * (
3700
- finalrawvariance[divlocs] / initialvariance[divlocs] - 1.0
3701
- )"""
3702
-
3703
- del fmri_data_valid
3704
- if optiondict["sharedmem"]:
3705
- tide_util.cleanup_shm(fmri_data_valid_shm)
3706
-
3707
3560
  LGR.info("End filtering operation")
3708
3561
  TimingLGR.info(
3709
- "GLM filtering end",
3562
+ "sLFO filtering end",
3710
3563
  {
3711
- "message2": voxelsprocessed_glm,
3564
+ "message2": voxelsprocessed_regressionfilt,
3712
3565
  "message3": "voxels",
3713
3566
  },
3714
3567
  )
3715
3568
  if optiondict["memprofile"]:
3716
3569
  memcheckpoint("...done")
3717
- tide_util.logmem("after glm filter")
3570
+ tide_util.logmem("after sLFO filter")
3718
3571
  LGR.info("")
3719
3572
  ####################################################
3720
- # GLM filtering end
3573
+ # sLFO filtering end
3721
3574
  ####################################################
3722
3575
 
3723
3576
  # Post refinement step 2 - make and save interesting histograms
@@ -3755,24 +3608,24 @@ def rapidtide_main(argparsingfunc):
3755
3608
  thedict=optiondict,
3756
3609
  )
3757
3610
  namesuffix = "_desc-lfofilterR2_hist"
3758
- if optiondict["doglmfilt"]:
3611
+ if optiondict["dolinfitfilt"]:
3759
3612
  tide_stats.makeandsavehistogram(
3760
3613
  r2value[np.where(fitmask > 0)],
3761
3614
  optiondict["histlen"],
3762
3615
  1,
3763
3616
  outputname + namesuffix,
3764
- displaytitle="Histogram of GLM filter R2 values",
3617
+ displaytitle="Histogram of sLFO filter R2 values",
3765
3618
  dictvarname="R2hist",
3766
3619
  thedict=optiondict,
3767
3620
  )
3768
3621
  namesuffix = "_desc-lfofilterInbandVarianceChange_hist"
3769
- if optiondict["doglmfilt"]:
3622
+ if optiondict["dolinfitfilt"]:
3770
3623
  tide_stats.makeandsavehistogram(
3771
3624
  varchange[np.where(fitmask > 0)],
3772
3625
  optiondict["histlen"],
3773
3626
  1,
3774
3627
  outputname + namesuffix,
3775
- displaytitle="Histogram of percent of inband variance removed by GLM filter",
3628
+ displaytitle="Histogram of percent of inband variance removed by sLFO filter",
3776
3629
  dictvarname="varchangehist",
3777
3630
  thedict=optiondict,
3778
3631
  )
@@ -3861,32 +3714,57 @@ def rapidtide_main(argparsingfunc):
3861
3714
  "second",
3862
3715
  "Lag time in seconds, refined",
3863
3716
  ),
3864
- (np.fabs(rvalue), "maxcorralt", "map", None, "R value of the GLM fit, with sign"),
3865
3717
  ]
3866
3718
  if (optiondict["outputlevel"] != "min") and (optiondict["outputlevel"] != "less"):
3867
- savelist += [
3868
- (
3869
- glmderivratios,
3870
- "glmderivratios",
3871
- "map",
3872
- None,
3873
- "Ratio of the first derivative of delayed sLFO to the delayed sLFO",
3874
- ),
3875
- (
3876
- medfiltglmderivratios,
3877
- "medfiltglmderivratios",
3878
- "map",
3879
- None,
3880
- "Median filtered version of the glmderivratios map",
3881
- ),
3882
- (
3883
- filteredglmderivratios,
3884
- "filteredglmderivratios",
3885
- "map",
3886
- None,
3887
- "glmderivratios, with outliers patched using median filtered data",
3888
- ),
3889
- ]
3719
+ if optiondict["refineregressderivs"] > 1:
3720
+ for i in range(optiondict["refineregressderivs"]):
3721
+ savelist += [
3722
+ (
3723
+ regressderivratios[i, :],
3724
+ f"regressderivratios_{i}",
3725
+ "map",
3726
+ None,
3727
+ f"Ratio of derivative {i + 1} of delayed sLFO to the delayed sLFO",
3728
+ ),
3729
+ (
3730
+ medfiltregressderivratios[i, :],
3731
+ f"medfiltregressderivratios_{i}",
3732
+ "map",
3733
+ None,
3734
+ f"Median filtered version of the regressderivratios_{i} map",
3735
+ ),
3736
+ (
3737
+ filteredregressderivratios[i, :],
3738
+ f"filteredregressderivratios_{i}",
3739
+ "map",
3740
+ None,
3741
+ f"regressderivratios_{i}, with outliers patched using median filtered data",
3742
+ ),
3743
+ ]
3744
+ else:
3745
+ savelist += [
3746
+ (
3747
+ regressderivratios,
3748
+ "regressderivratios",
3749
+ "map",
3750
+ None,
3751
+ "Ratio of the first derivative of delayed sLFO to the delayed sLFO",
3752
+ ),
3753
+ (
3754
+ medfiltregressderivratios,
3755
+ "medfiltregressderivratios",
3756
+ "map",
3757
+ None,
3758
+ "Median filtered version of the regressderivratios map",
3759
+ ),
3760
+ (
3761
+ filteredregressderivratios,
3762
+ "filteredregressderivratios",
3763
+ "map",
3764
+ None,
3765
+ "regressderivratios, with outliers patched using median filtered data",
3766
+ ),
3767
+ ]
3890
3768
  if optiondict["calccoherence"]:
3891
3769
  savelist += [
3892
3770
  (coherencepeakval, "coherencepeakval", "map", None, "Coherence peak value"),
@@ -3924,8 +3802,8 @@ def rapidtide_main(argparsingfunc):
3924
3802
  tide_util.cleanup_shm(coherencepeakfreq_shm)
3925
3803
 
3926
3804
  # write the optional 3D maps that need to be remapped
3927
- if optiondict["doglmfilt"] or optiondict["docvrmap"]:
3928
- if optiondict["doglmfilt"]:
3805
+ if optiondict["dolinfitfilt"] or optiondict["docvrmap"]:
3806
+ if optiondict["dolinfitfilt"]:
3929
3807
  maplist = [
3930
3808
  (
3931
3809
  initialvariance,
@@ -3949,22 +3827,22 @@ def rapidtide_main(argparsingfunc):
3949
3827
  "Change in inband variance after filtering, in percent",
3950
3828
  ),
3951
3829
  ]
3952
- if optiondict["saveminimumglmfiles"]:
3830
+ if optiondict["saveminimumsLFOfiltfiles"]:
3953
3831
  maplist += [
3954
3832
  (
3955
3833
  r2value,
3956
3834
  "lfofilterR2",
3957
3835
  "map",
3958
3836
  None,
3959
- "Squared R value of the GLM fit (proportion of variance explained)",
3837
+ "Squared R value of the sLFO fit (proportion of variance explained)",
3960
3838
  ),
3961
3839
  ]
3962
- if optiondict["savenormalglmfiles"]:
3840
+ if optiondict["savenormalsLFOfiltfiles"]:
3963
3841
  maplist += [
3964
- (np.fabs(rvalue), "lfofilterR", "map", None, "R value of the GLM fit"),
3965
- (glmmean, "lfofilterMean", "map", None, "Intercept from GLM fit"),
3842
+ (rvalue, "lfofilterR", "map", None, "R value of the sLFO fit"),
3843
+ (sLFOfitmean, "lfofilterMean", "map", None, "Intercept from sLFO fit"),
3966
3844
  ]
3967
- if optiondict["glmderivs"] > 0:
3845
+ if optiondict["regressderivs"] > 0:
3968
3846
  maplist += [
3969
3847
  (fitcoeff[:, 0], "lfofilterCoeff", "map", None, "Fit coefficient"),
3970
3848
  (
@@ -3975,7 +3853,7 @@ def rapidtide_main(argparsingfunc):
3975
3853
  "Normalized fit coefficient",
3976
3854
  ),
3977
3855
  ]
3978
- for thederiv in range(1, optiondict["glmderivs"] + 1):
3856
+ for thederiv in range(1, optiondict["regressderivs"] + 1):
3979
3857
  maplist += [
3980
3858
  (
3981
3859
  fitcoeff[:, thederiv],
@@ -3995,14 +3873,14 @@ def rapidtide_main(argparsingfunc):
3995
3873
  else:
3996
3874
  maplist += [
3997
3875
  (
3998
- fitcoeff[:, : optiondict["glmderivs"] + 1],
3876
+ fitcoeff[:, : optiondict["regressderivs"] + 1],
3999
3877
  "lfofilterCoeff",
4000
3878
  "map",
4001
3879
  None,
4002
3880
  "Fit coefficient",
4003
3881
  ),
4004
3882
  (
4005
- fitNorm[:, : optiondict["glmderivs"] + 1],
3883
+ fitNorm[:, : optiondict["regressderivs"] + 1],
4006
3884
  "lfofilterNorm",
4007
3885
  "map",
4008
3886
  None,
@@ -4033,15 +3911,15 @@ def rapidtide_main(argparsingfunc):
4033
3911
  "Percentage of inband variance attributable to CVR regressor",
4034
3912
  ),
4035
3913
  ]
4036
- if optiondict["savenormalglmfiles"]:
3914
+ if optiondict["savenormalsLFOfiltfiles"]:
4037
3915
  maplist = [
4038
- (rvalue, "CVRR", "map", None, "R value of the GLM fit"),
3916
+ (rvalue, "CVRR", "map", None, "R value of the sLFO fit"),
4039
3917
  (
4040
3918
  r2value,
4041
3919
  "CVRR2",
4042
3920
  "map",
4043
3921
  None,
4044
- "Squared R value of the GLM fit (proportion of variance explained)",
3922
+ "Squared R value of the sLFO fit (proportion of variance explained)",
4045
3923
  ),
4046
3924
  (
4047
3925
  fitcoeff,
@@ -4064,7 +3942,70 @@ def rapidtide_main(argparsingfunc):
4064
3942
  rt_floattype=rt_floattype,
4065
3943
  cifti_hdr=cifti_hdr,
4066
3944
  )
4067
- del glmmean
3945
+
3946
+ if optiondict["refinedelay"] and False:
3947
+ # filter the fmri data to the lfo band
3948
+ print("filtering fmri_data to sLFO band")
3949
+ for i in range(fmri_data_valid.shape[0]):
3950
+ fmri_data_valid[i, :] = theprefilter.apply(
3951
+ optiondict["fmrifreq"], fmri_data_valid[i, :]
3952
+ )
3953
+
3954
+ print("rerunning sLFO fit to get filtered R value")
3955
+ dummy, dummy, dummy = tide_regressfrommaps.regressfrommaps(
3956
+ fmri_data_valid,
3957
+ validvoxels,
3958
+ initial_fmri_x,
3959
+ lagstouse,
3960
+ fitmask,
3961
+ genlagtc,
3962
+ mode,
3963
+ outputname,
3964
+ oversamptr,
3965
+ sLFOfitmean,
3966
+ rvalue,
3967
+ r2value,
3968
+ fitNorm[:, : optiondict["regressderivs"] + 1],
3969
+ fitcoeff[:, : optiondict["regressderivs"] + 1],
3970
+ movingsignal,
3971
+ lagtc,
3972
+ filtereddata,
3973
+ LGR,
3974
+ TimingLGR,
3975
+ optiondict["regressfiltthreshval"],
3976
+ optiondict["saveminimumsLFOfiltfiles"],
3977
+ nprocs_makelaggedtcs=optiondict["nprocs_makelaggedtcs"],
3978
+ nprocs_regressionfilt=optiondict["nprocs_regressionfilt"],
3979
+ regressderivs=optiondict["regressderivs"],
3980
+ mp_chunksize=optiondict["mp_chunksize"],
3981
+ showprogressbar=optiondict["showprogressbar"],
3982
+ alwaysmultiproc=optiondict["alwaysmultiproc"],
3983
+ memprofile=optiondict["memprofile"],
3984
+ debug=optiondict["debug"],
3985
+ )
3986
+
3987
+ maplist = [
3988
+ (rvalue, "maxcorralt", "map", None, "R value of the inband sLFO fit, with sign"),
3989
+ ]
3990
+
3991
+ tide_io.savemaplist(
3992
+ outputname,
3993
+ maplist,
3994
+ validvoxels,
3995
+ nativespaceshape,
3996
+ theheader,
3997
+ bidsbasedict,
3998
+ textio=optiondict["textio"],
3999
+ fileiscifti=fileiscifti,
4000
+ rt_floattype=rt_floattype,
4001
+ cifti_hdr=cifti_hdr,
4002
+ )
4003
+
4004
+ del fmri_data_valid
4005
+ if optiondict["sharedmem"]:
4006
+ tide_util.cleanup_shm(fmri_data_valid_shm)
4007
+
4008
+ del sLFOfitmean
4068
4009
  del rvalue
4069
4010
  del r2value
4070
4011
  del fitcoeff
@@ -4073,7 +4014,7 @@ def rapidtide_main(argparsingfunc):
4073
4014
  del finalvariance
4074
4015
  del varchange
4075
4016
  if optiondict["sharedmem"]:
4076
- tide_util.cleanup_shm(glmmean_shm)
4017
+ tide_util.cleanup_shm(sLFOfitmean_shm)
4077
4018
  tide_util.cleanup_shm(rvalue_shm)
4078
4019
  tide_util.cleanup_shm(r2value_shm)
4079
4020
  tide_util.cleanup_shm(fitcoeff_shm)
@@ -4118,7 +4059,7 @@ def rapidtide_main(argparsingfunc):
4118
4059
  lagstrengths[voxel],
4119
4060
  sigfit,
4120
4061
  neglogpmax=neglogpmax,
4121
- debug=optiondict["focaldebug"],
4062
+ debug=optiondict["debug"],
4122
4063
  )
4123
4064
  masklist += [
4124
4065
  (
@@ -4147,6 +4088,7 @@ def rapidtide_main(argparsingfunc):
4147
4088
  if (optiondict["passes"] > 1 or optiondict["globalpreselect"]) and optiondict[
4148
4089
  "refinestopreason"
4149
4090
  ] != "emptymask":
4091
+ refinemask = theRegressorRefiner.getrefinemask()
4150
4092
  if optiondict["globalpreselect"]:
4151
4093
  masklist = [
4152
4094
  (
@@ -4253,8 +4195,10 @@ def rapidtide_main(argparsingfunc):
4253
4195
  cifti_hdr = None
4254
4196
 
4255
4197
  maplist = []
4256
- if optiondict["saveallglmfiles"] and (optiondict["doglmfilt"] or optiondict["docvrmap"]):
4257
- if optiondict["glmderivs"] > 0:
4198
+ if optiondict["saveallsLFOfiltfiles"] and (
4199
+ optiondict["dolinfitfilt"] or optiondict["docvrmap"]
4200
+ ):
4201
+ if optiondict["regressderivs"] > 0:
4258
4202
  maplist += [
4259
4203
  (
4260
4204
  regressorset[:, :, 0],
@@ -4264,7 +4208,7 @@ def rapidtide_main(argparsingfunc):
4264
4208
  "Shifted sLFO regressor to filter",
4265
4209
  ),
4266
4210
  ]
4267
- for thederiv in range(1, optiondict["glmderivs"] + 1):
4211
+ for thederiv in range(1, optiondict["regressderivs"] + 1):
4268
4212
  maplist += [
4269
4213
  (
4270
4214
  regressorset[:, :, thederiv],
@@ -4279,11 +4223,11 @@ def rapidtide_main(argparsingfunc):
4279
4223
  (regressorset, "lfofilterEV", "bold", None, "Shifted sLFO regressor to filter"),
4280
4224
  ]
4281
4225
 
4282
- if optiondict["passes"] > 1:
4226
+ if (optiondict["passes"] > 1) or optiondict["dofinalrefine"]:
4283
4227
  if optiondict["savelagregressors"]:
4284
4228
  maplist += [
4285
4229
  (
4286
- paddedshiftedtcs[:, numpadtrs:-numpadtrs],
4230
+ (theRegressorRefiner.getpaddedshiftedtcs())[:, numpadtrs:-numpadtrs],
4287
4231
  "shiftedtcs",
4288
4232
  "bold",
4289
4233
  None,
@@ -4291,8 +4235,8 @@ def rapidtide_main(argparsingfunc):
4291
4235
  ),
4292
4236
  ]
4293
4237
 
4294
- if optiondict["doglmfilt"]:
4295
- if optiondict["saveminimumglmfiles"]:
4238
+ if optiondict["dolinfitfilt"]:
4239
+ if optiondict["saveminimumsLFOfiltfiles"]:
4296
4240
  maplist += [
4297
4241
  (
4298
4242
  filtereddata,
@@ -4351,16 +4295,8 @@ def rapidtide_main(argparsingfunc):
4351
4295
 
4352
4296
  # clean up
4353
4297
  if optiondict["passes"] > 1:
4354
- del paddedshiftedtcs
4355
- del paddedweights
4356
- del shiftedtcs
4357
- del weights
4358
- if optiondict["sharedmem"]:
4359
- tide_util.cleanup_shm(paddedshiftedtcs_shm)
4360
- tide_util.cleanup_shm(paddedweights_shm)
4361
- tide_util.cleanup_shm(shiftedtcs_shm)
4362
- tide_util.cleanup_shm(weights_shm)
4363
- if optiondict["doglmfilt"]:
4298
+ theRegressorRefiner.cleanup()
4299
+ if optiondict["dolinfitfilt"]:
4364
4300
  del lagtc
4365
4301
  del filtereddata
4366
4302
  del movingsignal