antspymm 0.9.7__py3-none-any.whl → 1.0.0__py3-none-any.whl

antspymm/__init__.py

@@ -5,6 +5,10 @@ except:
     pass
 
 from .mm import get_data
+from .mm import get_dti
+from .mm import triangular_to_tensor
+from .mm import dti_numpy_to_image
+from .mm import transform_and_reorient_dti
 from .mm import get_models
 from .mm import nrg_format_path
 from .mm import highest_quality_repeat
@@ -84,3 +88,5 @@ from .mm import novelty_detection_quantile
 from .mm import generate_mm_dataframe
 from .mm import collect_blind_qc_by_modality
 from .mm import get_valid_modalities
+from .mm import study_dataframe_from_matched_dataframe
+from .mm import merge_wides_to_study_dataframe

antspymm/mm.py

@@ -13,6 +13,7 @@ __all__ = ['version',
     'merge_dwi_data',
     'outlierness_by_modality',
     'bvec_reorientation',
+    'get_dti',
     'dti_reg',
     'mc_reg',
     'get_data',
@@ -78,6 +79,8 @@ __all__ = ['version',
     'novelty_detection_loop',
     'novelty_detection_quantile',
     'generate_mm_dataframe',
+    'study_dataframe_from_matched_dataframe',
+    'merge_wides_to_study_dataframe',
     'wmh']
 
 from pathlib import Path
@@ -419,7 +422,7 @@ def collect_blind_qc_by_modality( modality_path, set_index_to_fn=True ):
 
 def outlierness_by_modality( qcdf, uid='fn', outlier_columns = ['noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi','reflection_err', 'EVR', 'msk_vol'], verbose=False ):
     """
-    Calculates outlierness scores for each modality in a dataframe based on given outlier columns using antspyt1w.loop_outlierness() and LOF.  LOF appears to be more conservative.
+    Calculates outlierness scores for each modality in a dataframe based on given outlier columns using antspyt1w.loop_outlierness() and LOF.  LOF appears to be more conservative.  This function will impute missing columns with the mean.
 
     Args:
     - qcdf: (Pandas DataFrame) Dataframe containing columns with outlier information for each modality.
@@ -450,6 +453,9 @@ def outlierness_by_modality( qcdf, uid='fn', outlier_columns = ['noise', 'snr',
         lof = LocalOutlierFactor()
         locsel = qcdfout["modality"] == mod
         rr = qcdfout[locsel][outlier_columns]
+        with pd.option_context('mode.use_inf_as_na', True):
+            for myolcol in outlier_columns:
+                rr[myolcol].fillna(rr[myolcol].mean(), inplace=True)
         if rr.shape[0] > 1:
             if verbose:
                 print(mod)
@@ -494,6 +500,98 @@ def nrg_format_path( projectID, subjectID, date, modality, imageID, separator='-
     thefilename = str(projectID) + separator + str(subjectID) + separator + str(date) + separator + str(modality) + separator + str(imageID)
     return os.path.join( thedirectory, thefilename )
 
+
+def study_dataframe_from_matched_dataframe( matched_dataframe, rootdir, outputdir, verbose=False ):
+    """
+    converts the output of antspymm.match_modalities dataframe (one row) to that needed for a study-driving dataframe for input to mm_csv
+
+    matched_dataframe : output of antspymm.match_modalities
+
+    rootdir : location for the input data root folder (in e.g. NRG format)
+
+    outputdir : location for the output data
+
+    verbose : boolean
+    """
+    iext='.nii.gz'
+    from os.path import exists
+    musthavecols = ['projectID', 'subjectID','date','imageID','fn']
+    for k in range(len(musthavecols)):
+        if not musthavecols[k] in matched_dataframe.keys():
+            raise ValueError('matched_dataframe is missing column ' +musthavecols[k] + ' in study_dataframe_from_qc_dataframe' )
+    csvrow=matched_dataframe.dropna(axis=1)
+    pid=str(csvrow['projectID'].iloc[0] )
+    sid=str(csvrow['subjectID'].iloc[0] )
+    dt=str(csvrow['date'].iloc[0])
+    iid=str(csvrow['imageID'].iloc[0])
+    nrgt1fn=os.path.join( rootdir, pid, sid, dt, 'T1w', iid, str(csvrow['fn'].iloc[0]+iext) )
+    if not exists( nrgt1fn ):
+        raise ValueError("T1 " + nrgt1fn + " does not exist in study_dataframe_from_qc_dataframe")
+    flList=[]
+    dtList=[]
+    rsfList=[]
+    nmList=[]
+    if 'flairfn' in csvrow.keys():
+        flid=str(int(csvrow['flairid'].iloc[0]))
+        nrgt2fn=os.path.join( rootdir, pid, sid, dt, 'T2Flair', flid, str(csvrow['flairfn'].iloc[0]+iext) )
+        if exists( nrgt2fn ):
+            flList.append( nrgt2fn )
+    if 'dtfn1' in csvrow.keys():
+        dtid=str(int(csvrow['dtid1'].iloc[0]))
+        dtfn1=glob.glob(os.path.join( rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn1'].iloc[0]+iext) ))[0]
+        if exists( dtfn1 ):
+            dtList.append( dtfn1 )
+    if 'dtfn2' in csvrow.keys():
+        dtid=str(int(csvrow['dtid2'].iloc[0]))
+        dtfn2=glob.glob(os.path.join(rootdir, pid, sid, dt, 'DTI*', dtid, str(csvrow['dtfn2'].iloc[0]+iext) ))[0]
+        if exists( dtfn2 ):
+            dtList.append( dtfn2 )
+    if 'rsffn1' in csvrow.keys():
+        rsid=str(int(csvrow['rsfid1'].iloc[0]))
+        rsfn1=glob.glob(os.path.join( rootdir, pid, sid, dt, 'rsfMRI*', rsid, str(csvrow['rsffn1'].iloc[0]+iext) ))[0]
+        if exists( rsfn1 ):
+            rsfList.append( rsfn1 )
+    if 'rsffn2' in csvrow.keys():
+        rsid=str(int(csvrow['rsfid2'].iloc[0]))
+        rsfn2=glob.glob(os.path.join( rootdir, pid, sid, dt, 'rsfMRI*', rsid, str(csvrow['rsffn2'].iloc[0]+iext) ))[0]
+        if exists( rsfn2 ):
+            rsfList.append( rsfn2 )
+    for j in range(11):
+        keyname="nmfn"+str(j)
+        keynameid="nmid"+str(j)
+        if keyname in csvrow.keys() and keynameid in csvrow.keys():
+            nmid=str(int(csvrow[keynameid].iloc[0]))
+            nmsearchpath=os.path.join( rootdir, pid, sid, dt, 'NM2DMT', nmid, "*"+nmid+iext)
+            nmfn=glob.glob( nmsearchpath )
+            nmfn=nmfn[0]
+            if exists( nmfn ):
+                nmList.append( nmfn )
+    if verbose:
+        print("assembled the image lists mapping to ....")
+        print(nrgt1fn)
+        print("NM")
+        print(nmList)
+        print("FLAIR")
+        print(flList)
+        print("DTI")
+        print(dtList)
+        print("rsfMRI")
+        print(rsfList)
+    studycsv = generate_mm_dataframe(
+        pid,
+        sid,
+        dt,
+        iid, # the T1 id
+        'T1w',
+        rootdir,
+        outputdir,
+        t1_filename=nrgt1fn,
+        flair_filename=flList,
+        dti_filenames=dtList,
+        rsf_filenames=rsfList,
+        nm_filenames=nmList)
+    return studycsv.dropna(axis=1)
+
 def highest_quality_repeat(mxdfin, idvar, visitvar, qualityvar):
     """
     This function returns a subset of the input dataframe that retains only the rows
@@ -652,6 +750,16 @@ def match_modalities( qc_dataframe, unique_identifier='fn', outlier_column='ol_l
                 mmdf.iloc[k, mmdf.columns.get_loc("flairfn")] = fldf['fn'][locsel].values[0]
                 mmdf.iloc[k, mmdf.columns.get_loc("flairloop")] = fldf[outlier_column][locsel].values[0]
                 mmdf.iloc[k, mmdf.columns.get_loc("flairlof")] = fldf['ol_lof_decision'][locsel].values[0]
+            elif sum(locsel) > 1:
+                locdf = fldf[locsel]
+                dedupe = locdf[["snr","cnr"]].duplicated()
+                locdf = locdf[~dedupe]
+                if locdf.shape[0] > 1:
+                    locdf = locdf.sort_values(outlier_column).iloc[:2]
+                mmdf.iloc[k, mmdf.columns.get_loc("flairid")] = locdf["imageID"].values[0]
+                mmdf.iloc[k, mmdf.columns.get_loc("flairfn")] = locdf["fn"].values[0]
+                mmdf.iloc[k, mmdf.columns.get_loc("flairloop")] = locdf[outlier_column].values[0]
+                mmdf.iloc[k, mmdf.columns.get_loc("flairlof")] = locdf['ol_lof_decision'].values[0]
 
         if nmdf is not None:
             locsel = nmdf['subjectIDdate'] == mmdf['subjectIDdate'].iloc[k]
@@ -1088,7 +1196,7 @@ def merge_dwi_data( img_LRdwp, bval_LR, bvec_LR, img_RLdwp, bval_RL, bvec_RL ):
     img_LRdwp = ants.list_to_ndimage( img_LRdwp, mimg )
     return img_LRdwp, bval_LR, bvec_LR
 
-def bvec_reorientation( motion_parameters, bvecs ):
+def bvec_reorientation( motion_parameters, bvecs, rebase=None ):
     if motion_parameters is None:
         return bvecs
     n = len(motion_parameters)
@@ -1129,8 +1237,207 @@ def bvec_reorientation( motion_parameters, bvecs ):
                     txparam = ants.get_ants_transform_parameters(txparam)[0:9].reshape( [3,3])
                     Rinv = inv( txparam )
                 bvecs[myidx,:] = np.dot( Rinv, bvecs[myidx,:] )
+                if rebase is not None:
+                    # FIXME - should combine these operations
+                    bvecs[myidx,:] = np.dot( rebase, bvecs[myidx,:] )
     return bvecs
 
+def get_dti( reference_image, tensormodel, upper_triangular=True, return_image=False ):
+    """
+    extract DTI data from a dipy tensormodel
+
+    reference_image : antsImage defining physical space (3D)
+
+    tensormodel : from dipy e.g. the variable myoutx['dtrecon_LR_dewarp']['tensormodel'] if myoutx is produced my joint_dti_recon
+
+    upper_triangular: boolean otherwise use lower triangular coding
+
+    return_image : boolean return the ANTsImage form of DTI otherwise return an array
+
+    Returns
+    -------
+    either an ANTsImage (dim=X.Y.Z with 6 component voxels, upper triangular form)
+        or a 5D NumPy array (dim=X.Y.Z.3.3)
+
+    Notes
+    -----
+    DiPy returns lower triangular form but ANTs expects upper triangular.
+        Here, we default to the ANTs standard but could generalize in the future 
+        because not much here depends on ANTs standards of tensor data.
+        ANTs xx,xy,xz,yy,yz,zz
+        DiPy Dxx, Dxy, Dyy, Dxz, Dyz, Dzz
+
+    """
+    # make the DTI - see 
+    # https://dipy.org/documentation/1.7.0/examples_built/07_reconstruction/reconst_dti/#sphx-glr-examples-built-07-reconstruction-reconst-dti-py
+    # By default, in DIPY, values are ordered as (Dxx, Dxy, Dyy, Dxz, Dyz, Dzz)
+    # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
+    reoind = np.array([0,1,3,2,4,5]) # arrays are faster than lists
+    import dipy.reconst.dti as dti
+    dtiut = dti.lower_triangular(tensormodel.quadratic_form)
+    it = np.ndindex( reference_image.shape )
+    yyind=2
+    xzind=3
+    if upper_triangular:
+        yyind=3
+        xzind=2
+        for i in it: # convert to upper triangular
+            dtiut[i] = dtiut[i][ reoind ] # do we care if this is doing extra work?
+    if return_image:
+        dtiAnts = ants.from_numpy(dtiut,has_components=True)
+        ants.copy_image_info( reference_image, dtiAnts )
+        return dtiAnts
+    # copy these data into a tensor 
+    dtinp = np.zeros(reference_image.shape + (3,3), dtype=float)  
+    dtix = np.zeros((3,3), dtype=float)  
+    it = np.ndindex( reference_image.shape )
+    for i in it:
+        dtivec = dtiut[i] # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
+        dtix[0,0]=dtivec[0]
+        dtix[1,1]=dtivec[yyind] # 2 for LT
+        dtix[2,2]=dtivec[5] 
+        dtix[0,1]=dtix[1,0]=dtivec[1]
+        dtix[0,2]=dtix[2,0]=dtivec[xzind] # 3 for LT
+        dtix[1,2]=dtix[2,1]=dtivec[4]
+        dtinp[i]=dtix
+    return dtinp
+
+def triangular_to_tensor( image, upper_triangular=True ):
+    """
+    convert triangular tensor image to a full tensor form (in numpy)
+
+    image : antsImage holding dti in either upper or lower triangular format 
+
+    upper_triangular: boolean
+
+    Note
+    --------
+    see get_dti function for more details
+    """
+    reoind = np.array([0,1,3,2,4,5]) # arrays are faster than lists
+    it = np.ndindex( image.shape )
+    yyind=2
+    xzind=3
+    if upper_triangular:
+        yyind=3
+        xzind=2
+    # copy these data into a tensor 
+    dtinp = np.zeros(image.shape + (3,3), dtype=float)
+    dtix = np.zeros((3,3), dtype=float)
+    it = np.ndindex( image.shape )
+    dtiut = image.numpy()
+    for i in it:
+        dtivec = dtiut[i] # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
+        dtix[0,0]=dtivec[0]
+        dtix[1,1]=dtivec[yyind] # 2 for LT
+        dtix[2,2]=dtivec[5] 
+        dtix[0,1]=dtix[1,0]=dtivec[1]
+        dtix[0,2]=dtix[2,0]=dtivec[xzind] # 3 for LT
+        dtix[1,2]=dtix[2,1]=dtivec[4]
+        dtinp[i]=dtix
+    return dtinp
+
+
+def dti_numpy_to_image( reference_image, tensorarray, upper_triangular=True):
+    """
+    convert numpy DTI data to antsImage
+
+    reference_image : antsImage defining physical space (3D)
+
+    tensorarray : numpy array X,Y,Z,3,3 shape
+
+    upper_triangular: boolean otherwise use lower triangular coding
+
+    Returns
+    -------
+    ANTsImage
+
+    Notes
+    -----
+    DiPy returns lower triangular form but ANTs expects upper triangular.
+        Here, we default to the ANTs standard but could generalize in the future 
+        because not much here depends on ANTs standards of tensor data.
+        ANTs xx,xy,xz,yy,yz,zz
+        DiPy Dxx, Dxy, Dyy, Dxz, Dyz, Dzz
+
+    """
+    dtiut = np.zeros(reference_image.shape + (6,), dtype=float)  
+    dtivec = np.zeros(6, dtype=float)  
+    it = np.ndindex( reference_image.shape )
+    yyind=2
+    xzind=3
+    if upper_triangular:
+        yyind=3
+        xzind=2
+    for i in it:
+        dtix = tensorarray[i] # in ANTs - we have: [xx,xy,xz,yy,yz,zz]
+        dtivec[0]=dtix[0,0]
+        dtivec[yyind]=dtix[1,1] # 2 for LT
+        dtivec[5]=dtix[2,2]
+        dtivec[1]=dtix[0,1]
+        dtivec[xzind]=dtix[2,0] # 3 for LT
+        dtivec[4]=dtix[1,2]
+        dtiut[i]=dtivec
+    dtiAnts = ants.from_numpy( dtiut, has_components=True )
+    ants.copy_image_info( reference_image, dtiAnts )
+    return dtiAnts
+
+def transform_and_reorient_dti( fixed, moving_dti, composite_transform, py_based=True, verbose=False, **kwargs):
+    """
+    apply a transform to DTI in the style of ants.apply_transforms. this function
+        expects a pre-computed composite transform which it will use to reorient 
+        the DTI using preservation of principle directions.
+    
+    fixed : antsImage reference space
+
+    moving_dti : antsImage DTI in upper triangular format
+
+    composite_transform : should be a composition of all transforms to be applied stored on disk ( a filename ) ... might change this in the future.
+
+    py_based : boolean
+
+    verbose : boolean
+
+    **kwargs : passed to ants.apply_transforms
+
+    """
+    if moving_dti.dimension != 3:
+        raise ValueError('moving image should have 3 dimensions')
+    if moving_dti.components != 6:
+        raise ValueError('moving image should have 6 components')
+    # now apply the transform to the template
+    # 1. transform the tensor components
+    dtsplit = moving_dti.split_channels()
+    dtiw = []
+    for k in range(len(dtsplit)):
+        dtiw.append( ants.apply_transforms( fixed, dtsplit[k], composite_transform ) )
+    dtiw=ants.merge_channels(dtiw)
+    if verbose:
+        print("reorient tensors locally: compose and get reo image")
+    locrot = ants.deformation_gradient( ants.image_read(composite_transform), 
+        to_rotation = True, py_based=py_based )
+    rebaser = np.dot( np.transpose( fixed.direction  ), moving_dti.direction )
+    if verbose:
+        print("convert UT to full tensor")
+    dtiw2tensor = triangular_to_tensor( dtiw )
+    if verbose:
+        print("rebase them to new space via iterator")
+    it = np.ndindex( fixed.shape )
+    for i in it:
+        # direction * dt * direction.transpose();
+        mmm = dtiw2tensor[i]
+        # transform rebase
+        locrotx = np.reshape( locrot[i], [3,3] )
+        mmm = np.dot( mmm, np.transpose( locrotx ) )
+        mmm = np.dot( locrotx, mmm )
+        # physical space rebase
+        mmm = np.dot( mmm, np.transpose( rebaser ) )
+        mmm = np.dot( rebaser, mmm )
+        dtiw2tensor[i] = mmm
+    if verbose:
+        print("done with rebasing")
+    return dti_numpy_to_image( fixed, dtiw2tensor )
+
 
 def dti_reg(
     image,
@@ -1344,7 +1651,9 @@ def dti_reg(
     if verbose:
         print("Reorient bvecs")
     if bvecs is not None:
-        bvecs = bvec_reorientation( motion_parameters, bvecs )
+        #    direction = target->GetDirection().GetTranspose() * img_mov->GetDirection().GetVnlMatrix();
+        rebase = np.dot( np.transpose( avg_b0.direction  ), ab0.direction )
+        bvecs = bvec_reorientation( motion_parameters, bvecs, rebase )
 
     if remove_it:
         import shutil
@@ -1525,7 +1834,7 @@ def mc_reg(
     }
 
 
-def get_data( name=None, force_download=False, version=11, target_extension='.csv' ):
+def get_data( name=None, force_download=False, version=13, target_extension='.csv' ):
     """
     Get ANTsPyMM data filename
 
@@ -2316,7 +2625,10 @@ def dipy_dti_recon(
         RGB = ants.merge_channels( [RGB0,RGB1,RGB2] )
         return tenfit, FA, MD1, RGB
 
-    gtab = gradient_table(bvals, bvecs)
+    import numpy as np
+    if abs(np.linalg.norm(bvecs)-1) > 0.009 and False:
+        bvecs=bvecs/np.linalg.norm(bvecs, axis=1)  
+    gtab = gradient_table(bvals, bvecs, atol=0.1 )
     tenfit, FA, MD1, RGB = justthefit( gtab, fit_method, image, maskdil )
     if verbose:
         print("recon dti.TensorModel done",flush=True)
@@ -2446,7 +2758,7 @@ def joint_dti_recon(
 
     t1w : antsimage t1w neuroimage (brain-extracted)
 
-    brain_mask : mask for the DWI - just 3D
+    brain_mask : mask for the DWI - just 3D - provided brain mask should be in reference_B0 space
 
     motion_correct : None Rigid or SyN
 
@@ -2483,11 +2795,11 @@ def joint_dti_recon(
             return( img )
 
     img_LR = fix_dwi_shape( img_LR, bval_LR, bvec_LR )
-    if denoise and srmodel is None:
+    if denoise :
         img_LR = mc_denoise( img_LR )
     if img_RL is not None:
         img_RL = fix_dwi_shape( img_RL, bval_RL, bvec_RL )
-        if denoise and srmodel is None:
+        if denoise :
             img_RL = mc_denoise( img_RL )
 
     if brain_mask is not None:
@@ -2608,7 +2920,9 @@ def joint_dti_recon(
     fa_SNR = mask_snr( reconFA, bgmask, fgmask, bias_correct=False )
     fa_evr = antspyt1w.patch_eigenvalue_ratio( reconFA, 512, [16,16,16], evdepth = 0.9, mask=recon_LR_dewarp['dwi_mask'] )
 
+    dti_itself = get_dti( reconFA, recon_LR_dewarp['tensormodel'], return_image=True )
     return {
+        'dti': dti_itself,
         'recon_fa':reconFA,
         'recon_fa_summary':df_FA_JHU_ORRL_bfwide,
         'recon_md':reconMD,
@@ -2850,7 +3164,10 @@ def dwi_deterministic_tracking(
     affine = dwi_img.affine
     if isinstance( bvals, str ) or isinstance( bvecs, str ):
         bvals, bvecs = read_bvals_bvecs(bvals, bvecs)
-    gtab = gradient_table(bvals, bvecs)
+    import numpy as np
+    if abs(np.linalg.norm(bvecs)-1) > 0.009 and False:
+        bvecs=bvecs/np.linalg.norm(bvecs, axis=1 )  
+    gtab = gradient_table(bvals, bvecs, atol=0.1 )
     if mask is None:
         mask = ants.threshold_image( fa, fa_thresh, 2.0 ).iMath("GetLargestComponent")
     dwi_data = dwi_img.get_fdata()
@@ -3018,7 +3335,10 @@ def dwi_closest_peak_tracking(
     affine = dwi_img.affine
     if isinstance( bvals, str ) or isinstance( bvecs, str ):
         bvals, bvecs = read_bvals_bvecs(bvals, bvecs)
-    gtab = gradient_table(bvals, bvecs)
+    import numpy as np
+    if abs(np.linalg.norm(bvecs)-1) > 0.009 and False:
+        bvecs=bvecs/np.linalg.norm(bvecs, axis=1)  
+    gtab = gradient_table(bvals, bvecs, atol=0.1 )
     if mask is None:
         mask = ants.threshold_image( fa, fa_thresh, 2.0 ).iMath("GetLargestComponent")
     dwi_data = dwi_img.get_fdata()
@@ -4003,15 +4323,18 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
     netname = re.sub( "-", "", netname )
     ww = np.where( powers_areal_mni_itk['SystemName'] == networks[mynet] )[0]
     dfnImg = ants.make_points_image(pts2bold.iloc[ww,:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
-    dfnmat = ants.timeseries_to_matrix( simg, ants.threshold_image( dfnImg, 1, dfnImg.max() ) )
-    dfnmat = ants.bandpass_filter_matrix( dfnmat, tr = tr, lowf=f[0], highf=f[1]  )
-    dfnmat = ants.regress_components( dfnmat, nuisance )
-    dfnsignal = dfnmat.mean( axis = 1 )
-    gmmatDFNCorr = np.zeros( gmmat.shape[1] )
-    for k in range( gmmat.shape[1] ):
-      gmmatDFNCorr[ k ] = pearsonr( dfnsignal, gmmat[:,k] )[0]
-    corrImg = ants.make_image( gmseg, gmmatDFNCorr  )
-    outdict[ netname ] = corrImg
+    if dfnImg.max() >= 1:
+        dfnmat = ants.timeseries_to_matrix( simg, ants.threshold_image( dfnImg, 1, dfnImg.max() ) )
+        dfnmat = ants.bandpass_filter_matrix( dfnmat, tr = tr, lowf=f[0], highf=f[1]  )
+        dfnmat = ants.regress_components( dfnmat, nuisance )
+        dfnsignal = dfnmat.mean( axis = 1 )
+        gmmatDFNCorr = np.zeros( gmmat.shape[1] )
+        for k in range( gmmat.shape[1] ):
+            gmmatDFNCorr[ k ] = pearsonr( dfnsignal, gmmat[:,k] )[0]
+        corrImg = ants.make_image( gmseg, gmmatDFNCorr  )
+        outdict[ netname ] = corrImg
+    else:
+        outdict[ netname ] = None
     ct = ct + 1
 
   A = np.zeros( ( len( numofnets ) , len( numofnets ) ) )
@@ -4105,11 +4428,15 @@ def write_bvals_bvecs(bvals, bvecs, prefix ):
     N = len(bvals)
     fname = prefix + '.bval'
     fmt = _VAL_FMT * N + '\n'
-    open(fname, 'wt').write(fmt % bvals)
+    myfile = open(fname, 'wt')
+    myfile.write(fmt % bvals)
+    myfile.close()
     fname = prefix + '.bvec'
     bvf = open(fname, 'wt')
     for dim_vals in bvecs.T:
         bvf.write(fmt % tuple(dim_vals))
+    bvf.close()
+    
 
 def crop_mcimage( x, mask, padder=None ):
     """
@@ -4152,6 +4479,8 @@ def mm(
     do_tractography = False,
     do_kk = False,
     do_normalization = None,
+    group_template = None,
+    group_transform = None,
     target_range = [0,1],
     dti_motion_correct = 'Rigid',
     dti_denoise = False,
@@ -4190,6 +4519,10 @@ def mm(
 
     do_normalization : template transformation if available
 
+    group_template : optional reference template corresponding to the group_transform
+
+    group_transform : optional transforms corresponding to the group_template
+
     target_range : 2-element tuple
         a tuple or array defining the (min, max) of the input image
         (e.g., [-127.5, 127.5] or [0,1]).  Output images will be scaled back to original
@@ -4225,6 +4558,9 @@ def mm(
     JHU_atlas = mm_read( JHU_atlasfn ) # Read in JHU atlas
     JHU_labels = mm_read( JHU_labelsfn ) # Read in JHU labels
     template = mm_read( templatefn ) # Read in template
+    if group_template is None:
+        group_template = template
+        group_transform = do_normalization['fwdtransforms']
     #####################
     #  T1 hierarchical  #
     #####################
@@ -4248,6 +4584,7 @@ def mm(
     normalization_dict = {
         'kk_norm': None,
         'NM_norm' : None,
+        'DTI_norm': None,
         'FA_norm' : None,
         'MD_norm' : None,
         'alff_norm' : None,
@@ -4454,24 +4791,32 @@ def mm(
         # t1reg = ants.registration( template, hier['brain_n4_dnz'], "antsRegistrationSyNQuickRepro[s]")
         t1reg = do_normalization
         if do_kk:
-            normalization_dict['kk_norm'] = ants.apply_transforms( template, output_dict['kk']['thickness_image'], t1reg['fwdtransforms'])
+            normalization_dict['kk_norm'] = ants.apply_transforms( group_template, output_dict['kk']['thickness_image'], group_transform )
         if output_dict['DTI'] is not None:
             mydti = output_dict['DTI']
             dtirig = ants.registration( hier['brain_n4_dnz'], mydti['recon_fa'], 'Rigid' )
-            normalization_dict['MD_norm'] = ants.apply_transforms( template, mydti['recon_md'],t1reg['fwdtransforms']+dtirig['fwdtransforms'] )
-            normalization_dict['FA_norm'] = ants.apply_transforms( template, mydti['recon_fa'],t1reg['fwdtransforms']+dtirig['fwdtransforms'] )
+            normalization_dict['MD_norm'] = ants.apply_transforms( group_template, mydti['recon_md'],group_transform+dtirig['fwdtransforms'] )
+            normalization_dict['FA_norm'] = ants.apply_transforms( group_template, mydti['recon_fa'],group_transform+dtirig['fwdtransforms'] )
+            output_directory = tempfile.mkdtemp()
+            comptx = ants.apply_transforms( group_template, group_template, 
+                group_transform+dtirig['fwdtransforms'], 
+                compose = output_directory + '/xxx' )
+            normalization_dict['DTI_norm'] = transform_and_reorient_dti(
+                group_template, mydti['dti'], comptx, py_based=True, verbose=True )
+            import shutil
+            shutil.rmtree(output_directory, ignore_errors=True )
         if output_dict['rsf'] is not None:
             rsfpro = output_dict['rsf']
             rsfrig = ants.registration( hier['brain_n4_dnz'], rsfpro['meanBold'], 'Rigid' )
             for netid in mynets:
                 rsfkey = netid + "_norm"
                 normalization_dict[rsfkey] = ants.apply_transforms(
-                    template, rsfpro[netid],
-                    t1reg['fwdtransforms']+rsfrig['fwdtransforms'] )
+                    group_template, rsfpro[netid],
+                    group_transform+rsfrig['fwdtransforms'] )
         if nm_image_list is not None:
             nmpro = output_dict['NM']
             nmrig = nmpro['t1_to_NM_transform'] # this is an inverse tx
-            normalization_dict['NM_norm'] = ants.apply_transforms( template, nmpro['NM_avg'],t1reg['fwdtransforms']+nmrig,
+            normalization_dict['NM_norm'] = ants.apply_transforms( group_template, nmpro['NM_avg'], group_transform+nmrig,
                 whichtoinvert=[False,False,True])
 
     if verbose:
@@ -4479,7 +4824,7 @@ def mm(
     return output_dict, normalization_dict
 
 
-def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_' ):
+def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbose=False ):
     """
     write the tabular and normalization output of the mm function
 
@@ -4496,6 +4841,8 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_' ):
 
     separator : string or character separator for filenames
 
+    verbose : boolean
+
     Returns
     ---------
 
@@ -4529,6 +4876,7 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_' ):
     if mm['DTI'] is not None:
         mydti = mm['DTI']
         myop = output_prefix + separator
+        ants.image_write( mydti['dti'],  myop + 'dti.nii.gz' )
         write_bvals_bvecs( mydti['bval_LR'], mydti['bvec_LR'], myop + 'reoriented' )
         image_write_with_thumbnail( mydti['dwi_LR_dewarped'],  myop + 'dwi.nii.gz' )
         image_write_with_thumbnail( mydti['dtrecon_LR_dewarp']['RGB'] ,  myop + 'DTIRGB.nii.gz' )
@@ -4639,6 +4987,8 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_' ):
             mm_wide['dti_FD_mean'] = mm_wide['dti_FD_max'] = 'NA'
     mmwidefn = output_prefix + separator + 'mmwide.csv'
     mm_wide.to_csv( mmwidefn )
+    if verbose:
+        print( output_prefix + " write_mm done." )
     return
 
 
@@ -4781,7 +5131,8 @@ def mm_nrg(
         'fwdtransforms': [ regout+'1Warp.nii.gz', regout+'0GenericAffine.mat'],
         'invtransforms': [ regout+'0GenericAffine.mat', regout+'1InverseWarp.nii.gz']  }
     if verbose:
-        print( "REGISTRATION EXISTENCE: " +
+        print( "-<REGISTRATION EXISTENCE>-: \n" + 
+              "NAMING: " + regout+'0GenericAffine.mat' + " \n " +
             str(exists( templateTx['fwdtransforms'][0])) + " " +
             str(exists( templateTx['fwdtransforms'][1])) + " " +
             str(exists( templateTx['invtransforms'][0])) + " " +
@@ -5133,14 +5484,14 @@ def mm_nrg(
                                 mydti = tabPro['DTI']
                                 if visualize:
                                     maxslice = np.min( [21, mydti['recon_fa'] ] )
-                                    ants.plot( mydti['recon_fa'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='FA (supposed to be better)', filename=mymm+mysep+"FAbetter.png"  )
+                                    ants.plot( mydti['recon_fa'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='FA', filename=mymm+mysep+"FAbetter.png"  )
                                     ants.plot( mydti['recon_fa'], mydti['jhu_labels'], axis=2, nslices=maxslice, ncol=7, crop=True, title='FA + JHU', filename=mymm+mysep+"FAJHU.png"  )
                                     ants.plot( mydti['recon_md'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='MD', filename=mymm+mysep+"MD.png"  )
                             if dowrite:
-                                write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep )
+                                write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep, verbose=True )
                                 for mykey in normPro.keys():
                                     if normPro[mykey] is not None:
-                                        if visualize:
+                                        if visualize and normPro[mykey].components == 1:
                                             ants.plot( template, normPro[mykey], axis=2, nslices=21, ncol=7, crop=True, title=mykey, filename=mymm+mysep+mykey+".png"   )
         if overmodX == nrg_modality_list[ len( nrg_modality_list ) - 1 ]:
             return
@@ -5158,9 +5509,13 @@ def mm_csv(
     srmodel_T1 = False, # optional - will add a great deal of time
     srmodel_NM = False, # optional - will add a great deal of time
     srmodel_DTI = False, # optional - will add a great deal of time
-    dti_motion_correct = 'Rigid',
-    dti_denoise = False,
-    nrg_modality_list = None
+    dti_motion_correct = 'SyN',
+    dti_denoise = True,
+    nrg_modality_list = None,
+    normalization_template = None,
+    normalization_template_output = None,
+    normalization_template_transform_type = "antsRegistrationSyNRepro[s]",
+    normalization_template_spacing=None
 ):
     """
     too dangerous to document ... use with care.
@@ -5220,6 +5575,17 @@ def mm_csv(
 
     nrg_modality_list : optional; defaults to None; use to focus on a given modality
 
+    normalization_template : optional; defaults to None; if present, all images will
+        be deformed into this space and the deformation will be stored with an extension
+        related to this variable.  this should be a brain extracted T1w image.
+
+    normalization_template_output : optional string; defaults to None; naming for the 
+        normalization_template outputs which will be in the T1w directory.
+
+    normalization_template_transform_type : optional string transform type passed to ants.registration
+
+    normalization_template_spacing : 3-tuple controlling the resolution at which registration is computed 
+
     Returns
     ---------
 
@@ -5305,7 +5671,6 @@ def mm_csv(
                         iid2=iid+"_"+t1iid
                     myoutputPrefix = outputdir + "/" + projid + "/" + sid + "/" + dtid + "/" + locmod + '/' + iid + "/" + projid + mysep + sid + mysep + dtid + mysep + locmod + mysep + iid2
         if verbose:
-            print("VERBOSE in docsamson")
             print( locmod )
             print( myimgsInput )
             print( myoutputPrefix )
@@ -5323,22 +5688,30 @@ def mm_csv(
     t1 = mm_read( t1fn, modality='T1w' )
     hierfn = outputdir + "/"  + projid + "/" + sid + "/" + dtid + "/" + "T1wHierarchical" + '/' + iid + "/" + projid + mysep + sid + mysep + dtid + mysep + "T1wHierarchical" + mysep + iid + mysep
     hierfnSR = outputdir + "/" + projid + "/"  + sid + "/" + dtid + "/" + "T1wHierarchicalSR" + '/' + iid + "/" + projid + mysep + sid + mysep + dtid + mysep + "T1wHierarchicalSR" + mysep + iid + mysep
-    hierfntest = hierfn + 'snseg.csv'
+    hierfntest = hierfn + 'cerebellum.csv'
     if verbose:
         print( hierfntest )
-    regout = hierfn + "syn"
+    regout = re.sub("T1wHierarchical","T1w",hierfn) + "syn"
     templateTx = {
         'fwdtransforms': [ regout+'1Warp.nii.gz', regout+'0GenericAffine.mat'],
         'invtransforms': [ regout+'0GenericAffine.mat', regout+'1InverseWarp.nii.gz']  }
+    groupTx = None
+    if normalization_template_output is not None:
+        normout = re.sub("T1wHierarchical","T1w",hierfn) +  normalization_template_output
+        templateNormTx = {
+            'fwdtransforms': [ normout+'1Warp.nii.gz', normout+'0GenericAffine.mat'],
+            'invtransforms': [ normout+'0GenericAffine.mat', normout+'1InverseWarp.nii.gz']  }
+        groupTx = templateNormTx['fwdtransforms']
     if verbose:
-        print( "REGISTRATION EXISTENCE: " +
+        print( "-<REGISTRATION EXISTENCE>-: \n" + 
+              "NAMING: " + regout+'0GenericAffine.mat' + " \n " +
             str(exists( templateTx['fwdtransforms'][0])) + " " +
             str(exists( templateTx['fwdtransforms'][1])) + " " +
             str(exists( templateTx['invtransforms'][0])) + " " +
             str(exists( templateTx['invtransforms'][1])) )
     if verbose:
         print( hierfntest )
-    hierexists = exists( hierfntest ) # FIXME should test this explicitly but we assume it here
+    hierexists = exists( hierfntest ) and exists( templateTx['fwdtransforms'][0]) and exists( templateTx['fwdtransforms'][1]) and exists( templateTx['invtransforms'][0]) and exists( templateTx['invtransforms'][1])
     hier = None
     if not hierexists and not testloop:
         subjectpropath = os.path.dirname( hierfn )
@@ -5392,6 +5765,43 @@ def mm_csv(
     if not testloop:
         t1imgbrn = hier['brain_n4_dnz']
         t1atropos = hier['dkt_parc']['tissue_segmentation']
+
+    if not exists( regout + "logjacobian.nii.gz" ) or not exists( regout+'1Warp.nii.gz' ):
+        if verbose:
+            print('start t1 registration')
+        ex_path = os.path.expanduser( "~/.antspyt1w/" )
+        templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
+        template = mm_read( templatefn )
+        template = ants.resample_image( template, [1,1,1], use_voxels=False )
+        t1reg = ants.registration( template, 
+            hier['brain_n4_dnz'],
+            "antsRegistrationSyNQuickRepro[s]", outprefix = regout, verbose=False )
+        myjac = ants.create_jacobian_determinant_image( template,
+            t1reg['fwdtransforms'][0], do_log=True, geom=True )
+        image_write_with_thumbnail( myjac, regout + "logjacobian.nii.gz", thumb=False )
+        if visualize:
+            ants.plot( ants.iMath(t1reg['warpedmovout'],"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='warped to template', filename=regout+"totemplate.png" )
+            ants.plot( ants.iMath(myjac,"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='jacobian', filename=regout+"jacobian.png" )
+
+    if normalization_template_output is not None and normalization_template is not None:
+        if verbose:
+            print("begin group template registration")
+        if not exists( normout+'0GenericAffine.mat' ):
+            if normalization_template_spacing is not None:
+                normalization_template_rr=ants.resample_image(normalization_template,normalization_template_spacing)
+            else:
+                normalization_template_rr=normalization_template
+            greg = ants.registration( 
+                normalization_template_rr, 
+                hier['brain_n4_dnz'],
+                normalization_template_transform_type,
+                outprefix = normout, verbose=False )
+            if verbose:
+                print("end group template registration")
+        else:
+            if verbose:
+                print("group template registration already done")
+
     # loop over modalities and then unique image IDs
     # we treat NM in a "special" way -- aggregating repeats
     # other modalities (beyond T1) are treated individually
@@ -5456,6 +5866,8 @@ def mm_csv(
                             do_tractography=False,
                             do_kk=False,
                             do_normalization=templateTx,
+                            group_template = normalization_template,
+                            group_transform = groupTx,
                             test_run=test_run,
                             verbose=True )
                     if not test_run:
@@ -5495,21 +5907,6 @@ def mm_csv(
                             img = mm_read( myimg )
                             ishapelen = len( img.shape )
                             if mymod == 'T1w' and ishapelen == 3: # for a real run, set to True
-                                if not exists( regout + "logjacobian.nii.gz" ) or not exists( regout+'1Warp.nii.gz' ):
-                                    if verbose:
-                                        print('start t1 registration')
-                                    ex_path = os.path.expanduser( "~/.antspyt1w/" )
-                                    templatefn = ex_path + 'CIT168_T1w_700um_pad_adni.nii.gz'
-                                    template = mm_read( templatefn )
-                                    template = ants.resample_image( template, [1,1,1], use_voxels=False )
-                                    t1reg = ants.registration( template, hier['brain_n4_dnz'],
-                                        "antsRegistrationSyNQuickRepro[s]", outprefix = regout, verbose=False )
-                                    myjac = ants.create_jacobian_determinant_image( template,
-                                        t1reg['fwdtransforms'][0], do_log=True, geom=True )
-                                    image_write_with_thumbnail( myjac, regout + "logjacobian.nii.gz", thumb=False )
-                                    if visualize:
-                                        ants.plot( ants.iMath(t1reg['warpedmovout'],"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='warped to template', filename=regout+"totemplate.png" )
-                                        ants.plot( ants.iMath(myjac,"Normalize"),  axis=2, nslices=21, ncol=7, crop=True, title='jacobian', filename=regout+"jacobian.png" )
                                 if not exists( mymm + mysep + "kk_norm.nii.gz" ):
                                     dowrite=True
                                     if verbose:
@@ -5519,6 +5916,8 @@ def mm_csv(
                                         do_tractography=False,
                                         do_kk=True,
                                         do_normalization=templateTx,
+                                        group_template = normalization_template,
+                                        group_transform = groupTx,
                                         test_run=test_run,
                                         verbose=True )
                                     if visualize:
@@ -5534,6 +5933,8 @@ def mm_csv(
                                     do_tractography=False,
                                     do_kk=False,
                                     do_normalization=templateTx,
+                                    group_template = normalization_template,
+                                    group_transform = groupTx,
                                     test_run=test_run,
                                     verbose=True )
                                 if visualize:
@@ -5556,6 +5957,8 @@ def mm_csv(
                                     do_tractography=False,
                                     do_kk=False,
                                     do_normalization=templateTx,
+                                    group_template = normalization_template,
+                                    group_transform = groupTx,
                                     test_run=test_run,
                                     verbose=True )
                                 if tabPro['rsf'] is not None and visualize:
@@ -5571,12 +5974,12 @@ def mm_csv(
                                     ants.plot( tabPro['rsf']['meanBold'], tabPro['rsf']['FrontoparietalTaskControl'],
                                         axis=2, nslices=maxslice, ncol=7, crop=True, title='FrontoparietalTaskControl', filename=mymm+mysep+"boldFrontoparietalTaskControl.png"  )
                             if ( mymod == 'DTI_LR' or mymod == 'DTI_RL' or mymod == 'DTI' ) and ishapelen == 4:
-                                dowrite=True
                                 bvalfn = re.sub( '.nii.gz', '.bval' , myimg )
                                 bvecfn = re.sub( '.nii.gz', '.bvec' , myimg )
                                 imgList = [ img ]
                                 bvalfnList = [ bvalfn ]
                                 bvecfnList = [ bvecfn ]
+                                missing_dti_data=False # bval, bvec or images
                                 if len( myimgsr ) > 1:  # find DTI_RL
                                     dtilrfn = myimgsr[myimgcount+1]
                                     if exists( dtilrfn ):
@@ -5589,44 +5992,50 @@ def mm_csv(
                                 # check existence of all files expected ...
                                 for dtiex in bvalfnList+bvecfnList+myimgsr:
                                     if not exists(dtiex):
-                                        raise ValueError('mm_csv dti data ' + dtiex )
-                                srmodel_DTI_mdl=None
-                                if srmodel_DTI is not False:
-                                    temp = ants.get_spacing(img)
-                                    dtspc=[temp[0],temp[1],temp[2]]
-                                    bestup = siq.optimize_upsampling_shape( dtspc, modality='DTI' )
-                                    mdlfn = ex_pathmm + "siq_default_sisr_" + bestup + "_1chan_featvggL6_best_mdl.h5"
-                                    if isinstance( srmodel_DTI, str ):
-                                        srmodel_DTI = re.sub( "bestup", bestup, srmodel_DTI )
-                                        mdlfn = os.path.join( ex_pathmm, srmodel_DTI )
-                                    if exists( mdlfn ):
-                                        if verbose:
-                                            print(mdlfn)
-                                        srmodel_DTI_mdl = tf.keras.models.load_model( mdlfn, compile=False )
-                                    else:
-                                        print(mdlfn + " does not exist - wont use SR")
-                                tabPro, normPro = mm( t1, hier,
-                                    dw_image=imgList,
-                                    bvals = bvalfnList,
-                                    bvecs = bvecfnList,
-                                    srmodel=srmodel_DTI_mdl,
-                                    do_tractography=not test_run,
-                                    do_kk=False,
-                                    do_normalization=templateTx,
-                                    dti_motion_correct = dti_motion_correct,
-                                    dti_denoise = dti_denoise,
-                                    test_run=test_run,
-                                    verbose=True )
-                                mydti = tabPro['DTI']
-                                if visualize:
-                                    maxslice = np.min( [21, mydti['recon_fa'] ] )
-                                    ants.plot( mydti['recon_fa'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='FA (supposed to be better)', filename=mymm+mysep+"FAbetter.png"  )
-                                    ants.plot( mydti['recon_fa'], mydti['jhu_labels'], axis=2, nslices=maxslice, ncol=7, crop=True, title='FA + JHU', filename=mymm+mysep+"FAJHU.png"  )
-                                    ants.plot( mydti['recon_md'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='MD', filename=mymm+mysep+"MD.png"  )
+                                        print('mm_csv: missing dti data ' + dtiex )
+                                        missing_dti_data=True
+                                        dowrite=False
+                                if not missing_dti_data:
+                                    dowrite=True
+                                    srmodel_DTI_mdl=None
+                                    if srmodel_DTI is not False:
+                                        temp = ants.get_spacing(img)
+                                        dtspc=[temp[0],temp[1],temp[2]]
+                                        bestup = siq.optimize_upsampling_shape( dtspc, modality='DTI' )
+                                        mdlfn = ex_pathmm + "siq_default_sisr_" + bestup + "_1chan_featvggL6_best_mdl.h5"
+                                        if isinstance( srmodel_DTI, str ):
+                                            srmodel_DTI = re.sub( "bestup", bestup, srmodel_DTI )
+                                            mdlfn = os.path.join( ex_pathmm, srmodel_DTI )
+                                        if exists( mdlfn ):
+                                            if verbose:
+                                                print(mdlfn)
+                                            srmodel_DTI_mdl = tf.keras.models.load_model( mdlfn, compile=False )
+                                        else:
+                                            print(mdlfn + " does not exist - wont use SR")
+                                    tabPro, normPro = mm( t1, hier,
+                                        dw_image=imgList,
+                                        bvals = bvalfnList,
+                                        bvecs = bvecfnList,
+                                        srmodel=srmodel_DTI_mdl,
+                                        do_tractography=not test_run,
+                                        do_kk=False,
+                                        do_normalization=templateTx,
+                                        group_template = normalization_template,
+                                        group_transform = groupTx,
+                                        dti_motion_correct = dti_motion_correct,
+                                        dti_denoise = dti_denoise,
+                                        test_run=test_run,
+                                        verbose=True )
+                                    mydti = tabPro['DTI']
+                                    if visualize:
+                                        maxslice = np.min( [21, mydti['recon_fa'] ] )
+                                        ants.plot( mydti['recon_fa'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='FA (supposed to be better)', filename=mymm+mysep+"FAbetter.png"  )
+                                        ants.plot( mydti['recon_fa'], mydti['jhu_labels'], axis=2, nslices=maxslice, ncol=7, crop=True, title='FA + JHU', filename=mymm+mysep+"FAJHU.png"  )
+                                        ants.plot( mydti['recon_md'],  axis=2, nslices=maxslice, ncol=7, crop=True, title='MD', filename=mymm+mysep+"MD.png"  )
                             if dowrite:
                                 write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep )
                                 for mykey in normPro.keys():
-                                    if normPro[mykey] is not None:
+                                    if normPro[mykey] is not None and normPro[mykey].components == 1:
                                         if visualize:
                                             ants.plot( template, normPro[mykey], axis=2, nslices=21, ncol=7, crop=True, title=mykey, filename=mymm+mysep+mykey+".png"   )
         if overmodX == nrg_modality_list[ len( nrg_modality_list ) - 1 ]:
@@ -6030,6 +6439,110 @@ def read_mm_csv( x, is_t1=False, colprefix=None, separator='-', verbose=False ):
         xdf.columns=colprefix + xdf.columns
     return pd.concat( [df,xdf], axis=1 )
 
+def merge_wides_to_study_dataframe( sdf, processing_dir, separator='-', sid_is_int=True, id_is_int=True, date_is_int=True, report_missing=False,
+progress=False, verbose=False ):
+    """
+    extend a study data frame with wide outputs
+
+    sdf : the input study dataframe
+
+    processing_dir:  the directory location of the processed data 
+
+    separator : string usually '-' or '_'
+
+    sid_is_int : boolean set to True to cast unique subject ids to int; can be useful if they are inadvertently stored as float by pandas
+
+    date_is_int : boolean set to True to cast date to int; can be useful if they are inadvertently stored as float by pandas
+
+    id_is_int : boolean set to True to cast unique image ids to int; can be useful if they are inadvertently stored as float by pandas
+
+    report_missing : boolean combined with verbose will report missing modalities
+
+    progress : integer reports percent progress modulo progress value 
+
+    verbose : boolean
+    """
+    from os.path import exists
+    musthavecols = ['projectID', 'subjectID','date','imageID','fn']
+    for k in range(len(musthavecols)):
+        if not musthavecols[k] in sdf.keys():
+            raise ValueError('sdf is missing column ' +musthavecols[k] + ' in merge_wides_to_study_dataframe' )
+    possible_iids = [ 'imageID', 'imageID', 'imageID', 'flairid', 'dtid1', 'dtid2', 'rsfid1', 'rsfid2', 'nmid1', 'nmid2', 'nmid3', 'nmid4', 'nmid5', 'nmid6', 'nmid7', 'nmid8', 'nmid9', 'nmid10' ]
+    modality_ids = [ 'T1wHierarchical', 'T1wHierarchicalSR', 'T1w', 'T2Flair', 'DTI', 'DTI', 'rsfMRI', 'rsfMRI', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT']
+    alldf=pd.DataFrame()
+    for myk in sdf.index:
+        if progress > 0 and int(myk) % int(progress) == 0:
+            print( str( round( myk/sdf.shape[0]*100.0)) + "%...", end='', flush=True)
+        if verbose:
+            print( "DOROW " + str(myk) + ' of ' + str( sdf.shape[0] ) )
+        csvrow = sdf.loc[sdf.index == myk].dropna(axis=1)
+        ct=-1
+        for iidkey in possible_iids:
+            ct=ct+1
+            mod_name = modality_ids[ct]
+            if iidkey in csvrow.keys():
+                if id_is_int:
+                    iid = str( int( csvrow[iidkey].iloc[0] ) )
+                else:
+                    iid = str( csvrow[iidkey].iloc[0] )
+                if verbose:
+                    print( "iidkey " + iidkey + " modality " + mod_name + ' iid '+ iid )
+                pid=str(csvrow['projectID'].iloc[0] )
+                if sid_is_int:
+                    sid=str(int(csvrow['subjectID'].iloc[0] ))
+                else:
+                    sid=str(csvrow['subjectID'].iloc[0] )
+                if date_is_int:
+                    dt=str(int(csvrow['date'].iloc[0]))
+                else:
+                    dt=str(csvrow['date'].iloc[0])
+                if id_is_int:
+                    t1iid=str(int(csvrow['imageID'].iloc[0]))
+                else:
+                    t1iid=str(csvrow['imageID'].iloc[0])
+                if t1iid != iid:
+                    iidj=iid+"_"+t1iid
+                else:
+                    iidj=iid
+                rootid = pid +separator+ sid +separator+dt+separator+mod_name+separator+iidj
+                myext = rootid +separator+'mmwide.csv'
+                nrgwidefn=os.path.join( processing_dir, pid, sid, dt, mod_name, iid, myext )
+                moddersub = mod_name
+                is_t1=False
+                if mod_name == 'T1wHierarchical':
+                    is_t1=True
+                    moddersub='T1Hier'
+                elif mod_name == 'T1wHierarchicalSR':
+                    is_t1=True
+                    moddersub='T1HSR'
+                if exists( nrgwidefn ):
+                    if verbose:
+                        print( nrgwidefn + " exists")
+                    mm=read_mm_csv( nrgwidefn, colprefix=moddersub+'_', is_t1=is_t1, separator=separator, verbose=verbose )
+                    if mm is not None:
+                        if mod_name == 'T1wHierarchical':
+                            a=list( csvrow.keys() )
+                            b=list( mm.keys() )
+                            abintersect=list(set(b).intersection( set(a) ) )
+                            if len( abintersect  ) > 0 :
+                                for qq in abintersect:
+                                    mm.pop( qq )
+                        mm.index=csvrow.index
+                        uidname = mod_name + '_mmwide_filename'
+                        mm[ uidname ] = rootid
+                        csvrow=pd.concat( [csvrow,mm], axis=1 )
+                else:
+                    if verbose and report_missing:
+                        print( nrgwidefn + " absent")
+        if alldf.shape[0] == 0:
+            alldf = csvrow.copy()
+            alldf = alldf.loc[:,~alldf.columns.duplicated()]
+        else:
+            csvrow=csvrow.loc[:,~csvrow.columns.duplicated()]
+            alldf = alldf.loc[:,~alldf.columns.duplicated()]
+            alldf = pd.concat( [alldf, csvrow], axis=0, ignore_index=True)
+    return alldf
+
 def assemble_modality_specific_dataframes( mm_wide_csvs, hierdfin, nrg_modality, separator='-', progress=None, verbose=False ):
     moddersub = re.sub( "[*]","",nrg_modality)
     nmdf=pd.DataFrame()
@@ -6136,7 +6649,7 @@ def boot_wmh( flair, t1, t1seg, mmfromconvexhull = 0.0, strict=True,
     if prior_probability is not None:
         augprob_prior = flair * 0.0
     for n in range(n_simulations):
-        augflair, tx, itx = augment_image( flair, 5 )
+        augflair, tx, itx = augment_image( ants.iMath(flair,"Normalize"), 5, 0.01 )
         locwmh = wmh( augflair, t1, t1seg, mmfromconvexhull = mmfromconvexhull,
             strict=strict, probability_mask=None, prior_probability=prior_probability )
         if verbose:
@@ -6581,7 +7094,7 @@ def blind_image_assessment(
     viz_filename=None,
     title=False,
     pull_rank=False,
-    resample='max',
+    resample=None,
     verbose=False
 ):
     """
@@ -6638,7 +7151,7 @@ def blind_image_assessment(
         json_name = re.sub(".nii.gz",".json",image_filename)
         if exists( json_name ):
             with open(json_name, 'r') as fcc_file:
-                mymeta = json.load(fcc_file)
+                mymeta = json.load(fcc_file, strict=False)
                 if verbose:
                     print(json.dumps(mymeta, indent=4))
         mystem=Path( image ).stem
@@ -6681,16 +7194,30 @@ def blind_image_assessment(
             else:
                 image_compare = ants.image_clone( image_b0 )
         image = ants.iMath( image, 'TruncateIntensity',0.01,0.995)
+        minspc = np.min(ants.get_spacing(image))
+        maxspc = np.max(ants.get_spacing(image))
         if resample is not None:
             if resample == 'min':
-                newspc = np.repeat( np.min(ants.get_spacing(image)), 3 )
+                if minspc < 1e-12:
+                    minspc = np.max(ants.get_spacing(image))
+                newspc = np.repeat( minspc, 3 )
             elif resample == 'max':
-                newspc = np.repeat( np.max(ants.get_spacing(image)), 3 )
+                newspc = np.repeat( maxspc, 3 )
             else:
                 newspc = np.repeat( resample, 3 )
             image = ants.resample_image( image, newspc )
             image_compare = ants.resample_image( image_compare, newspc )
+        else:
+            # check for spc close to zero
+            spc = list(ants.get_spacing(image))
+            for spck in range(len(spc)):
+                if spc[spck] < 1e-12:
+                    spc[spck]=1
+            ants.set_spacing( image, spc )
+            ants.set_spacing( image_compare, spc )
         # if "NM2DMT" in image_filename or "FIXME" in image_filename or "SPECT" in image_filename or "UNKNOWN" in image_filename:
+        minspc = np.min(ants.get_spacing(image))
+        maxspc = np.max(ants.get_spacing(image))
         msk = ants.threshold_image( ants.iMath(image,'Normalize'), 0.15, 1.0 )
         # else:
         #    msk = ants.get_mask( image )
@@ -6727,8 +7254,17 @@ def blind_image_assessment(
         imagereflect = ants.reflect_image(image, axis=0)
         asym_err = ( image - imagereflect ).abs().mean()
         # estimate noise by center cropping, denoizing and taking magnitude of difference
-        mycc = antspyt1w.special_crop( image,
-            ants.get_center_of_mass( msk *0 + 1 ), patch_shape )
+        nocrop=False
+        if image.dimension == 3:
+            if image.shape[2] == 1:
+                nocrop=True        
+        if maxspc/minspc > 10:
+            nocrop=True
+        if nocrop:
+            mycc = ants.image_clone( image )
+        else:
+            mycc = antspyt1w.special_crop( image,
+                ants.get_center_of_mass( msk *0 + 1 ), patch_shape )
         myccd = ants.denoise_image( mycc, p=2,r=2,noise_model='Gaussian' )
         noizlevel = ( mycc - myccd ).abs().mean()
 #        ants.plot_ortho( image, crop=False, filename=viz_filename, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0 )
@@ -6745,7 +7281,10 @@ def blind_image_assessment(
         cnrref = ( fgmean - bgmean ) / bgstd
         psnrref = antspynet.psnr(  image_compare, image  )
         ssimref = antspynet.ssim(  image_compare, image  )
-        mymi = ants.image_mutual_information( image_compare, image )
+        if nocrop:
+            mymi = math.inf
+        else:
+            mymi = ants.image_mutual_information( image_compare, image )
         mriseries='NA'
         mrimfg='NA'
         mrimodel='NA'
@@ -6794,9 +7333,7 @@ def average_blind_qc_by_modality(qc_full,verbose=False):
     uid = qc_full['fn'] + "_" + qc_full['modality'].astype(str)
     to_average = uid.unique()
     # Define column indices
-    contcols = ['noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi',
-       'reflection_err', 'EVR', 'msk_vol', 'spc0', 'spc1', 'spc2', 'dimx',
-       'dimy', 'dimz', 'slice']
+    contcols = ['noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi','reflection_err', 'EVR', 'msk_vol', 'spc0', 'spc1', 'spc2', 'org0','org1','org2', 'dimx', 'dimy', 'dimz', 'slice']
     ocols = ['fn','modality', 'mriseries', 'mrimfg', 'mrimodel']
     # restrict to columns we "know"
     qc_full = qc_full[ocols+contcols]

{antspymm-0.9.7.dist-info → antspymm-1.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: antspymm
-Version: 0.9.7
+Version: 1.0.0
 Summary: multi-channel/time-series medical image processing with antspyx
 Home-page: https://github.com/stnava/ANTsPyMM
 Author: Avants, Gosselin, Tustison, Reardon
@@ -11,7 +11,7 @@ License-File: LICENSE
 Requires-Dist: h5py (>=2.10.0)
 Requires-Dist: numpy (>=1.19.4)
 Requires-Dist: pandas (>=1.0.1)
-Requires-Dist: antspyx (>=0.2.7)
+Requires-Dist: antspyx (>=0.4.1)
 Requires-Dist: antspyt1w (>=0.2.3)
 Requires-Dist: pathlib
 Requires-Dist: dipy
@@ -91,6 +91,17 @@ will all modalities will take around 2 hours on an average laptop.
 
 documentation of functions [here](http://htmlpreview.github.io/?https://github.com/stnava/ANTsPyMM/blob/main/docs/antspymm/mm.html).
 
+
+achieved through four steps (recommended approach):
+
+1. organize data in NRG format
+
+2. perform blind QC
+
+3. compute outlierness per modality and select optimally matched modalities ( steps 3.1 and 3.2 )
+
+4. run the main antspymm function
+
 # first time setup
 
 ```python

antspymm-1.0.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+antspymm/__init__.py,sha256=BTNN_iisXoz2on4gVcAQWFB_qJYDM-yqZOJU1xho9sw,2931
+antspymm/mm.py,sha256=UMAxi91xK-rFUQT6StQIAkLKyphGMpM_Jy0pFGiY_8E,308235
+antspymm-1.0.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+antspymm-1.0.0.dist-info/METADATA,sha256=RjWizo9kiWyCCLDYjdP2LLDlcqDT0zmqU6Q8H4x8Zh8,10717
+antspymm-1.0.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+antspymm-1.0.0.dist-info/top_level.txt,sha256=iyD1sRhCKzfwKRJLq5ZUeV9xsv1cGQl8Ejp6QwXM1Zg,9
+antspymm-1.0.0.dist-info/RECORD,,

antspymm-0.9.7.dist-info/RECORD

@@ -1,7 +0,0 @@
-antspymm/__init__.py,sha256=81mPpK_dfpuD0bVTpTr1U5oxOA4dvUqcShgxo0herXc,2690
-antspymm/mm.py,sha256=6ETAYzaAw1-MznWTS6JN693OJshqa2AV6whPdLBoKDY,285326
-antspymm-0.9.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-antspymm-0.9.7.dist-info/METADATA,sha256=roFi7QJkXiLyAgdcPTsW8HgXKLlk2cuUAdkpTGAHPTM,10476
-antspymm-0.9.7.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-antspymm-0.9.7.dist-info/top_level.txt,sha256=iyD1sRhCKzfwKRJLq5ZUeV9xsv1cGQl8Ejp6QwXM1Zg,9
-antspymm-0.9.7.dist-info/RECORD,,