antspymm 1.1.7__py3-none-any.whl → 1.1.9__py3-none-any.whl

antspymm/__init__.py

@@ -103,9 +103,14 @@ from .mm import docsamson
 from .mm import enantiomorphic_filling_without_mask
 from .mm import loop_timeseries_censoring
 from .mm import score_fmri_censoring
+from .mm import replace_elements_in_numpy_array
 from .mm import remove_elements_from_numpy_array
 from .mm import remove_volumes_from_timeseries
 from .mm import impute_timeseries
 from .mm import impute_dwi
+from .mm import scrub_dwi
 from .mm import timeseries_n3
 from .mm import calculate_CBF
+from .mm import estimate_optimal_pca_components
+from .mm import filter_df
+

antspymm/mm.py

@@ -111,7 +111,7 @@ import re
 import datetime as dt
 from collections import Counter
 import tempfile
-
+import warnings
 
 from dipy.core.histeq import histeq
 import dipy.reconst.dti as dti
@@ -686,7 +686,7 @@ def collect_blind_qc_by_modality( modality_path, set_index_to_fn=True ):
         temp=pd.read_csv(fns[k])
         if not 'filename' in temp.keys():
             temp['filename']=fns[k]
-        jdf=pd.concat( [jdf,temp])
+        jdf=pd.concat( [jdf,temp], axis=0, ignore_index=False )
     if set_index_to_fn:
         jdf.reset_index(drop=True)
         if "Unnamed: 0" in jdf.columns:
@@ -3151,7 +3151,8 @@ def joint_dti_recon(
     dewarp_modality = 'FA',
     denoise=False,
     fit_method='WLS',
-    impute = True,
+    impute = False,
+    scrub = True,
     verbose = False ):
     """
     1. pass in subject data and 1mm JHU atlas/labels
@@ -3204,6 +3205,8 @@ def joint_dti_recon(
 
     impute : boolean
 
+    scrub : boolean
+
     verbose : boolean
 
     Returns
@@ -3291,8 +3294,12 @@ def joint_dti_recon(
 
     if impute:
         img_LRdwp=impute_dwi( img_LRdwp, verbose=True )
+    elif scrub:
+        img_LRdwp, reg_LR['bvals'], reg_LR['bvecs'] = scrub_dwi( img_LRdwp, reg_LR['bvals'], reg_LR['bvecs'], verbose=True )
     if impute and img_RL is not None:
         img_RLdwp=impute_dwi( img_RLdwp, verbose=True )
+    elif scrub and img_RL is not None:
+        img_RLdwp, reg_RL['bvals'], reg_RL['bvecs'] = scrub_dwi( img_RLdwp, reg_RL['bvals'], reg_RL['bvecs'], verbose=True )
 
     if img_RL is not None:
         img_LRdwp, bval_LR, bvec_LR = merge_dwi_data(
@@ -4171,7 +4178,7 @@ def dwi_streamline_connectivity_old(
             pathdfw,
             Mdfw,
             Tdfw,
-            Ctdfw ], axis=1 )
+            Ctdfw ], axis=1, ignore_index=False )
 
     return {
           'connectivity': allconnexwide,
@@ -4236,7 +4243,7 @@ def hierarchical_modality_summary(
         if verbose:
             print( mappedw.keys() )
         if mydf.shape[0] > 0:
-            mydf = pd.concat( [ mydf, mappedw], axis=1 )
+            mydf = pd.concat( [ mydf, mappedw], axis=1, ignore_index=False )
         else:
             mydf = mappedw
         return mydf
@@ -4596,10 +4603,53 @@ def neuromelanin( list_nm_images, t1, t1_head, t1lab, brain_stem_dilation=8,
       'NM_count': len( list_nm_images )
        }
 
+
+
+def estimate_optimal_pca_components(data, variance_threshold=0.80, plot=False):
+    """
+    Estimate the optimal number of PCA components to represent the given data.
+
+    :param data: The data matrix (samples x features).
+    :param variance_threshold: Threshold for cumulative explained variance (default 0.95).
+    :param plot: If True, plot the cumulative explained variance graph (default False).
+    :return: The optimal number of principal components.
+    """
+    import numpy as np
+    from sklearn.decomposition import PCA
+    import matplotlib.pyplot as plt
+
+    # Perform PCA
+    pca = PCA()
+    pca.fit(data)
+
+    # Calculate cumulative explained variance
+    cumulative_variance = np.cumsum(pca.explained_variance_ratio_)
+
+    # Determine the number of components for desired explained variance
+    n_components = np.where(cumulative_variance >= variance_threshold)[0][0] + 1
+
+    # Optionally plot the explained variance
+    if plot:
+        plt.figure(figsize=(8, 4))
+        plt.plot(cumulative_variance, linewidth=2)
+        plt.axhline(y=variance_threshold, color='r', linestyle='--')
+        plt.axvline(x=n_components - 1, color='r', linestyle='--')
+        plt.xlabel('Number of Components')
+        plt.ylabel('Cumulative Explained Variance')
+        plt.title('Explained Variance by Number of Principal Components')
+        plt.show()
+
+    return n_components
+
+
 def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
-    f=[0.03,0.08], FD_threshold=0.5, spa = 1.5, spt = 0.5, nc = 6, type_of_transform='Rigid',
+    f=[0.008,0.1], FD_threshold=0.5, spa = None, spt = None, 
+    nc = 0.90, type_of_transform='Rigid',
+    outlier_threshold=0.50,
+    ica_components = 0,
+    impute = False,
+    scrub = True,
     verbose=False ):
-
   """
   Compute resting state network correlation maps based on the J Power labels.
   This will output a map for each of the major network systems.
@@ -4618,14 +4668,20 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
 
   f : band pass limits for frequency filtering
 
-  spa : gaussian smoothing for spatial component
+  spa : gaussian smoothing for spatial component (physical coordinates)
 
   spt : gaussian smoothing for temporal component
 
-  nc  : number of components for compcor filtering
+  nc  : number of components for compcor filtering; if less than 1 we estimate on the fly based on explained variance
 
   type_of_transform : SyN or Rigid
 
+  ica_components : integer if greater than 0 then include ica components
+
+  impute : boolean if True, then use imputation
+
+  scrub : boolean if True, then use censoring (scrubbing)
+
   verbose : boolean
 
   Returns
@@ -4633,16 +4689,66 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
   a dictionary containing the derived network maps
 
   """
+
+  import numpy as np
+# Assuming core and utils are modules or packages with necessary functions
+
+  def compute_tSTD(M, quantile, x=0, axis=0):
+    stdM = np.std(M, axis=axis)
+    # set bad values to x
+    stdM[stdM == 0] = x
+    stdM[np.isnan(stdM)] = x
+    tt = round(quantile * 100)
+    threshold_std = np.percentile(stdM, tt)
+    return {'tSTD': stdM, 'threshold_std': threshold_std}
+
+  def get_compcor_matrix(boldImage, mask, quantile):
+    """
+    Compute the compcor matrix.
+
+    :param boldImage: The bold image.
+    :param mask: The mask to apply, if None, it will be computed.
+    :param quantile: Quantile for computing threshold in tSTD.
+    :return: The compor matrix.
+    """
+    if mask is None:
+        temp = ants.slice_image(boldImage, axis=boldImage.dimension - 1, idx=0)
+        mask = ants.get_mask(temp)
+
+    imagematrix = ants.timeseries_to_matrix(boldImage, mask)
+    temp = compute_tSTD(imagematrix, quantile, 0)
+    tsnrmask = ants.make_image(mask, temp['tSTD'])
+    tsnrmask = ants.threshold_image(tsnrmask, temp['threshold_std'], temp['tSTD'].max())
+    M = ants.timeseries_to_matrix(boldImage, tsnrmask)
+    return M
+
+
+  from sklearn.decomposition import FastICA
+  def find_indices(lst, value):
+    return [index for index, element in enumerate(lst) if element > value]
+
+  def mean_of_list(lst):
+    if not lst:  # Check if the list is not empty
+        return 0  # Return 0 or appropriate value for an empty list
+    return sum(lst) / len(lst)
+  fmrispc = list(ants.get_spacing( fmri ))
+  if spa is None:
+    spa = mean_of_list( fmrispc[0:3] ) * 1.0
+  if spt is None:
+    spt = fmrispc[3] * 0.5
+      
   import numpy as np
   import pandas as pd
   import re
   import math
+  # point data resources
   A = np.zeros((1,1))
   powers_areal_mni_itk = pd.read_csv( get_data('powers_mni_itk', target_extension=".csv")) # power coordinates
   fmri = ants.iMath( fmri, 'Normalize' )
   bmask = antspynet.brain_extraction( fmri_template, 'bold' ).threshold_image(0.5,1).iMath("FillHoles")
   if verbose:
       print("Begin rsfmri motion correction")
+  # mot-co
   corrmo = timeseries_reg(
     fmri, fmri_template,
     type_of_transform=type_of_transform,
@@ -4654,13 +4760,21 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
     syn_metric='cc',
     syn_sampling=2,
     reg_iterations=[40,20,5] )
+  
   if verbose:
       print("End rsfmri motion correction")
+      print("big code block below does anatomically based mapping")
+
+  high_motion_count=(corrmo['FD'] > FD_threshold ).sum()
+  high_motion_pct=high_motion_count / fmri.shape[3]
 
+  # filter mask based on TSNR
   mytsnr = tsnr( corrmo['motion_corrected'], bmask )
   mytsnrThresh = np.quantile( mytsnr.numpy(), 0.995 )
   tsnrmask = ants.threshold_image( mytsnr, 0, mytsnrThresh ).morphology("close",2)
   bmask = bmask * tsnrmask
+
+  # anatomical mapping
   und = fmri_template * bmask
   t1reg = ants.registration( und, t1, "SyNBold" )
   if verbose:
@@ -4677,14 +4791,7 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
                ants.threshold_image( t1segmentation, 3, 3 ) ).morphology("erode",1)
   csfAndWM = ants.apply_transforms( und, csfAndWM,
     t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
-
-  # get falff and alff
-  mycompcor = ants.compcor( corrmo['motion_corrected'],
-    ncompcor=nc, quantile=0.90, mask = csfAndWM,
-    filter_type='polynomial', degree=2 )
-
   nt = corrmo['motion_corrected'].shape[3]
-
   myvoxes = range(powers_areal_mni_itk.shape[0])
   anat = powers_areal_mni_itk['Anatomy']
   syst = powers_areal_mni_itk['SystemName']
@@ -4698,26 +4805,98 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
   locations = pts2bold.iloc[:,:3].values
   ptImg = ants.make_points_image( locations, bmask, radius = 2 )
 
+  # optional smoothing
   tr = ants.get_spacing( corrmo['motion_corrected'] )[3]
-  highMotionTimes = np.where( corrmo['FD'] >= 1.0 )
-  goodtimes = np.where( corrmo['FD'] < 0.5 )
-  smth = ( spa, spa, spa, spt ) # this is for sigmaInPhysicalCoordinates = F
-  simg = ants.smooth_image(corrmo['motion_corrected'], smth, sigma_in_physical_coordinates = False )
+  smth = ( spa, spa, spa, spt ) # this is for sigmaInPhysicalCoordinates = TRUE
+  simg = ants.smooth_image( corrmo['motion_corrected'], smth, sigma_in_physical_coordinates = True )
 
+  ants.image_write(corrmo['motion_corrected'],'/tmp/tempZZZ.nii.gz')
+
+  # collect censoring indices
+  hlinds = find_indices( corrmo['FD'], FD_threshold )
+  if verbose:
+    print("high motion indices")
+    print( hlinds )
+  if outlier_threshold < 1.0 and outlier_threshold > 0.0:
+    fmrimotcorr, hlinds2 = loop_timeseries_censoring( corrmo['motion_corrected'], 
+      threshold=outlier_threshold, verbose=verbose )
+    hlinds.extend( hlinds2 )
+  hlinds = list(set(hlinds)) # make unique
+    
+  # nuisance
+  globalmat = ants.timeseries_to_matrix( corrmo['motion_corrected'], bmask )
+  globalsignal = globalmat.mean( axis = 1 )
+  del globalmat
+  compcorquantile=0.975
+  if nc < 1:
+    globalmat = get_compcor_matrix( corrmo['motion_corrected'], csfAndWM, compcorquantile )
+    nc = estimate_optimal_pca_components( data=globalmat, variance_threshold=nc)
+    del globalmat
+  if verbose:
+    print("include compcor components as nuisance: " + str(nc))
+  mycompcor = ants.compcor( corrmo['motion_corrected'],
+    ncompcor=nc, quantile=compcorquantile, mask = csfAndWM,
+    filter_type='polynomial', degree=1 )
   nuisance = mycompcor[ 'components' ]
+
+  if ica_components > 0:
+    if verbose:
+        print("include ica components as nuisance: " + str(ica_components))
+    ica = FastICA(n_components=ica_components, max_iter=10000, tol=0.001, random_state=42 )
+    globalmat = ants.timeseries_to_matrix( corrmo['motion_corrected'], csfAndWM )
+    nuisance_ica = ica.fit_transform(globalmat)  # Reconstruct signals
+    nuisance = np.c_[ nuisance, nuisance_ica ]
+    del globalmat
+
+  # concat all nuisance data
   nuisance = np.c_[ nuisance, mycompcor['basis'] ]
   nuisance = np.c_[ nuisance, corrmo['FD'] ]
+  nuisance = np.c_[ nuisance, globalsignal ]
 
-  gmmat = ants.timeseries_to_matrix( simg, gmseg )
-  gmmat = ants.bandpass_filter_matrix( gmmat, tr = tr, lowf=f[0], highf=f[1] ) # some would argue against this
-  gmsignal = gmmat.mean( axis = 1 )
-  nuisance = np.c_[ nuisance, gmsignal ]
+  # bandpass any data collected before here -- if bandpass requested
+  if f[0] > 0 and f[1] < 1.0:
+    if verbose:
+        print( "bandpass: " + str(f[0]) + " <=> " + str( f[1] ) )
+    globalmat = ants.timeseries_to_matrix( corrmo['motion_corrected'], bmask )
+    globalmat = ants.bandpass_filter_matrix( globalmat, tr = tr, lowf=f[0], highf=f[1] ) # some would argue against this
+    corrmo['motion_corrected'] = ants.matrix_to_timeseries( corrmo['motion_corrected'], globalmat, bmask )
+    globalmat = ants.timeseries_to_matrix( simg, bmask )
+    globalmat = ants.bandpass_filter_matrix( globalmat, tr = tr, lowf=f[0], highf=f[1] ) # some would argue against this
+    simg = ants.matrix_to_timeseries( simg, globalmat, bmask )
+    nuisance = ants.bandpass_filter_matrix( nuisance, tr = tr, lowf=f[0], highf=f[1] ) # some would argue against this
+
+  nuisanceall = nuisance.copy()
+  if len( hlinds ) > 0 :
+    if impute and not scrub:
+        corrmo['FD'] = replace_elements_in_numpy_array( corrmo['FD'], hlinds, corrmo['FD'].mean() )
+        corrmo['motion_corrected'] = impute_timeseries( corrmo['motion_corrected'], hlinds, method='linear')
+        simg = simgimp = impute_timeseries( simg, hlinds, method='linear')
+    elif scrub:
+        corrmo['FD'] = remove_elements_from_numpy_array( corrmo['FD'], hlinds  )
+        corrmo['motion_corrected'] = remove_volumes_from_timeseries( corrmo['motion_corrected'], hlinds )
+        simgimp = impute_timeseries( simg, hlinds, method='linear')
+        nuisance = remove_elements_from_numpy_array( nuisance, hlinds  )
+        simg = remove_volumes_from_timeseries( simg, hlinds )
+    else:
+        simgimp = simg
+  else:
+    simgimp = simg
+
+  if verbose:
+    print("now regress nuisance")
+
+  gmmat = ants.timeseries_to_matrix( simgimp, bmask )
+  gmmat = ants.regress_components( gmmat, nuisanceall )
+  simgimp = ants.matrix_to_timeseries(simgimp, gmmat, bmask)
+
+  gmmat = ants.timeseries_to_matrix( simg, bmask )
   gmmat = ants.regress_components( gmmat, nuisance )
-  # turn data following nuisance and gsr back to image format
-  gsrbold = ants.matrix_to_timeseries(simg, gmmat, gmseg)
+  simg = ants.matrix_to_timeseries(simg, gmmat, bmask)
 
-  myfalff=alff_image( simg, bmask, flo=f[0], fhi=f[1], nuisance=nuisance )
+  # falff/alff stuff
+  myfalff=alff_image( simgimp, bmask  )
 
+  # structure the output data
   outdict = {}
   outdict['meanBold'] = und
   outdict['pts2bold'] = pts2bold
@@ -4736,7 +4915,7 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
     roiLabel = "ROI" + str(pts2bold.loc[i,'ROI']) + '_' + netLabel
     roiNames.append( roiLabel )
     ptImage = ants.make_points_image(pts2bold.iloc[[i],:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
-    meanROI[:,i] = ants.timeseries_to_matrix( gsrbold, ptImage).mean(axis=1)
+    meanROI[:,i] = ants.timeseries_to_matrix( simg, ptImage).mean(axis=1)
 
   # get full correlation matrix
   corMat = np.corrcoef(meanROI, rowvar=False)
@@ -4763,14 +4942,12 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
     dfnImg = ants.make_points_image(pts2bold.iloc[ww,:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
     if dfnImg.max() >= 1:
         dfnmat = ants.timeseries_to_matrix( simg, ants.threshold_image( dfnImg, 1, dfnImg.max() ) )
-        dfnmat = ants.bandpass_filter_matrix( dfnmat, tr = tr, lowf=f[0], highf=f[1]  )
-        dfnmat = ants.regress_components( dfnmat, nuisance )
         dfnsignal = dfnmat.mean( axis = 1 )
         gmmatDFNCorr = np.zeros( gmmat.shape[1] )
         for k in range( gmmat.shape[1] ):
             gmmatDFNCorr[ k ] = pearsonr( dfnsignal, gmmat[:,k] )[0]
-        corrImg = ants.make_image( gmseg, gmmatDFNCorr  )
-        outdict[ netname ] = corrImg
+        corrImg = ants.make_image( bmask, gmmatDFNCorr  )
+        outdict[ netname ] = corrImg * gmseg
     else:
         outdict[ netname ] = None
     ct = ct + 1
@@ -4784,7 +4961,6 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
       netnamei = re.sub( " ", "", networks[numofnets[i]] )
       netnamei = re.sub( "-", "", netnamei )
       newnames.append( netnamei  )
-      binmask = ants.threshold_image( outdict[ netnamei ], 0.2, 1.0 )
       ww = np.where( powers_areal_mni_itk['SystemName'] == networks[numofnets[i]] )[0]
       dfnImg = ants.make_points_image(pts2bold.iloc[ww,:3].values, bmask, radius=1).threshold_image( 1, 1e9 )
       for j in range( len( numofnets ) ):
@@ -4803,6 +4979,7 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
   outdict['corr'] = A
   outdict['corr_wide'] = A_wide
   outdict['brainmask'] = bmask
+  outdict['gmmask'] = gmseg
   outdict['alff'] = myfalff['alff']
   outdict['falff'] = myfalff['falff']
   # add global mean and standard deviation for post-hoc z-scoring
@@ -4829,16 +5006,17 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
   trimmask = ants.iMath( bmask, "ME",2)
   edgemask = ants.iMath( bmask, "ME",1) - trimmask
   outdict['motion_corrected'] = corrmo['motion_corrected']
-  outdict['brain_mask'] = bmask
   outdict['nuisance'] = rsfNuisance
   outdict['tsnr'] = mytsnr
   outdict['ssnr'] = slice_snr( corrmo['motion_corrected'], csfAndWM, gmseg )
   outdict['dvars'] = dvars( corrmo['motion_corrected'], gmseg )
-  outdict['high_motion_count'] = (rsfNuisance['FD'] > FD_threshold ).sum()
-  outdict['high_motion_pct'] = (rsfNuisance['FD'] > FD_threshold ).sum() / rsfNuisance.shape[0]
+  outdict['high_motion_count'] = high_motion_count
+  outdict['high_motion_pct'] = high_motion_pct
   outdict['FD_max'] = rsfNuisance['FD'].max()
   outdict['FD_mean'] = rsfNuisance['FD'].mean()
   outdict['bold_evr'] =  antspyt1w.patch_eigenvalue_ratio( und, 512, [16,16,16], evdepth = 0.9, mask = bmask )
+  outdict['n_outliers'] = len(hlinds)
+  outdict['nc'] = nc
   return outdict
 
 
@@ -5010,8 +5188,9 @@ def bold_perfusion_minimal(
   bmask = bmask * ants.iMath( tsnrmask, "FillHoles" )
   fmrimotcorr=corrmo['motion_corrected']
   und = fmri_template * bmask
+  compcorquantile = 0.975
   mycompcor = ants.compcor( fmrimotcorr,
-    ncompcor=nc, quantile=0.975, mask = bmask,
+    ncompcor=nc, quantile=compcorquantile, mask = bmask,
     filter_type='polynomial', degree=2 )
   tr = ants.get_spacing( fmrimotcorr )[3]
   simg = ants.smooth_image(fmrimotcorr, spa, sigma_in_physical_coordinates = True )
@@ -5342,8 +5521,9 @@ def bold_perfusion( fmri, t1head, t1, t1segmentation, t1dktcit,
     t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
   wmseg = ants.apply_transforms( und, wmseg,
     t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
+  compcorquantile = 0.975
   mycompcor = ants.compcor( fmrimotcorr,
-    ncompcor=nc, quantile=0.975, mask = csfAndWM,
+    ncompcor=nc, quantile=compcorquantile, mask = csfAndWM,
     filter_type='polynomial', degree=2 )
   tr = ants.get_spacing( fmrimotcorr )[3]
   simg = ants.smooth_image(fmrimotcorr, spa, sigma_in_physical_coordinates = True )
@@ -5472,7 +5652,7 @@ Where:
               {'cbf' : df_cbf},
               col_names = ['Mean'] )
   df_cbf = df_cbf.add_prefix('cbf_')
-  df_perf = pd.concat( [df_perf,df_cbf], axis=1 )
+  df_perf = pd.concat( [df_perf,df_cbf], axis=1, ignore_index=False )
   if verbose:
       print("perfusion dataframe end")
 
@@ -5496,6 +5676,7 @@ Where:
   outdict['bold_evr'] =  antspyt1w.patch_eigenvalue_ratio( und, 512, [16,16,16], evdepth = 0.9, mask = bmask )
   outdict['t1reg'] = t1reg
   outdict['outlier_volumes']=hlinds
+  outdict['n_outliers']=len(hlinds)
   outdict['negative_voxels']=negative_voxels
   return outdict
 
@@ -5739,8 +5920,10 @@ def mm(
             # build a template then join the images
             if verbose:
                 print("initial average for rsf")
-            rsfavg1=get_average_rsf(rsf_image1)
-            rsfavg2=get_average_rsf(rsf_image2)
+            rsfavg1, hlinds = loop_timeseries_censoring( rsf_image1, 0.1 )
+            rsfavg1=get_average_rsf(rsfavg1)
+            rsfavg2, hlinds = loop_timeseries_censoring( rsf_image2, 0.1 )
+            rsfavg2=get_average_rsf(rsfavg2)
             if verbose:
                 print("template average for rsf")
             init_temp = ants.image_clone( rsfavg1 )
@@ -5760,17 +5943,14 @@ def mm(
                 rsf_image = rsf_image2
         elif len( rsf_image ) == 1:
             rsf_image = rsf_image[0]
-            boldTemplate=get_average_rsf(rsf_image)
+            boldTemplate, hlinds = loop_timeseries_censoring( rsf_image, 0.1 )
+            boldTemplate = get_average_rsf(boldTemplate)
         if rsf_image.shape[3] > 10: # FIXME - better heuristic?
             output_dict['rsf'] = resting_state_fmri_networks(
                 rsf_image,
                 boldTemplate,
                 hier['brain_n4_dnz'],
-                t1atropos,
-                f=[0.03,0.08],
-                spa = 1.0,
-                spt = 0.5,
-                nc = 6, verbose=verbose )
+                t1atropos, verbose=verbose )
     if nm_image_list is not None:
         if verbose:
             print('nm')
@@ -6036,7 +6216,7 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
         fat1derk,
         mdt1derk,
         cnxderk
-        ], axis=1 )
+        ], axis=1, ignore_index=False )
     mm_wide = mm_wide.copy()
     if mm['NM'] is not None:
         mm_wide['NM_avg_signaltonoise'] = mm['NM']['NM_avg_signaltonoise']
@@ -6065,7 +6245,7 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
         mm_wide['flair_evr'] = mm['flair']['wmh_evr']
         mm_wide['flair_SNR'] = mm['flair']['wmh_SNR']
     if mm['rsf'] is not None:
-        mynets = list([ 'meanBold', 'brain_mask', 'motion_corrected', 'alff', 'falff',
+        mynets = list([ 'meanBold', 'brainmask', 'motion_corrected', 'alff', 'falff',
             'CinguloopercularTaskControl', 'DefaultMode', 'MemoryRetrieval',
             'VentralAttention', 'Visual', 'FrontoparietalTaskControl', 'Salience',
             'Subcortical', 'DorsalAttention', 'tsnr'] )
@@ -6074,7 +6254,7 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
             myop = output_prefix + separator + mykey + '.nii.gz'
             image_write_with_thumbnail( rsfpro[mykey], myop, thumb=True )
         rsfpro['corr_wide'].set_index( mm_wide.index, inplace=True )
-        mm_wide = pd.concat( [ mm_wide, rsfpro['corr_wide'] ], axis=1 )
+        mm_wide = pd.concat( [ mm_wide, rsfpro['corr_wide'] ], axis=1, ignore_index=False )
         # falff and alff
         search_key='alffPoint'
         alffkeys = [key for key, val in rsfpro.items() if search_key in key]
@@ -6084,14 +6264,17 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
         mm_wide['rsf_dvars_mean'] =  rsfpro['dvars'].mean()
         mm_wide['rsf_ssnr_mean'] =  rsfpro['ssnr'].mean()
         mm_wide['rsf_high_motion_count'] =  rsfpro['high_motion_count']
-        # mm_wide['rsf_high_motion_pct'] = rsfpro['rsf_high_motion_pct'] # BUG : rsf_high_motion_pct does not exist
+        mm_wide['rsf_high_motion_pct'] = rsfpro['high_motion_pct']
         mm_wide['rsf_evr'] =  rsfpro['bold_evr']
+        mm_wide['rsf_n_outliers'] =  rsfpro['n_outliers']
         mm_wide['rsf_FD_mean'] = rsfpro['FD_mean']
         mm_wide['rsf_FD_max'] = rsfpro['FD_max']
         mm_wide['rsf_alff_mean'] = rsfpro['alff_mean']
         mm_wide['rsf_alff_sd'] = rsfpro['alff_sd']
         mm_wide['rsf_falff_mean'] = rsfpro['falff_mean']
         mm_wide['rsf_falff_sd'] = rsfpro['falff_sd']
+        mm_wide['rsf_nc'] = rsfpro['nc']
+        mm_wide['rsf_n_outliers'] = rsfpro['n_outliers']
         ofn = output_prefix + separator + 'rsfcorr.csv'
         rsfpro['corr'].to_csv( ofn )
         # apply same principle to new correlation matrix, doesn't need to be incorporated with mm_wide
@@ -6125,11 +6308,12 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
         mm_wide['ssnr_mean'] =  perfpro['ssnr'].mean()
         mm_wide['high_motion_count'] =  perfpro['high_motion_count']
         mm_wide['evr'] =  perfpro['bold_evr']
+        mm_wide['n_outliers'] =  perfpro['n_outliers']
         mm_wide['FD_mean'] = perfpro['FD_mean']
         mm_wide['FD_max'] = perfpro['FD_max']
         if 'perf_dataframe' in perfpro.keys():
             pderk = perfpro['perf_dataframe'].iloc[: , 1:]
-            mm_wide = pd.concat( [ mm_wide, pderk ], axis=1 )
+            mm_wide = pd.concat( [ mm_wide, pderk ], axis=1, ignore_index=False )
         else:
             print("FIXME - perfusion dataframe")
         for mykey in ['perfusion','cbf']:
@@ -7584,7 +7768,7 @@ def read_mm_csv( x, is_t1=False, colprefix=None, separator='-', verbose=False ):
         return None
     if colprefix is not None:
         xdf.columns=colprefix + xdf.columns
-    return pd.concat( [df,xdf], axis=1 )
+    return pd.concat( [df,xdf], axis=1, ignore_index=False )
 
 def merge_wides_to_study_dataframe( sdf, processing_dir, separator='-', sid_is_int=True, id_is_int=True, date_is_int=True, report_missing=False,
 progress=False, verbose=False ):
@@ -7610,7 +7794,7 @@ progress=False, verbose=False ):
     verbose : boolean
     """
     from os.path import exists
-    musthavecols = ['projectID', 'subjectID','date','imageID','filename']
+    musthavecols = ['projectID', 'subjectID','date','imageID']
     for k in range(len(musthavecols)):
         if not musthavecols[k] in sdf.keys():
             raise ValueError('sdf is missing column ' +musthavecols[k] + ' in merge_wides_to_study_dataframe' )
@@ -7677,7 +7861,7 @@ progress=False, verbose=False ):
                         # mm.index=csvrow.index
                         uidname = mod_name + '_mmwide_filename'
                         mm[ uidname ] = rootid
-                        csvrow=pd.concat( [csvrow,mm], axis=1 )
+                        csvrow=pd.concat( [csvrow,mm], axis=1, ignore_index=False )
                 else:
                     if verbose and report_missing:
                         print( nrgwidefn + " absent")
@@ -7687,7 +7871,7 @@ progress=False, verbose=False ):
         else:
             csvrow=csvrow.loc[:,~csvrow.columns.duplicated()]
             alldf = alldf.loc[:,~alldf.columns.duplicated()]
-            alldf = pd.concat( [alldf, csvrow], axis=0, ignore_index=True)
+            alldf = pd.concat( [alldf, csvrow], axis=0, ignore_index=True )
     return alldf
 
 def assemble_modality_specific_dataframes( mm_wide_csvs, hierdfin, nrg_modality, separator='-', progress=None, verbose=False ):
@@ -7707,7 +7891,7 @@ def assemble_modality_specific_dataframes( mm_wide_csvs, hierdfin, nrg_modality,
             for y in fnsnm:
                 temp=read_mm_csv( y, colprefix=moddersub+'_', is_t1=False, separator=separator, verbose=verbose )
                 if temp is not None:
-                    nmdf=pd.concat( [nmdf, temp], axis=0)
+                    nmdf=pd.concat( [nmdf, temp], axis=0, ignore_index=False )
     return nmdf
 
 def bind_wide_mm_csvs( mm_wide_csvs, merge=True, separator='-', verbose = 0 ) :
@@ -7732,7 +7916,7 @@ def bind_wide_mm_csvs( mm_wide_csvs, merge=True, separator='-', verbose = 0 ) :
     for y in mm_wide_csvs:
         temp=read_mm_csv( y, colprefix='T1Hier_', separator=separator, is_t1=True )
         if temp is not None:
-            hierdf=pd.concat( [hierdf, temp], axis=0)
+            hierdf=pd.concat( [hierdf, temp], axis=0, ignore_index=False )
     if verbose > 0:
         mypro=50
     else:
@@ -7845,8 +8029,8 @@ def threaded_bind_wide_mm_csvs( mm_wide_csvs, n_workers ):
         results = []
         for future in futures.as_completed(to_do):
             res0, res1 = future.result()
-            alldf=pd.concat(  [alldf, res0 ], axis=0 )
-            alldfavg=pd.concat(  [alldfavg, res1 ], axis=0 )
+            alldf=pd.concat(  [alldf, res0 ], axis=0, ignore_index=False )
+            alldfavg=pd.concat(  [alldfavg, res1 ], axis=0, ignore_index=False )
     return alldf, alldfavg
 
 
@@ -7857,6 +8041,9 @@ def get_names_from_data_frame(x, demogIn, exclusions=None):
     antspymm.get_names_from_data_frame( ['a','e'], df )
     antspymm.get_names_from_data_frame( ['e'], df, exclusions='N' )
     """
+    # Check if x is a string and convert it to a list
+    if isinstance(x, str):
+        x = [x]
     def get_unique( qq ):
         unique = []
         for number in qq:
@@ -7935,7 +8122,7 @@ def average_mm_df( jmm_in, diagnostic_n=25, corr_thresh=0.9, verbose=False ):
                         jmm.loc[k, dt0[1:]] = nanList * len(v1)
                     if verbose:
                         print( joinDiagnosticsLoc )
-                    joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0)
+                    joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0, ignore_index=False )
 
     if verbose:
         print("do DTI")
@@ -7989,7 +8176,7 @@ def average_mm_df( jmm_in, diagnostic_n=25, corr_thresh=0.9, verbose=False ):
                         jmm.loc[k, dt0[1:]] = nanList * len( dt0[1:] )
                     if verbose:
                         print( joinDiagnosticsLoc )
-                    joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0)
+                    joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0, ignore_index=False )
 
 
     # first task - sort by u_hier_id
@@ -8048,7 +8235,8 @@ def average_mm_df( jmm_in, diagnostic_n=25, corr_thresh=0.9, verbose=False ):
                     jmmUniq.loc[u][fl_names[1:]] = temp.mean(axis=0)
                 else:
                     jmmUniq.loc[u][fl_names[1:]] = nanList * temp.shape[1]
-                joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], axis=0)
+                joinDiagnostics = pd.concat( [joinDiagnostics, joinDiagnosticsLoc], 
+                                            axis=0, ignore_index=False )
 
     return jmmUniq, jmm, joinDiagnostics
 
@@ -8452,7 +8640,7 @@ def blind_image_assessment(
             viz_filename_use = re.sub( ".png", "_slice"+str(jjj).zfill(4)+".png", viz_filename )
             ants.plot_ortho( image, crop=False, filename=viz_filename_use, flat=True, xyz_lines=False, orient_labels=False, xyz_pad=0,  title=ttl, titlefontsize=12, title_dy=-0.02,textfontcolor='red' )
         df = pd.DataFrame([[ mystem, noizlevel, snrref, cnrref, psnrref, ssimref, mymi, asym_err, myevr, msk_vol, spc[0], spc[1], spc[2],org[0], org[1], org[2], image.shape[0], image.shape[1], image.shape[2], jjj, modality, mriseries, mrimfg, mrimodel ]], columns=['filename', 'noise', 'snr', 'cnr', 'psnr', 'ssim', 'mi', 'reflection_err', 'EVR', 'msk_vol', 'spc0','spc1','spc2','org0','org1','org2','dimx','dimy','dimz','slice','modality', 'mriseries', 'mrimfg', 'mrimodel' ])
-        outdf = pd.concat( [outdf, df ], axis=0 )
+        outdf = pd.concat( [outdf, df ], axis=0, ignore_index=False )
         if verbose:
             print( outdf )
     if viz_filename is not None:
@@ -8620,6 +8808,46 @@ def wmh( flair, t1, t1seg,
         'convexhull_mask': distmask }
 
 
+def replace_elements_in_numpy_array(original_array, indices_to_replace, new_value):
+    """
+    Replace specified elements or rows in a numpy array with a new value.
+
+    Parameters:
+    original_array (numpy.ndarray): A numpy array in which elements or rows are to be replaced.
+    indices_to_replace (list or numpy.ndarray): Indices of elements or rows to be replaced.
+    new_value: The new value to replace the specified elements or rows.
+
+    Returns:
+    numpy.ndarray: A new numpy array with the specified elements or rows replaced. If the input array is None,
+                   the function returns None.
+    """
+
+    if original_array is None:
+        return None
+
+    max_index = original_array.size if original_array.ndim == 1 else original_array.shape[0]
+
+    # Filter out invalid indices and check for any out-of-bounds indices
+    valid_indices = []
+    for idx in indices_to_replace:
+        if idx < max_index:
+            valid_indices.append(idx)
+        else:
+            warnings.warn(f"Warning: Index {idx} is out of bounds and will be ignored.")
+
+    if original_array.ndim == 1:
+        # Replace elements in a 1D array
+        original_array[valid_indices] = new_value
+    elif original_array.ndim == 2:
+        # Replace rows in a 2D array
+        original_array[valid_indices, :] = new_value
+    else:
+        raise ValueError("original_array must be either 1D or 2D.")
+
+    return original_array
+
+
+
 def remove_elements_from_numpy_array(original_array, indices_to_remove):
     """
     Remove specified elements or rows from a numpy array.
@@ -8668,13 +8896,14 @@ def remove_volumes_from_timeseries(time_series, volumes_to_remove):
     return ants.copy_image_info( time_series, filtered_time_series )
 
 
-def impute_timeseries(time_series, volumes_to_impute, method='linear'):
+def impute_timeseries(time_series, volumes_to_impute, method='linear', verbose=False):
     """
     Impute specified volumes from a time series with interpolated values.
 
     :param time_series: ANTsImage representing the time series (4D image).
     :param volumes_to_impute: List of volume indices to impute.
     :param method: Interpolation method ('linear' or other methods if implemented).
+    :param verbose: boolean
     :return: ANTsImage with specified volumes imputed.
     """
     if not isinstance(time_series, ants.ANTsImage):
@@ -8685,25 +8914,36 @@ def impute_timeseries(time_series, volumes_to_impute, method='linear'):
 
     # Convert time_series to numpy for manipulation
     time_series_np = time_series.numpy()
+    total_volumes = time_series_np.shape[3]
+
+    # Create a complement list of volumes not to impute
+    volumes_not_to_impute = [i for i in range(total_volumes) if i not in volumes_to_impute]
+
+    # Define the lower and upper bounds
+    min_valid_index = min(volumes_not_to_impute)
+    max_valid_index = max(volumes_not_to_impute)
 
     for vol_idx in volumes_to_impute:
         # Ensure the volume index is within the valid range
-        if vol_idx < 0 or vol_idx >= time_series_np.shape[3]:
+        if vol_idx < 0 or vol_idx >= total_volumes:
             raise ValueError(f"Volume index {vol_idx} is out of bounds.")
 
-        # Find neighboring volumes for interpolation
-        lower_idx = max(0, vol_idx - 1)
-        upper_idx = min(time_series_np.shape[3] - 1, vol_idx + 1)
+        # Find the nearest valid lower index within the bounds
+        lower_candidates = [v for v in volumes_not_to_impute if v <= vol_idx]
+        lower_idx = max(lower_candidates) if lower_candidates else min_valid_index
+
+        # Find the nearest valid upper index within the bounds
+        upper_candidates = [v for v in volumes_not_to_impute if v >= vol_idx]
+        upper_idx = min(upper_candidates) if upper_candidates else max_valid_index
+
+        if verbose:
+            print(f"Imputing volume {vol_idx} using indices {lower_idx} and {upper_idx}")
 
         if method == 'linear':
             # Linear interpolation between the two nearest volumes
-            if lower_idx == vol_idx or upper_idx == vol_idx:
-                # Edge case: duplicate the closest volume
-                interpolated_volume = time_series_np[..., lower_idx]
-            else:
-                lower_volume = time_series_np[..., lower_idx]
-                upper_volume = time_series_np[..., upper_idx]
-                interpolated_volume = (lower_volume + upper_volume) / 2
+            lower_volume = time_series_np[..., lower_idx]
+            upper_volume = time_series_np[..., upper_idx]
+            interpolated_volume = (lower_volume + upper_volume) / 2
         else:
             # Placeholder for other interpolation methods
             raise NotImplementedError("Currently, only linear interpolation is implemented.")
@@ -8717,7 +8957,6 @@ def impute_timeseries(time_series, volumes_to_impute, method='linear'):
 
     return imputed_time_series
 
-
 def impute_dwi( dwi, threshold = 0.20, verbose=False ):
     """
     Identify bad volumes in a dwi and impute them fully automatically.
@@ -8746,6 +8985,36 @@ def impute_dwi( dwi, threshold = 0.20, verbose=False ):
         return dwi
     return impute_timeseries( dwi, complement )
 
+def scrub_dwi( dwi, bval, bvec, threshold = 0.20, verbose=False ):
+    """
+    Identify bad volumes in a dwi and impute them fully automatically.
+
+    :param dwi: ANTsImage representing the time series (4D image).
+    :param bval: bval array
+    :param bvec: bvec array
+    :param threshold: threshold (0,1) for outlierness (lower means impute more data)
+    :param verbose: boolean
+    :return: ANTsImage automatically imputed.
+    """
+    list1 = segment_timeseries_by_meanvalue( dwi )['highermeans']
+    looped, list2 = loop_timeseries_censoring( dwi, threshold )
+    if verbose:
+        print( list1 )
+        print( list2 )
+    # Convert lists to sets
+    set1 = set(list(list1))
+    set2 = set(list(list2))
+    # Find the intersection
+    intersection = set1 & set2
+    # Find the complement of the intersection
+    complement = list( (set1 | set2) - intersection )
+    if verbose:
+        print( "scrubbing:")
+        print( complement )
+    if len( complement ) == 0:
+        return dwi, bval, bvec
+    return remove_volumes_from_timeseries( dwi, complement ), remove_elements_from_numpy_array( bval, complement ), remove_elements_from_numpy_array( bvec, complement )
+
 
 def flatten_time_series(time_series):
     """
@@ -9176,6 +9445,42 @@ def brainmap_figure(statistical_df, data_dictionary_path, output_prefix, brain_i
     return addem
 
 
+def filter_df(indf, myprefix):
+    """
+    Process and filter a pandas DataFrame, removing certain columns, 
+    filtering based on data types, computing the mean of numeric columns, 
+    and adding a prefix to column names.
+
+    Parameters:
+    indf (pandas.DataFrame): The input DataFrame to be processed.
+    myprefix (str): A string prefix to be added to the column names 
+                    of the processed DataFrame.
+
+    Steps:
+    1. Removes columns with names containing 'Unnamed'.
+    2. If the DataFrame has no rows, it returns the empty DataFrame.
+    3. Filters out columns based on the type of the first element, 
+       keeping those that are of type `object`, `int`, or `float`.
+    4. Removes columns that are of `object` dtype.
+    5. Calculates the mean of the remaining columns, skipping NaN values.
+    6. Adds the specified `myprefix` to the column names.
+
+    Returns:
+    pandas.DataFrame: A transformed DataFrame with a single row containing 
+                      the mean values of the filtered columns, and with 
+                      column names prefixed as specified.
+    """
+    indf = indf.loc[:, ~indf.columns.str.contains('Unnamed*', na=False, regex=True)]
+    if indf.shape[0] == 0:
+        return indf
+    nums = [isinstance(indf[col].iloc[0], (object, int, float)) for col in indf.columns]
+    indf = indf.loc[:, nums]
+    indf = indf.loc[:, indf.dtypes != 'object']
+    indf = pd.DataFrame(indf.mean(axis=0, skipna=True)).T
+    indf = indf.add_prefix(myprefix)
+    return indf
+
+
 def aggregate_antspymm_results(input_csv, subject_col='subjectID', date_col='date', image_col='imageID', date_column='ses-1', base_path="./Processed/ANTsExpArt/", hiervariable='T1wHierarchical', valid_modalities=None, verbose=False ):
     """
     Aggregate ANTsPyMM results from the specified CSV file and save the aggregated results to a new CSV file.
@@ -9204,15 +9509,6 @@ def aggregate_antspymm_results(input_csv, subject_col='subjectID', date_col='dat
     import numpy as np
     from glob import glob
 
-    def filter_df( indf, myprefix ):
-        nums = [isinstance(indf[col].iloc[0], (int, float)) for col in indf.columns]
-        indf = indf.loc[:, nums]
-        indf=indf.loc[:, indf.dtypes != 'object' ]
-        indf = indf.loc[:, ~indf.columns.str.contains('Unnamed*', na=False, regex=True)]
-        indf = pd.DataFrame(indf.mean(axis=0, skipna=True)).T
-        indf = indf.add_prefix( myprefix )
-        return( indf )
-
     def myread_csv(x, cnms):
         """
         Reads a CSV file and returns a DataFrame excluding specified columns.
@@ -9308,13 +9604,13 @@ def aggregate_antspymm_results(input_csv, subject_col='subjectID', date_col='dat
                     t1df = filter_df( t1df, mymod+'_')
                     dflist = dflist + [t1df]
                 
-            hdf = pd.concat( dflist, axis=1)
+            hdf = pd.concat( dflist, axis=1, ignore_index=False )
             if verbose:
                 print( df.loc[locind,'filename'] )
             if myct == 1:
                 subdf = df.iloc[[x]]
                 hdf.index = subdf.index.copy()
-                df = pd.concat( [df,hdf], axis=1)
+                df = pd.concat( [df,hdf], axis=1, ignore_index=False )
             else:
                 commcols = list(set(hdf.columns).intersection(df.columns))
                 df.loc[locind, commcols] = hdf.loc[0, commcols]
@@ -9369,14 +9665,18 @@ def aggregate_antspymm_results_sdf(
     import numpy as np
     from glob import glob
 
-    def filter_df( indf, myprefix ):
-        nums = [isinstance(indf[col].iloc[0], (int, float)) for col in indf.columns]
-        indf = indf.loc[:, nums]
-        indf=indf.loc[:, indf.dtypes != 'object' ]
-        indf = indf.loc[:, ~indf.columns.str.contains('Unnamed*', na=False, regex=True)]
-        indf = pd.DataFrame(indf.mean(axis=0, skipna=True)).T
-        indf = indf.add_prefix( myprefix )
-        return( indf )
+    def progress_reporter(current_step, total_steps, width=50):
+        # Calculate the proportion of progress
+        progress = current_step / total_steps
+        # Calculate the number of 'filled' characters in the progress bar
+        filled_length = int(width * progress)
+        # Create the progress bar string
+        bar = '█' * filled_length + '-' * (width - filled_length)
+        # Print the progress bar with percentage
+        print(f'\rProgress: |{bar}| {int(100 * progress)}%', end='\r')
+        # Print a new line when the progress is complete
+        if current_step == total_steps:
+            print()
 
     def myread_csv(x, cnms):
         """
@@ -9420,8 +9720,8 @@ def aggregate_antspymm_results_sdf(
         myfn = os.path.basename( df['filename'].iloc[x] )
         temp = myfn.split( splitsep )
         # Generalized search paths
-        sid0 = temp[0]
-        sid = str(df[subject_col].iloc[x])
+        sid0 = str( temp[0] )
+        sid = str( df[subject_col].iloc[x] )
         if sid0 != sid:
             warnings.warn("OUTER: the id derived from the filename " + sid + " does not match the id stored in the data frame " + sid )
             warnings.warn( "filename is : " +  myfn )
@@ -9449,18 +9749,20 @@ def aggregate_antspymm_results_sdf(
     dfout = pd.DataFrame()
     myct = 0
     for x in range( df.shape[0]):
-        print("\n\n-------------------------------------------------")
         if verbose:
+            print("\n\n-------------------------------------------------")
             print(f"{x}...")
+        else:
+            progress_reporter(x, df.shape[0], width=500)
         locind = df.index[x]
         myfn = os.path.basename( df['filename'].iloc[x] )
-        sid = df[subject_col].iloc[x]
+        sid = str( df[subject_col].iloc[x] )
         tempB = myfn.split( splitsep )
-        sid0 = tempB[1]
-        if sid0 != sid:
-            warnings.warn("INNER: the id derived from the filename " + sid + " does not match the id stored in the data frame " + sid0 )
-            warnings.warn( "filename is : " +  myfn )
-            warnings.warn( "sid is : " + sid )
+        sid0 = str(tempB[1])
+        if sid0 != sid and verbose:
+            warnings.warn("INNER: the id derived from the filename " + str(sid) + " does not match the id stored in the data frame " + str(sid0) )
+            warnings.warn( "filename is : " +  str(myfn) )
+            warnings.warn( "sid is : " + str(sid) )
             warnings.warn( "x is : " + str(x) )
             warnings.warn( "index is : " + str(locind) )
         myproj = str(df[project_col].iloc[x])
@@ -9495,7 +9797,8 @@ def aggregate_antspymm_results_sdf(
             dflist = [hdf]
 
             for mymod in vmoddict.keys():
-                print("\n\n************************* " + mymod + " *************************")
+                if verbose:
+                    print("\n\n************************* " + mymod + " *************************")
                 modalityclass = vmoddict[ mymod ]
                 if wild_card_modality_id:
                     mymodid = '*'
@@ -9508,7 +9811,8 @@ def aggregate_antspymm_results_sdf(
                         temp = mymodid.split( idsep )
                         mymodid = temp[ len( temp )-1 ]
                     else:
-                        print("missing")
+                        if verbose:
+                            print("missing")
                         continue
                 if verbose:
                     print( "modality id is " + mymodid + " for modality " + modalityclass + ' modality specific subj ' + sid + ' modality specific id is ' + myid + " its date " +  mydate )
@@ -9521,7 +9825,10 @@ def aggregate_antspymm_results_sdf(
                     print( modsearch )
                 t1wfn = sorted( glob( modsearch ) )
                 if len( t1wfn ) > 1:
-                    raise ValueError("there are " + str( len( t1wfn ) ) + " number of wide fns with search path " + modsearch )
+                    nlarge = len(t1wfn)
+                    t1wfn = [ t1wfn[ len(t1wfn)-1 ] ]
+                    warnings.warn("there are " + str( nlarge ) + " number of wide fns with search path " + modsearch + " we take the last of these " + t1wfn[0] )
+#                    raise ValueError("there are " + str( len( t1wfn ) ) + " number of wide fns with search path " + modsearch )
                 if len( t1wfn ) == 1:
                     if verbose:
                         print(t1wfn)
@@ -9532,13 +9839,13 @@ def aggregate_antspymm_results_sdf(
                     if verbose:
                         print( " cannot find " + modsearch )
                 
-            hdf = pd.concat( dflist, axis=1)
+            hdf = pd.concat( dflist, axis=1, ignore_index=False)
             if verbose:
                 print( "count: " + str( myct ) )
             subdf = df.iloc[[x]]
             hdf.index = subdf.index.copy()
-            subdf = pd.concat( [subdf,hdf], axis=1)
-            dfout = pd.concat( [dfout,subdf], axis=0)
+            subdf = pd.concat( [subdf,hdf], axis=1, ignore_index=False)
+            dfout = pd.concat( [dfout,subdf], axis=0, ignore_index=False )
     badnames = get_names_from_data_frame( ['Unnamed'], dfout )
     dfout=dfout.drop(badnames, axis=1)
     return( dfout )

{antspymm-1.1.7.dist-info → antspymm-1.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: antspymm
-Version: 1.1.7
+Version: 1.1.9
 Summary: multi-channel/time-series medical image processing with antspyx
 Home-page: https://github.com/stnava/ANTsPyMM
 Author: Avants, Gosselin, Tustison, Reardon
@@ -337,8 +337,8 @@ for each subject/timepoint, one would run:
 studyfn="matched_mm_data2.csv"
 df=pd.read_csv( studyfn )
 index = 20 # 20th subject/timepoint
-csvfns = df['fn']
-csvrow = df[ df['fn'] == csvfns[index] ]
+csvfns = df['filename']
+csvrow = df[ df['filename'] == csvfns[index] ]
 csvrow['projectID']='MyStudy'
 
 ############################################################################################

antspymm-1.1.9.dist-info/RECORD

@@ -0,0 +1,7 @@
+antspymm/__init__.py,sha256=xQWo4DJQ0uK0_7G0kjgrj7SZoar5oB1FCV6jeTpTDcc,3805
+antspymm/mm.py,sha256=Z1tcO04yl3CHmCbQpDdZocllIZbOw6wRGM-sKEcXcIQ,405265
+antspymm-1.1.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+antspymm-1.1.9.dist-info/METADATA,sha256=FgaDoUndsazM2rNGB5ZG-s3x9pt2e5TFnMDWumUDoT8,14162
+antspymm-1.1.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+antspymm-1.1.9.dist-info/top_level.txt,sha256=iyD1sRhCKzfwKRJLq5ZUeV9xsv1cGQl8Ejp6QwXM1Zg,9
+antspymm-1.1.9.dist-info/RECORD,,

{antspymm-1.1.7.dist-info → antspymm-1.1.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

antspymm-1.1.7.dist-info/RECORD

@@ -1,7 +0,0 @@
-antspymm/__init__.py,sha256=DaLNk25DbKwtBnX_IKBnSK99tHCeU17M-OYUTfxR3pw,3656
-antspymm/mm.py,sha256=hQIzKhltVHICtS9z9iYjE0rMjQO-zxXm0pHihLjkNV8,392560
-antspymm-1.1.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-antspymm-1.1.7.dist-info/METADATA,sha256=uK0Aigxxr7MWmsgs58OCa2CH9ykZvrY0RosKdIE1u8c,14150
-antspymm-1.1.7.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-antspymm-1.1.7.dist-info/top_level.txt,sha256=iyD1sRhCKzfwKRJLq5ZUeV9xsv1cGQl8Ejp6QwXM1Zg,9
-antspymm-1.1.7.dist-info/RECORD,,