antspymm 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

antspymm/__init__.py

@@ -56,6 +56,8 @@ from .mm import average_mm_df
 from .mm import get_names_from_data_frame
 from .mm import read_mm_csv
 from .mm import assemble_modality_specific_dataframes
+from .mm import aggregate_antspymm_results
+from .mm import aggregate_antspymm_results_sdf
 from .mm import mc_denoise
 from .mm import mc_reg
 from .mm import dti_reg
@@ -90,3 +92,7 @@ from .mm import collect_blind_qc_by_modality
 from .mm import get_valid_modalities
 from .mm import study_dataframe_from_matched_dataframe
 from .mm import merge_wides_to_study_dataframe
+from .mm import map_scalar_to_labels
+from .mm import template_figure_with_overlay
+from .mm import brainmap_figure
+from .mm import bold_perfusion

antspymm/mm.py

@@ -79,6 +79,8 @@ __all__ = ['version',
     'novelty_detection_loop',
     'novelty_detection_quantile',
     'generate_mm_dataframe',
+    'aggregate_antspymm_results',
+    'aggregate_antspymm_results_sdf',
     'study_dataframe_from_matched_dataframe',
     'merge_wides_to_study_dataframe',
     'wmh']
@@ -159,11 +161,11 @@ def get_valid_modalities( long=False, asString=False, qc=False ):
     asString - concat list to string
     """
     if long:
-        mymod = ["T1w", "NM2DMT", "rsfMRI", "rsfMRI_LR", "rsfMRI_RL", "DTI", "DTI_LR","DTI_RL","T2Flair", "dwi", "func" ]
+        mymod = ["T1w", "NM2DMT", "rsfMRI", "rsfMRI_LR", "rsfMRI_RL", "DTI", "DTI_LR","DTI_RL","T2Flair", "dwi", "func", "perf" ]
     elif qc:
-        mymod = [ 'T1w', 'T2Flair', 'NM2DMT','DTIdwi','DTIb0', 'rsfMRI']
+        mymod = [ 'T1w', 'T2Flair', 'NM2DMT','DTIdwi','DTIb0', 'rsfMRI', "perf" ]
     else:
-        mymod = ["T1w", "NM2DMT", "DTI","T2Flair", "rsfMRI" ]
+        mymod = ["T1w", "NM2DMT", "DTI","T2Flair", "rsfMRI", "perf"  ]
     if not asString:
         return mymod
     else:
@@ -184,7 +186,8 @@ def generate_mm_dataframe(
         flair_filename=[],
         rsf_filenames=[],
         dti_filenames=[],
-        nm_filenames=[]
+        nm_filenames=[],
+        perf_filename=[]
 ):
     from os.path import exists
     valid_modalities = get_valid_modalities()
@@ -219,6 +222,17 @@ def generate_mm_dataframe(
                 flair_filename=flair_filename[0]
     if flair_filename is not None and not "lair" in flair_filename:
             raise ValueError("flair is not flair filename " + flair_filename)
+    ## perfusion
+    if perf_filename is not None:
+        if isinstance(perf_filename,list):
+            if (len(perf_filename) == 0):
+                perf_filename=None
+            else:
+                print("Take first entry from perf_filename list")
+                perf_filename=perf_filename[0]
+    if perf_filename is not None and not "perf" in perf_filename:
+            raise ValueError("perf_filename is not perf filename " + perf_filename)
+    
     for k in nm_filenames:
         if k is not None:
             if not "NM" in k:
@@ -231,7 +245,10 @@ def generate_mm_dataframe(
         if k is not None:
             if not "fMRI" in k and not "func" in k:
                 raise ValueError("rsfMRI/func is not rsfmri filename " + k)
-    allfns = [t1_filename] + [flair_filename] + nm_filenames + dti_filenames + rsf_filenames
+    if perf_filename is not None:
+        if not "perf" in perf_filename:
+                raise ValueError("perf_filename is not a valid perfusion (perf) filename " + k)
+    allfns = [t1_filename] + [flair_filename] + nm_filenames + dti_filenames + rsf_filenames + [perf_filename]
     for k in allfns:
         if k is not None:
             if not isinstance(k, str):
@@ -247,7 +264,8 @@ def generate_mm_dataframe(
         source_image_directory,
         output_image_directory,
         t1_filename,
-        flair_filename]
+        flair_filename, 
+        perf_filename]
     mydata0 = coredata +  rsf_filenames + dti_filenames
     mydata = mydata0 + nm_filenames
     corecols = [
@@ -259,7 +277,8 @@ def generate_mm_dataframe(
         'sourcedir',
         'outputdir',
         'filename',
-        'flairid']
+        'flairid',
+        'perfid']
     mycols0 = corecols + [
         'rsfid1', 'rsfid2',
         'dtid1', 'dtid2']
@@ -268,10 +287,6 @@ def generate_mm_dataframe(
         'nmid6', 'nmid7','nmid8', 'nmid9', 'nmid10', 'nmid11'
     ]
     mycols = mycols0 + nmext
-    print(len(mydata0))
-    print(len(nm_filenames))
-    print(len(mycols0))
-    print(len(nmext))
     studycsv = pd.DataFrame([ mydata ],
         columns=mycols)
     return studycsv
@@ -343,6 +358,10 @@ def nrg_2_bids( nrg_filename ):
         bids_modality_folder = 'func'
         bids_modality_filename = 'func'
 
+    if nrg_modality == 'perf'  :
+        bids_modality_folder = 'perf'
+        bids_modality_filename = 'perf'
+
     bids_suffix = nrg_suffix[1:]
     bids_filename = f'{bids_subject}_{bids_session}_{bids_modality_filename}.{bids_suffix}'
 
@@ -385,6 +404,9 @@ def bids_2_nrg( bids_filename, project_name, date, nrg_modality=None ):
     if bids_modality == 'func' and nrg_modality is None  :
         nrg_modality = 'rsfMRI'
 
+    if bids_modality == 'perf' and nrg_modality is None  :
+        nrg_modality = 'perf'
+
     nrg_suffix = bids_suffix[1:]
     nrg_filename = f'{project_name}-{nrg_subject_id}-{date}-{nrg_modality}-{nrg_image_id}.{nrg_suffix}'
 
@@ -1833,7 +1855,128 @@ def mc_reg(
         "FD": FD,
     }
 
-def get_data( name=None, force_download=False, version=14, target_extension='.csv' ):
+def map_scalar_to_labels(dataframe, label_image_template):
+    """
+    Map scalar values from a DataFrame to associated integer image labels.
+
+    Parameters:
+    - dataframe (pd.DataFrame): A Pandas DataFrame containing a label column and scalar_value column.
+    - label_image_template (ants.ANTsImage): ANTs image with (at least some of) the same values as labels.
+
+    Returns:
+    - ants.ANTsImage: A label image with scalar values mapped to associated integer labels.
+    """
+
+    # Create an empty label image with the same geometry as the template
+    mapped_label_image = label_image_template.clone() * 0.0
+
+    # Loop through DataFrame and map scalar values to labels
+    for index, row in dataframe.iterrows():
+        label = int(row['label'])  # Assuming the DataFrame has a 'label' column
+        scalar_value = row['scalar_value']  # Replace with your column name
+        mapped_label_image[label_image_template == label] = scalar_value
+
+    return mapped_label_image
+
+
+def template_figure_with_overlay(scalar_label_df, prefix, outputfilename=None, template='cit168', xyz=None, mask_dilation=25, padding=12, verbose=True):
+    """
+    Process and visualize images with mapped scalar values.
+
+    Parameters:
+    - scalar_label_df (pd.DataFrame): A Pandas DataFrame containing scalar values and labels.
+    - prefix (str): The prefix for input image files.
+    - template (str, optional): Template for selecting image data (default is 'cit168').
+    - xyz (str, optional): The integer index of the slices to display.
+    - mask_dilation (int, optional): Dilation factor for creating a mask (default is 25).
+    - padding (int, optional): Padding value for the mapped images (default is 12).
+    - verbose (bool, optional): Enable verbose mode for printing (default is True).
+
+    Example Usage:
+    >>> scalar_label_df = pd.DataFrame({'label': [1, 2, 3], 'scalar_value': [0.5, 0.8, 1.2]})
+    >>> prefix = '../PPMI_template0_'
+    >>> process_and_visualize_images(scalar_label_df, prefix, template='cit168', xyz=None, mask_dilation=25, padding=12, verbose=True)
+    """
+
+    # Template image paths
+    template_paths = {
+        'cit168': 'cit168lab.nii.gz',
+        'bf': 'bf.nii.gz',
+        'cerebellum': 'cerebellum.nii.gz',
+        'mtl': 'mtl.nii.gz',
+        'ctx': 'dkt_cortex.nii.gz',
+        'jhuwm': 'JHU_wm.nii.gz'
+    }
+
+    if template not in template_paths:
+        print( "Valid options:")
+        print( template_paths )
+        raise ValueError(f"Template option '{template}' does not exist.")
+
+    template_image_path = template_paths[template]
+    template_image = ants.image_read(f'{prefix}{template_image_path}')
+
+    # Load image data
+    edgeimg = ants.image_read(f'{prefix}edge.nii.gz')
+    dktimg = ants.image_read(f'{prefix}dkt_parcellation.nii.gz')
+    segimg = ants.image_read(f'{prefix}tissue_segmentation.nii.gz')
+    ttl = ''
+
+    # Load and process the template image
+    ventricles = ants.threshold_image(dktimg, 4, 4) + ants.threshold_image(dktimg, 43, 43)
+    seggm = ants.mask_image(segimg, segimg, [2, 4], binarize=False)
+    edgeimg = edgeimg.clone()
+    edgeimg[edgeimg == 0] = ventricles[edgeimg == 0]
+    segwm = ants.threshold_image(segimg, 3, 4).morphology("open", 1)
+
+    # Define cropping mask
+    cmask = ants.threshold_image(template_image, 1, 1.e9).iMath("MD", mask_dilation)
+
+    mapped_image = map_scalar_to_labels(scalar_label_df, template_image)
+    tcrop = ants.crop_image(template_image, cmask)
+    toviz = ants.crop_image(mapped_image, cmask)
+    seggm = ants.crop_image(edgeimg, cmask)
+       
+    # Map scalar values to labels and visualize
+    toviz = ants.pad_image(toviz, pad_width=(padding, padding, padding))
+    seggm = ants.pad_image(seggm, pad_width=(padding, padding, padding))
+    tcrop = ants.pad_image(tcrop, pad_width=(padding, padding, padding))
+
+    if xyz is None:
+        if template == 'cit168':
+            xyz=[140, 89, 94]
+        elif template == 'bf':
+            xyz=[114,92,76]
+        elif template == 'cerebellum':
+            xyz=[169, 128, 137]
+        elif template == 'mtl':
+            xyz=[154, 112, 113]
+        elif template == 'ctx':
+            xyz=[233, 190, 174]
+        elif template == 'jhuwm':
+            xyz=[146, 133, 182]
+
+    if verbose:
+        print("plot xyz for " + template )
+        print( xyz )
+        
+    if outputfilename is None:
+        temp = ants.plot_ortho( seggm, overlay=toviz, crop=False,
+                        xyz=xyz, cbar_length=0.2, cbar_vertical=False,
+                        flat=True, xyz_lines=False, resample=False, orient_labels=False,
+                        title=ttl, titlefontsize=12, title_dy=-0.02, textfontcolor='red', 
+                        cbar=True, allow_xyz_change=False)
+    else:
+        temp = ants.plot_ortho( seggm, overlay=toviz, crop=False,
+                    xyz=xyz, cbar_length=0.2, cbar_vertical=False,
+                    flat=True, xyz_lines=False, resample=False, orient_labels=False,
+                    title=ttl, titlefontsize=12, title_dy=-0.02, textfontcolor='red', 
+                    cbar=True, allow_xyz_change=False, filename=outputfilename )
+    seggm = temp['image']
+    toviz = temp['overlay']
+    return { "underlay": seggm, 'overlay': toviz, 'seg': tcrop  }
+
+def get_data( name=None, force_download=False, version=19, target_extension='.csv' ):
     """
     Get ANTsPyMM data filename
 
@@ -2187,7 +2330,9 @@ def get_average_rsf( x, min_t=10, max_t=35 ):
         bavg = bavg + ants.registration(oavg,b0,'Rigid',outprefix=ofn)['warpedmovout']
     import shutil
     shutil.rmtree(output_directory, ignore_errors=True )
-    return ants.n4_bias_field_correction(bavg)
+    bavg = ants.iMath( bavg, 'Normalize' )
+    return bavg
+    # return ants.n4_bias_field_correction(bavg, mask=ants.get_mask( bavg ) )
 
 
 def get_average_dwi_b0( x, fixed_b0=None, fixed_dwi=None, fast=False ):
@@ -4214,7 +4359,6 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
     reg_iterations=[40,20,5] )
   if verbose:
       print("End rsfmri motion correction")
-      # ants.image_write( corrmo['motion_corrected'], '/tmp/temp.nii.gz' )
 
   mytsnr = tsnr( corrmo['motion_corrected'], bmask )
   mytsnrThresh = np.quantile( mytsnr.numpy(), 0.995 )
@@ -4224,8 +4368,6 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
   t1reg = ants.registration( und, t1, "SyNBold" )
   if verbose:
       print("t1 2 bold done")
-#      ants.image_write( und, '/tmp/template_bold_masked.nii.gz' )
-#      ants.image_write( t1reg['warpedmovout'], '/tmp/t1tobold.nii.gz' )
   boldseg = ants.apply_transforms( und, t1segmentation,
     t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
   gmseg = ants.threshold_image( t1segmentation, 2, 2 )
@@ -4403,6 +4545,216 @@ def resting_state_fmri_networks( fmri, fmri_template, t1, t1segmentation,
   return outdict
 
 
+def bold_perfusion( fmri, fmri_template, t1head, t1, t1segmentation, t1dktcit, f=[0.0,math.inf], FD_threshold=0.5, spa = 1.5, nc = 6, type_of_transform='Rigid', tc='alternating', deepmask=False, add_FD_to_nuisance=False, verbose=False ):
+  """
+  Estimate perfusion from a BOLD time series image.  Will attempt to figure out the T-C labels from the data.
+
+  Arguments
+  ---------
+  fmri : BOLD fmri antsImage
+
+  fmri_template : reference space for BOLD
+
+  t1head : ANTsImage
+    input 3-D T1 brain image (not brain extracted)
+
+  t1 : ANTsImage
+    input 3-D T1 brain image (brain extracted)
+
+  t1segmentation : ANTsImage
+    t1 segmentation - a six tissue segmentation image in T1 space
+
+  t1dktcit : ANTsImage
+    t1 dkt cortex plus cit parcellation
+
+  f : band pass limits for frequency filtering
+
+  spa : gaussian smoothing for spatial and temporal component e.g. (1,1,1,0) in physical space coordinates
+
+  nc  : number of components for compcor filtering
+
+  type_of_transform : SyN or Rigid
+
+  tc: string either alternating or split (default is alternating ie CTCTCT; split is CCCCTTTT)
+
+  deepmask: boolean
+
+  add_FD_to_nuisance: boolean
+
+  verbose : boolean
+
+  Returns
+  ---------
+  a dictionary containing the derived network maps
+
+  """
+  import numpy as np
+  import pandas as pd
+  import re
+  import math
+  from sklearn.linear_model import LinearRegression
+
+  ex_path = os.path.expanduser( "~/.antspyt1w/" )
+  cnxcsvfn = ex_path + "dkt_cortex_cit_deep_brain.csv"
+
+  def replicate_list(user_list, target_size):
+    # Calculate the number of times the list should be replicated
+    replication_factor = target_size // len(user_list)
+    # Replicate the list and handle any remaining elements
+    replicated_list = user_list * replication_factor
+    remaining_elements = target_size % len(user_list)
+    replicated_list += user_list[:remaining_elements]
+    return replicated_list
+
+  def one_hot_encode(char_list):
+    unique_chars = list(set(char_list))
+    encoding_dict = {char: [1 if char == c else 0 for c in unique_chars] for char in unique_chars}
+    encoded_matrix = np.array([encoding_dict[char] for char in char_list])
+    return encoded_matrix
+  
+  A = np.zeros((1,1))
+  fmri = ants.iMath( fmri, 'Normalize' )
+  if deepmask:
+      bmask = antspynet.brain_extraction( fmri_template, 'bold' ).threshold_image(0.5,1).morphology("close",2).iMath("FillHoles")
+  else:
+      rig = ants.registration( fmri_template, t1head, 'BOLDRigid' )
+      bmask = ants.apply_transforms( fmri_template, ants.threshold_image(t1segmentation,1,6), rig['fwdtransforms'][0], interpolator='genericLabel' )
+  if verbose:
+      print("Begin perfusion motion correction")
+  mytrim=4 # trim will guarantee an even length
+  if fmri.shape[3] % 2 == 1:
+      mytrim = 5
+  corrmo = timeseries_reg(
+    fmri, fmri_template,
+    type_of_transform=type_of_transform,
+    total_sigma=0.0,
+    fdOffset=2.0,
+    trim = mytrim,
+    output_directory=None,
+    verbose=verbose,
+    syn_metric='cc',
+    syn_sampling=2,
+    reg_iterations=[40,20,5] )
+  if verbose:
+      print("End rsfmri motion correction")
+
+  ntp = corrmo['motion_corrected'].shape[3]
+  if tc == 'alternating':
+      tclist = replicate_list( ['C','T'], ntp )
+  else:
+      tclist = replicate_list( ['C'], int(ntp/2) ) + replicate_list( ['T'],  int(ntp/2) )
+
+  tclist = one_hot_encode( tclist )
+  mytsnr = tsnr( corrmo['motion_corrected'], bmask )
+  mytsnrThresh = np.quantile( mytsnr.numpy(), 0.995 )
+  tsnrmask = ants.threshold_image( mytsnr, 0, mytsnrThresh ).morphology("close",2)
+  bmask = bmask * ants.iMath( tsnrmask, "FillHoles" )
+  und = fmri_template * bmask
+  t1reg = ants.registration( und, t1, "SyNBold" )
+  if verbose:
+      print("t1 2 bold done")
+  boldseg = ants.apply_transforms( und, t1segmentation,
+    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
+  dktseg = ants.apply_transforms( und, t1dktcit,
+    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
+  gmseg = ants.threshold_image( t1segmentation, 2, 2 )
+  gmseg = gmseg + ants.threshold_image( t1segmentation, 4, 4 )
+  gmseg = ants.threshold_image( gmseg, 1, 4 )
+  gmseg = ants.iMath( gmseg, 'MD', 1 )
+  gmseg = ants.apply_transforms( und, gmseg,
+    t1reg['fwdtransforms'], interpolator = 'genericLabel' ) * bmask
+  csfAndWM = ( ants.threshold_image( t1segmentation, 1, 1 ) +
+               ants.threshold_image( t1segmentation, 3, 3 ) ).morphology("erode",1)
+  csfAndWM = ants.apply_transforms( und, csfAndWM,
+    t1reg['fwdtransforms'], interpolator = 'nearestNeighbor' )  * bmask
+
+  mycompcor = ants.compcor( corrmo['motion_corrected'],
+    ncompcor=nc, quantile=0.90, mask = csfAndWM,
+    filter_type='polynomial', degree=2 )
+
+  nt = corrmo['motion_corrected'].shape[3]
+  tr = ants.get_spacing( corrmo['motion_corrected'] )[3]
+  highMotionTimes = np.where( corrmo['FD'] >= 1.0 )
+  goodtimes = np.where( corrmo['FD'] < 0.5 )
+  simg = ants.smooth_image(corrmo['motion_corrected'], 
+                           spa, sigma_in_physical_coordinates = True )
+
+  nuisance = mycompcor[ 'components' ]
+  nuisance = np.c_[ nuisance, mycompcor['basis'] ]
+  if add_FD_to_nuisance:
+      nuisance = np.c_[ nuisance, corrmo['FD'] ]
+
+  if verbose:
+    print("make sure nuisance is independent of TC")
+  nuisance = ants.regress_components( nuisance, tclist )
+
+  regression_mask = bmask.clone()
+  gmmat = ants.timeseries_to_matrix( simg, regression_mask )
+  if f[0] > 0 and f[1] < 1: # some would argue against this
+      gmmat = ants.bandpass_filter_matrix( gmmat, tr = tr, lowf=f[0], highf=f[1] ) 
+  # gmmat = ants.regress_components( gmmat, nuisance )
+  # Perform linear regression to estimate perfusion
+  regression_model = LinearRegression()
+  regvars = np.hstack( (nuisance, tclist ))
+  coefind = regvars.shape[1]-1
+  regvars = regvars[:,range(coefind)]
+  regression_model.fit( regvars, gmmat )
+  coefind = regression_model.coef_.shape[1]-1
+  perfimg = ants.make_image( regression_mask, regression_model.coef_[:,coefind] )
+  meangmval = ( perfimg[ gmseg == 1 ] ).mean()
+  if meangmval < 0:
+      perfimg = perfimg * (-1.0)
+      meangmval = ( perfimg[ gmseg == 1 ] ).mean()
+  if verbose:
+    print("Coefficients:", regression_model.coef_)
+    print("Coef mean", regression_model.coef_.mean(axis=0)) 
+    print( regression_model.coef_.shape )
+    print( perfimg.max() )
+  gsrbold = ants.matrix_to_timeseries(simg, gmmat, regression_mask)
+  outdict = {}
+  outdict['meanBold'] = und
+  outdict['brainmask'] = bmask
+  rsfNuisance = pd.DataFrame( nuisance )
+  rsfNuisance['FD']=corrmo['FD']
+
+
+  nonbrainmask = ants.iMath( bmask, "MD",2) - bmask
+  trimmask = ants.iMath( bmask, "ME",2)
+  edgemask = ants.iMath( bmask, "ME",1) - trimmask
+
+  if verbose:
+      print("perfusion dataframe begin")
+  df_perf = antspyt1w.map_intensity_to_dataframe(
+        'dkt_cortex_cit_deep_brain',
+        perfimg,
+        dktseg)
+  df_perf = antspyt1w.merge_hierarchical_csvs_to_wide_format(
+              {'perf' : df_perf},
+              col_names = ['Mean'] )
+  if verbose:
+      print("perfusion dataframe end")
+      print( df_perf )
+
+  outdict['perfusion']=perfimg
+  outdict['perfusion_gm_mean']=meangmval
+  outdict['perf_dataframe']=df_perf
+  outdict['motion_corrected'] = corrmo['motion_corrected']
+  outdict['gmseg'] = gmseg
+  outdict['gsrbold'] = gsrbold
+  outdict['brain_mask'] = bmask
+  outdict['nuisance'] = rsfNuisance
+  outdict['tsnr'] = mytsnr
+  outdict['ssnr'] = slice_snr( corrmo['motion_corrected'], csfAndWM, gmseg )
+  outdict['dvars'] = dvars( corrmo['motion_corrected'], gmseg )
+  outdict['high_motion_count'] = (rsfNuisance['FD'] > FD_threshold ).sum()
+  outdict['high_motion_pct'] = (rsfNuisance['FD'] > FD_threshold ).sum() / rsfNuisance.shape[0]
+  outdict['FD_max'] = rsfNuisance['FD'].max()
+  outdict['FD_mean'] = rsfNuisance['FD'].mean()
+  outdict['bold_evr'] =  antspyt1w.patch_eigenvalue_ratio( und, 512, [16,16,16], evdepth = 0.9, mask = bmask )
+  outdict['t1reg'] = t1reg
+  return outdict
+
+
 
 def write_bvals_bvecs(bvals, bvecs, prefix ):
     ''' Write FSL FDT bvals and bvecs files
@@ -4474,6 +4826,7 @@ def mm(
     flair_image=None,
     nm_image_list=None,
     dw_image=[], bvals=[], bvecs=[],
+    perfusion_image=None,
     srmodel=None,
     do_tractography = False,
     do_kk = False,
@@ -4510,6 +4863,8 @@ def mm(
 
     bvecs : list of bvecs file names
 
+    perfusion_image : single perfusion image
+
     srmodel : optional srmodel
 
     do_tractography : boolean
@@ -4578,7 +4933,8 @@ def mm(
         'FA_summ' : None,
         'MD_summ' : None,
         'tractography' : None,
-        'tractography_connectivity' : None
+        'tractography_connectivity' : None,
+        'perf' : None,
     }
     normalization_dict = {
         'kk_norm': None,
@@ -4586,6 +4942,7 @@ def mm(
         'DTI_norm': None,
         'FA_norm' : None,
         'MD_norm' : None,
+        'perf_norm' : None,
         'alff_norm' : None,
         'falff_norm' : None,
         'CinguloopercularTaskControl_norm' : None,
@@ -4606,6 +4963,19 @@ def mm(
             print('kk')
         output_dict['kk'] = antspyt1w.kelly_kapowski_thickness( hier['brain_n4_dnz'],
             labels=hier['dkt_parc']['dkt_cortex'], iterations=45 )
+    if  perfusion_image is not None:
+        boldTemplate=get_average_rsf(perfusion_image)
+        if perfusion_image.shape[3] > 8: # FIXME - better heuristic?
+            output_dict['perf'] = bold_perfusion(
+                perfusion_image,
+                boldTemplate,
+                t1_image,
+                hier['brain_n4_dnz'],
+                t1atropos,
+                hier['dkt_parc']['dkt_cortex'] + hier['cit168lab'],
+                f=[0.0,math.inf],
+                spa = (1.0, 1.0, 1.0, 0.0),
+                nc = 6, verbose=verbose )    
     ################################## do the rsf .....
     if len(rsf_image) > 0:
         rsf_image = [i for i in rsf_image if i is not None]
@@ -4786,7 +5156,7 @@ def mm(
         if verbose:
             print('normalization')
         # might reconsider this template space - cropped and/or higher res?
-        template = ants.resample_image( template, [1,1,1], use_voxels=False )
+        # template = ants.resample_image( template, [1,1,1], use_voxels=False )
         # t1reg = ants.registration( template, hier['brain_n4_dnz'], "antsRegistrationSyNQuickRepro[s]")
         t1reg = do_normalization
         if do_kk:
@@ -4812,6 +5182,11 @@ def mm(
                 normalization_dict[rsfkey] = ants.apply_transforms(
                     group_template, rsfpro[netid],
                     group_transform+rsfrig['fwdtransforms'] )
+        if output_dict['perf'] is not None: # zizzer
+            comptx = group_transform + output_dict['perf']['t1reg']['invtransforms']
+            normalization_dict['perf_norm'] = ants.apply_transforms( group_template,
+                output_dict['perf']['perfusion'], comptx,
+                whichtoinvert=[False,False,True,False] )
         if nm_image_list is not None:
             nmpro = output_dict['NM']
             nmrig = nmpro['t1_to_NM_transform'] # this is an inverse tx
@@ -4872,6 +5247,7 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
             image_write_with_thumbnail( mm['NM'][mykey], tempfn, thumb=False )
 
     faderk = mdderk = fat1derk = mdt1derk = None
+
     if mm['DTI'] is not None:
         mydti = mm['DTI']
         myop = output_prefix + separator
@@ -4984,6 +5360,26 @@ def write_mm( output_prefix, mm, mm_norm=None, t1wide=None, separator='_', verbo
             # mm_wide.to_csv( fdfn )
         else:
             mm_wide['dti_FD_mean'] = mm_wide['dti_FD_max'] = 'NA'
+
+    if mm['perf'] is not None:
+        perfpro = mm['perf']
+        mm_wide['gm_mean'] =  perfpro['perfusion_gm_mean']
+        mm_wide['tsnr_mean'] =  perfpro['tsnr'].mean()
+        mm_wide['dvars_mean'] =  perfpro['dvars'].mean()
+        mm_wide['ssnr_mean'] =  perfpro['ssnr'].mean()
+        mm_wide['high_motion_count'] =  perfpro['high_motion_count']
+        mm_wide['evr'] =  perfpro['bold_evr']
+        mm_wide['FD_mean'] = perfpro['FD_mean']
+        mm_wide['FD_max'] = perfpro['FD_max']
+        if 'perf_dataframe' in perfpro.keys():
+            pderk = perfpro['perf_dataframe'].iloc[: , 1:]
+            mm_wide = pd.concat( [ mm_wide, pderk ], axis=1 )
+        else:
+            print("FIXME - perfusion dataframe")
+        mykey='perfusion'
+        tempfn = output_prefix + separator + mykey + '.nii.gz'
+        image_write_with_thumbnail( mm['perf'][mykey], tempfn )
+
     mmwidefn = output_prefix + separator + 'mmwide.csv'
     mm_wide.to_csv( mmwidefn )
     if verbose:
@@ -5490,7 +5886,7 @@ def mm_nrg(
                                 write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep, verbose=True )
                                 for mykey in normPro.keys():
                                     if normPro[mykey] is not None:
-                                        if visualize and normPro[mykey].components == 1:
+                                        if visualize and normPro[mykey].components == 1 and False:
                                             ants.plot( template, normPro[mykey], axis=2, nslices=21, ncol=7, crop=True, title=mykey, filename=mymm+mysep+mykey+".png"   )
         if overmodX == nrg_modality_list[ len( nrg_modality_list ) - 1 ]:
             return
@@ -5639,6 +6035,8 @@ def mm_csv(
             imfns=['filename']
         elif locmod == 'T2Flair':
             imfns=['flairid']
+        elif locmod == 'perf':
+            imfns=['perfid']
         elif locmod == 'NM2DMT':
             imfns=[]
             for i in range(11):
@@ -5797,6 +6195,9 @@ def mm_csv(
                 hier['brain_n4_dnz'],
                 normalization_template_transform_type,
                 outprefix = normout, verbose=False )
+            myjac = ants.create_jacobian_determinant_image( template,
+                    greg['fwdtransforms'][0], do_log=True, geom=True )
+            image_write_with_thumbnail( myjac, normout + "logjacobian.nii.gz", thumb=False )
             if verbose:
                 print("end group template registration")
         else:
@@ -5974,6 +6375,22 @@ def mm_csv(
                                         axis=2, nslices=maxslice, ncol=7, crop=True, title='DefaultMode', filename=mymm+mysep+"boldDefaultMode.png" )
                                     ants.plot( tabPro['rsf']['meanBold'], tabPro['rsf']['FrontoparietalTaskControl'],
                                         axis=2, nslices=maxslice, ncol=7, crop=True, title='FrontoparietalTaskControl', filename=mymm+mysep+"boldFrontoparietalTaskControl.png"  )
+                            if ( mymod == 'perf' )  and ishapelen == 4:
+                                dowrite=True
+                                tabPro, normPro = mm( t1, hier,
+                                    perfusion_image=img,
+                                    srmodel=None,
+                                    do_tractography=False,
+                                    do_kk=False,
+                                    do_normalization=templateTx,
+                                    group_template = normalization_template,
+                                    group_transform = groupTx,
+                                    test_run=test_run,
+                                    verbose=True )
+                                if tabPro['perf'] is not None and visualize:
+                                    maxslice = np.min( [21, tabPro['perf']['meanBold'].shape[2] ] )
+                                    ants.plot( tabPro['perf']['perfusion'],
+                                        axis=2, nslices=maxslice, ncol=7, crop=True, title='perfusion image', filename=mymm+mysep+"perfusion.png" )
                             if ( mymod == 'DTI_LR' or mymod == 'DTI_RL' or mymod == 'DTI' ) and ishapelen == 4:
                                 bvalfn = re.sub( '.nii.gz', '.bval' , myimg )
                                 bvecfn = re.sub( '.nii.gz', '.bvec' , myimg )
@@ -6037,7 +6454,7 @@ def mm_csv(
                                 write_mm( output_prefix=mymm, mm=tabPro, mm_norm=normPro, t1wide=t1wide, separator=mysep )
                                 for mykey in normPro.keys():
                                     if normPro[mykey] is not None and normPro[mykey].components == 1:
-                                        if visualize:
+                                        if visualize and False:
                                             ants.plot( template, normPro[mykey], axis=2, nslices=21, ncol=7, crop=True, title=mykey, filename=mymm+mysep+mykey+".png"   )
         if overmodX == nrg_modality_list[ len( nrg_modality_list ) - 1 ]:
             return
@@ -6445,7 +6862,7 @@ progress=False, verbose=False ):
     """
     extend a study data frame with wide outputs
 
-    sdf : the input study dataframe
+    sdf : the input study dataframe from antspymm QC output
 
     processing_dir:  the directory location of the processed data 
 
@@ -6468,8 +6885,8 @@ progress=False, verbose=False ):
     for k in range(len(musthavecols)):
         if not musthavecols[k] in sdf.keys():
             raise ValueError('sdf is missing column ' +musthavecols[k] + ' in merge_wides_to_study_dataframe' )
-    possible_iids = [ 'imageID', 'imageID', 'imageID', 'flairid', 'dtid1', 'dtid2', 'rsfid1', 'rsfid2', 'nmid1', 'nmid2', 'nmid3', 'nmid4', 'nmid5', 'nmid6', 'nmid7', 'nmid8', 'nmid9', 'nmid10' ]
-    modality_ids = [ 'T1wHierarchical', 'T1wHierarchicalSR', 'T1w', 'T2Flair', 'DTI', 'DTI', 'rsfMRI', 'rsfMRI', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT']
+    possible_iids = [ 'imageID', 'imageID', 'imageID', 'flairid', 'dtid1', 'dtid2', 'rsfid1', 'rsfid2', 'nmid1', 'nmid2', 'nmid3', 'nmid4', 'nmid5', 'nmid6', 'nmid7', 'nmid8', 'nmid9', 'nmid10', 'perfid' ]
+    modality_ids = [ 'T1wHierarchical', 'T1wHierarchicalSR', 'T1w', 'T2Flair', 'DTI', 'DTI', 'rsfMRI', 'rsfMRI', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'NM2DMT', 'perf']
     alldf=pd.DataFrame()
     for myk in sdf.index:
         if progress > 0 and int(myk) % int(progress) == 0:
@@ -7640,3 +8057,501 @@ def novelty_detection_quantile(df_train, df_test):
         temp = (myqs[mykey][0] >  df_train[mykey]).sum() / n
         myqs[mykey] = abs( temp - 0.5 ) / 0.5
     return myqs
+
+
+def brainmap_figure(statistical_df, data_dictionary_path, output_prefix, brain_image, overlay_cmap='bwr', nslices=21, ncol=7, edge_image_dilation = 0, black_bg=True, axes = [0,1,2], fixed_overlay_range=None, crop=True, verbose=False ):
+    """
+    Create figures based on statistical data and an underlying brain image.
+
+    Assumes both ~/.antspyt1w and ~/.antspymm data is available
+
+    Parameters:
+    - statistical_df (pandas dataframe): with 2 columns named anat and value
+        the anat column should have names that meet *partial matching* criterion 
+        with respect to regions that are measured in antspymm.   value will be 
+        the value to be displayed.   if two examples of a given region exist in 
+        statistical_df, then the largest absolute value will be taken for display.
+    - data_dictionary_path (str): Path to the data dictionary CSV file.
+    - output_prefix (str): Prefix for the output figure filenames.
+    - brain_image (antsImage): the brain image on which results will overlay.
+    - overlay_cmap (str): see matplotlib
+    - nslices (int): number of slices to show
+    - ncol (int): number of columns to show
+    - edge_image_dilation (int): integer greater than or equal to zero
+    - black_bg (bool): boolean
+    - axes (list): integer list typically [0,1,2] sagittal coronal axial
+    - fixed_overlay_range (list): scalar pair will try to keep a constant cbar and will truncate the overlay at these min/max values
+    - crop (bool): crops the image to display by the extent of the overlay
+    - verbose (bool): boolean
+
+    Returns:
+    an image with values mapped to the associated regions
+    """
+
+    # Read the statistical file
+    zz = statistical_df 
+    
+    # Read the data dictionary from a CSV file
+    mydict = pd.read_csv(data_dictionary_path)
+    mydict = mydict[~mydict['Measurement'].str.contains("tractography-based connectivity", na=False)]
+
+    # Load image and process it
+    edgeimg = ants.iMath(brain_image,"Normalize")
+    if edge_image_dilation > 0:
+        edgeimg = ants.iMath( edgeimg, "MD", edge_image_dilation)
+
+    # Define lists and data frames
+    postfix = ['bf', 'deep_cit168lab', 'mtl', 'cerebellum', 'dkt_cortex','brainstem']
+    atlas = ['BF', 'CIT168', 'MTL', 'TustisonCobra', 'desikan-killiany-tourville','brainstem']
+    postdesc = ['nbm3CH13', 'CIT168_Reinf_Learn_v1_label_descriptions_pad', 'mtl_description', 'cerebellum', 'dkt','CIT168_T1w_700um_pad_adni_brainstem']
+    statdf = pd.DataFrame({'img': postfix, 'atlas': atlas, 'csvdescript': postdesc})
+    templateprefix = '~/.antspymm/PPMI_template0_'
+    # Iterate through columns and create figures
+    col2viz = 'value'
+    if True:
+        anattoshow = zz['anat'].unique()
+        if verbose:
+            print(col2viz)
+            print(anattoshow)
+        # Rest of your code for figure creation goes here...
+        addem = edgeimg * 0
+        for k in range(len(anattoshow)):
+            if verbose:
+                print(str(k) +  " " + anattoshow[k]  )
+            mysub = zz[zz['anat'].str.contains(anattoshow[k])]
+            vals2viz = mysub[col2viz].agg(['min', 'max'])
+            vals2viz = vals2viz[abs(vals2viz).idxmax()]
+            myext = None
+            if 'dktcortex' in anattoshow[k]:
+                myext = 'dkt_cortex'
+            elif 'cit168' in anattoshow[k]:
+                myext = 'deep_cit168lab'
+            elif 'mtl' in anattoshow[k]:
+                myext = 'mtl'
+            elif 'cerebellum' in anattoshow[k]:
+                myext = 'cerebellum'
+            elif 'brainstem' in anattoshow[k]:
+                myext = 'brainstem'
+            elif any(item in anattoshow[k] for item in ['nbm', 'bf']):
+                myext = 'bf'
+            for j in postfix:
+                if j == "dkt_cortex":
+                    j = 'dktcortex'
+                if j == "deep_cit168lab":
+                    j = 'deep_cit168'
+                anattoshow[k] = anattoshow[k].replace(j, "")
+            if verbose:
+                print( anattoshow[k] + " " + str( vals2viz ) )
+            myatlas = atlas[postfix.index(myext)]
+            correctdescript = postdesc[postfix.index(myext)]
+            locfilename =  templateprefix + myext + '.nii.gz'
+            if verbose:
+                print( locfilename )
+            myatlas = ants.image_read(locfilename)
+            atlasDescript = pd.read_csv(f"~/.antspyt1w/{correctdescript}.csv")
+            atlasDescript['Description'] = atlasDescript['Description'].str.lower()
+            atlasDescript['Description'] = atlasDescript['Description'].str.replace(" ", "_")
+            atlasDescript['Description'] = atlasDescript['Description'].str.replace("_left_", "_")
+            atlasDescript['Description'] = atlasDescript['Description'].str.replace("_right_", "_")
+            atlasDescript['Description'] = atlasDescript['Description'].str.replace("_left", "")
+            atlasDescript['Description'] = atlasDescript['Description'].str.replace("_right", "")
+            if myext == 'cerebellum':
+                atlasDescript['Description'] = atlasDescript['Description'].str.replace("l_", "")
+                atlasDescript['Description'] = atlasDescript['Description'].str.replace("r_", "")
+                whichindex = atlasDescript.index[atlasDescript['Description'] == anattoshow[k]].values[0]
+            else:
+                whichindex = atlasDescript.index[atlasDescript['Description'].str.contains(anattoshow[k])]
+
+            if type(whichindex) is np.int64:
+                labelnums = atlasDescript.loc[whichindex, 'Label']
+            else:
+                labelnums = list(atlasDescript.loc[whichindex, 'Label'])
+            if not isinstance(labelnums, list):
+                labelnums=[labelnums]
+            addemiszero = ants.threshold_image(addem, 0, 0)
+            temp = ants.image_read(locfilename)
+            temp = ants.mask_image(temp, temp, level=labelnums, binarize=True)
+            temp[temp == 1] = (vals2viz)
+            temp[addemiszero == 0] = 0
+            addem = addem + temp
+
+        if verbose:
+            print('Done Adding')
+        for axx in axes:
+            figfn=output_prefix+f"fig{col2viz}ax{axx}_py.jpg"
+            if crop:
+                cmask = ants.threshold_image( addem,1e-5, 1e9 ).iMath("MD",3) + ants.threshold_image( addem,-1e9, -1e-5 ).iMath("MD",3)
+                addemC = ants.crop_image( addem, cmask )
+                edgeimgC = ants.crop_image( edgeimg, cmask )
+            else:
+                addemC = addem
+                edgeimgC = edgeimg
+            if fixed_overlay_range is not None:
+                addemC[0:3,0:3,0:3]=fixed_overlay_range[0]
+                addemC[4:7,4:7,4:7]=fixed_overlay_range[1]
+                addemC[ addemC < fixed_overlay_range[0] ] = fixed_overlay_range[0]
+                addemC[ addemC > fixed_overlay_range[1] ] = fixed_overlay_range[1]
+            ants.plot(edgeimgC, addemC, axis=axx, nslices=nslices, ncol=ncol,       
+                overlay_cmap=overlay_cmap, resample=False,
+                filename=figfn, cbar=axx==axes[0], crop=True, black_bg=black_bg )
+        if verbose:
+            print(f"{col2viz} done")
+    if verbose:
+        print("DONE brain map figures")
+    return addem
+
+
+def aggregate_antspymm_results(input_csv, subject_col='subjectID', date_col='date', image_col='imageID', date_column='ses-1', base_path="./Processed/ANTsExpArt/", hiervariable='T1wHierarchical', valid_modalities=None, verbose=False ):
+    """
+    Aggregate ANTsPyMM results from the specified CSV file and save the aggregated results to a new CSV file.
+
+    Parameters:
+    - input_csv (str): File path of the input CSV file containing ANTsPyMM QC results averaged and with outlier measurements.
+    - subject_col (str): Name of the column to store subject IDs.
+    - date_col (str): Name of the column to store date information.
+    - image_col (str): Name of the column to store image IDs.
+    - date_column (str): Name of the column representing the date information.
+    - base_path (str): Base path for search paths. Defaults to "./Processed/ANTsExpArt/".
+    - hiervariable (str) : the string variable denoting the Hierarchical output
+    - valid_modalities (str array) : identifies for each modality; if None will be replaced by get_valid_modalities(long=True)
+    - verbose : boolean
+
+    Note:
+    This function is tested under limited circumstances. Use with caution.
+
+    Example usage:
+    agg_df = aggregate_antspymm_results("qcdfaol.csv", subject_col='subjectID', date_col='date', image_col='imageID', date_column='ses-1', base_path="./Your/Custom/Path/")
+
+    Author:
+    Avants and ChatGPT
+    """
+    import pandas as pd
+    import numpy as np
+    from glob import glob
+
+    def filter_df( indf, myprefix ):
+        nums = [isinstance(indf[col].iloc[0], (int, float)) for col in indf.columns]
+        indf = indf.loc[:, nums]
+        indf=indf.loc[:, indf.dtypes != 'object' ]
+        indf = indf.loc[:, ~indf.columns.str.contains('Unnamed*', na=False, regex=True)]
+        indf = pd.DataFrame(indf.mean(axis=0, skipna=True)).T
+        indf = indf.add_prefix( myprefix )
+        return( indf )
+
+    def myread_csv(x, cnms):
+        """
+        Reads a CSV file and returns a DataFrame excluding specified columns.
+
+        Parameters:
+        - x (str): File path of the input CSV file describing the blind QC output
+        - cnms (list): List of column names to exclude from the DataFrame.
+
+        Returns:
+        pd.DataFrame: DataFrame with specified columns excluded.
+        """
+        df = pd.read_csv(x)
+        return df.loc[:, ~df.columns.isin(cnms)]
+
+    import warnings
+    # Warning message for untested function
+    warnings.warn("Warning: This function is not well tested. Use with caution.")
+
+    if valid_modalities is None:
+        valid_modalities = get_valid_modalities('long')
+
+    # Read the input CSV file
+    df = pd.read_csv(input_csv)
+
+    # Filter rows where modality is 'T1w'
+    df = df[df['modality'] == 'T1w']
+    badnames = get_names_from_data_frame( ['Unnamed'], df )
+    df=df.drop(badnames, axis=1)
+
+    # Add new columns for subject ID, date, and image ID
+    df[subject_col] = np.nan
+    df[date_col] = date_column
+    df[image_col] = np.nan
+    df = df.astype({subject_col: str, date_col: str, image_col: str })
+
+#    if verbose:
+#        print( df.shape )
+#        print( df.dtypes )
+
+    # prefilter df for data that exists
+    keep = np.tile( False, df.shape[0] )
+    for x in range(df.shape[0]):
+        temp = df['fn'].iloc[x].split("_")
+        # Generalized search paths
+        path_template = f"{base_path}{temp[0]}/{date_column}/*/*/*"
+        hierfn = sorted(glob( path_template + "-" + hiervariable + "-*wide.csv" ) )
+        if len( hierfn ) > 0:
+            keep[x]=True
+
+    
+    df=df[keep]
+    
+    if verbose:
+        print( "original input had shape " + str( df.shape[0] ) + " (T1 only) and we find " + str( (keep).sum() ) + " with hierarchical output defined by variable: " + hiervariable )
+        print( df.shape )
+
+    myct = 0
+    for x in range( df.shape[0]):
+        if verbose:
+            print(f"{x}...")
+        locind = df.index[x]
+        temp = df['fn'].iloc[x].split("_")
+        if verbose:
+            print( temp )
+        df[subject_col].iloc[x]=temp[0]
+        df[date_col].iloc[x]=date_column
+        df[image_col].iloc[x]=temp[1]
+
+        # Generalized search paths
+        path_template = f"{base_path}{temp[0]}/{date_column}/*/*/*"
+        if verbose:
+            print(path_template)
+        hierfn = sorted(glob( path_template + "-" + hiervariable + "-*wide.csv" ) )
+        if len( hierfn ) > 0:
+            hdf=t1df=dtdf=rsdf=perfdf=nmdf=flairdf=None
+            if verbose:
+                print(hierfn)
+            hdf = pd.read_csv(hierfn[0])
+            badnames = get_names_from_data_frame( ['Unnamed'], hdf )
+            hdf=hdf.drop(badnames, axis=1)
+            nums = [isinstance(hdf[col].iloc[0], (int, float)) for col in hdf.columns]
+            corenames = list(np.array(hdf.columns)[nums])
+            hdf.loc[:, nums] = hdf.loc[:, nums].add_prefix("T1Hier_")
+            myct = myct + 1
+            dflist = [hdf]
+
+            for mymod in valid_modalities:
+                t1wfn = sorted(glob( path_template+ "-" + mymod + "-*wide.csv" ) )
+                if len( t1wfn ) > 0 :
+                    if verbose:
+                        print(t1wfn)
+                    t1df = myread_csv(t1wfn[0], corenames)
+                    t1df = filter_df( t1df, mymod+'_')
+                    dflist = dflist + [t1df]
+                
+            hdf = pd.concat( dflist, axis=1)
+            if verbose:
+                print( df.loc[locind,'fn'] )
+            if myct == 1:
+                subdf = df.iloc[[x]]
+                hdf.index = subdf.index.copy()
+                df = pd.concat( [df,hdf], axis=1)
+            else:
+                commcols = list(set(hdf.columns).intersection(df.columns))
+                df.loc[locind, commcols] = hdf.loc[0, commcols]
+    badnames = get_names_from_data_frame( ['Unnamed'], df )
+    df=df.drop(badnames, axis=1)
+    return( df )
+
+def aggregate_antspymm_results_sdf(
+    study_df, 
+    project_col='projectID',
+    subject_col='subjectID', 
+    date_col='date', 
+    image_col='imageID', 
+    base_path="./", 
+    hiervariable='T1wHierarchical', 
+    splitsep='-',
+    idsep='-',
+    wild_card_modality_id=False,
+    verbose=False ):
+    """
+    Aggregate ANTsPyMM results from the specified study data frame and store the aggregated results in a new data frame.  This assumes data is organized on disk 
+    as follows:  rootdir/projectID/subjectID/date/outputid/imageid/ where 
+    outputid is modality-specific and created by ANTsPyMM processing.
+
+    Parameters:
+    - study_df (pandas df): pandas data frame, output of generate_mm_dataframe.
+    - project_col (str): Name of the column that stores the project ID
+    - subject_col (str): Name of the column to store subject IDs.
+    - date_col (str): Name of the column to store date information.
+    - image_col (str): Name of the column to store image IDs.
+    - base_path (str): Base path for searching for processing outputs of ANTsPyMM.
+    - hiervariable (str) : the string variable denoting the Hierarchical output
+    - splitsep (str):  the separator used to split the filename
+    - idsep (str): the separator used to partition subjectid date and imageid 
+        for example, if idsep is - then we have subjectid-date-imageid
+    - wild_card_modality_id (bool): keep if False for safer execution
+    - verbose : boolean
+
+    Note:
+    This function is tested under limited circumstances. Use with caution.
+
+    Example usage:
+    agg_df = aggregate_antspymm_results_sdf( studydf, subject_col='subjectID', date_col='date', image_col='imageID', base_path="./Your/Custom/Path/")
+
+    Author:
+    Avants and ChatGPT
+    """
+    import pandas as pd
+    import numpy as np
+    from glob import glob
+
+    def filter_df( indf, myprefix ):
+        nums = [isinstance(indf[col].iloc[0], (int, float)) for col in indf.columns]
+        indf = indf.loc[:, nums]
+        indf=indf.loc[:, indf.dtypes != 'object' ]
+        indf = indf.loc[:, ~indf.columns.str.contains('Unnamed*', na=False, regex=True)]
+        indf = pd.DataFrame(indf.mean(axis=0, skipna=True)).T
+        indf = indf.add_prefix( myprefix )
+        return( indf )
+
+    def myread_csv(x, cnms):
+        """
+        Reads a CSV file and returns a DataFrame excluding specified columns.
+
+        Parameters:
+        - x (str): File path of the input CSV file describing the blind QC output
+        - cnms (list): List of column names to exclude from the DataFrame.
+
+        Returns:
+        pd.DataFrame: DataFrame with specified columns excluded.
+        """
+        df = pd.read_csv(x)
+        return df.loc[:, ~df.columns.isin(cnms)]
+
+    import warnings
+    # Warning message for untested function
+    warnings.warn("Warning: This function is not well tested. Use with caution.")
+
+    # if valid_modalities is None:
+    valid_modalities = get_valid_modalities('long')
+    vmoddict = {}
+    # Add key-value pairs
+    vmoddict['imageID'] = 'T1w'
+    vmoddict['flairid'] = 'T2Flair'
+    vmoddict['perfid'] = 'perf'
+    vmoddict['rsfid1'] = 'rsfMRI'
+    vmoddict['dtid1'] = 'DTI'
+    vmoddict['nmid1'] = 'NM2DMT'
+
+    # Filter rows where modality is 'T1w'
+    df = study_df[study_df['modality'] == 'T1w']
+    badnames = get_names_from_data_frame( ['Unnamed'], df )
+    df=df.drop(badnames, axis=1)
+    # prefilter df for data that exists
+    keep = np.tile( False, df.shape[0] )
+    for x in range(df.shape[0]):
+        myfn = os.path.basename( df['filename'].iloc[x] )
+        temp = myfn.split( splitsep )
+        # Generalized search paths
+        sid0 = temp[0]
+        sid = str(df[subject_col].iloc[x])
+        if sid0 != sid:
+            warnings.warn("the id derived from the filename " + sid + " does not match the id stored in the data frame " + sid )
+        myproj = str(df[project_col].iloc[x])
+        mydate = str(df[date_col].iloc[x])
+        myid = str(df[image_col].iloc[x])
+        path_template = base_path + "/" + myproj +  "/" + sid + "/" + mydate + '/' + hiervariable + '/' + str(myid) + "/"
+        hierfn = sorted(glob( path_template + "*" + hiervariable + "*wide.csv" ) )
+        if len( hierfn ) > 0:
+            keep[x]=True
+
+    df=df[keep]
+
+    if not df.index.is_unique:
+        warnings.warn("data frame does not have unique indices.  we therefore reset the index to allow the function to continue on." )
+        df = df.reset_index()
+
+    
+    if verbose:
+        print( "original input had shape " + str( df.shape[0] ) + " (T1 only) and we find " + str( (keep).sum() ) + " with hierarchical output defined by variable: " + hiervariable )
+        print( df.shape )
+
+    myct = 0
+    for x in range( df.shape[0]):
+        print("\n\n-------------------------------------------------")
+        if verbose:
+            print(f"{x}...")
+        locind = df.index[x]
+        myfn = os.path.basename( df['filename'].iloc[x] )
+        sid = df[subject_col].iloc[x]
+        if sid0 != sid:
+            warnings.warn("the id derived from the filename " + sid + " does not match the id stored in the data frame " + sid )
+        myproj = str(df[project_col].iloc[x])
+        mydate = str(df[date_col].iloc[x])
+        myid = str(df[image_col].iloc[x])
+        if verbose:
+            print( myfn )
+            print( temp )
+            print( "id " + sid  )
+        path_template = base_path + "/" + myproj +  "/" + sid + "/" + mydate + '/' + hiervariable + '/' + str(myid) + "/"
+        searchhier = path_template + "*" + hiervariable + "*wide.csv"
+        if verbose:
+            print( searchhier )
+        hierfn = sorted( glob( searchhier ) )
+        if len( hierfn ) > 1:
+            raise ValueError("there are " + str( len( hierfn ) ) + " number of hier fns with search path " + searchhier )
+        if len( hierfn ) == 1:
+            hdf=t1df=dtdf=rsdf=perfdf=nmdf=flairdf=None
+            if verbose:
+                print(hierfn)
+            hdf = pd.read_csv(hierfn[0])
+            badnames = get_names_from_data_frame( ['Unnamed'], hdf )
+            hdf=hdf.drop(badnames, axis=1)
+            nums = [isinstance(hdf[col].iloc[0], (int, float)) for col in hdf.columns]
+            corenames = list(np.array(hdf.columns)[nums])
+            hdf.loc[:, nums] = hdf.loc[:, nums].add_prefix("T1Hier_")
+            myct = myct + 1
+            hdf = hdf.add_prefix( "T1Hier_" )
+            dflist = [hdf]
+
+            for mymod in vmoddict.keys():
+                print("\n\n************************* " + mymod + " *************************")
+                modalityclass = vmoddict[ mymod ]
+                if wild_card_modality_id:
+                    mymodid = '*'
+                else:
+                    mymodid = str( df[mymod].iloc[x] )
+                    if mymodid.lower() != "nan" and mymodid.lower() != "na":
+                        mymodid = os.path.basename( mymodid )
+                        mymodid = os.path.splitext( mymodid )[0]
+                        mymodid = os.path.splitext( mymodid )[0]
+                        temp = mymodid.split( idsep )
+                        mymodid = temp[ len( temp )-1 ]
+                    else:
+                        print("missing")
+                        continue
+                if verbose:
+                    print( "modality id is " + mymodid + " for modality " + modalityclass )
+                modalityclasssearch = modalityclass
+                if modalityclass in ['rsfMRI','DTI']:
+                    modalityclasssearch=modalityclass+"*"
+                path_template_m = base_path + "/" + myproj +  "/" + sid + "/" + mydate + '/' + modalityclasssearch + '/' + mymodid + "/"
+                modsearch = path_template_m + "*" + modalityclasssearch + "*wide.csv"
+                if verbose:
+                    print( modsearch )
+                t1wfn = sorted( glob( modsearch ) )
+                if len( t1wfn ) > 1:
+                    raise ValueError("there are " + str( len( t1wfn ) ) + " number of wide fns with search path " + modsearch )
+                if len( t1wfn ) == 1:
+                    if verbose:
+                        print(t1wfn)
+                    t1df = myread_csv(t1wfn[0], corenames)
+                    t1df = filter_df( t1df, modalityclass+'_')
+                    dflist = dflist + [t1df]
+                else:
+                    if verbose:
+                        print( " cannot find " + modsearch )
+                
+            hdf = pd.concat( dflist, axis=1)
+            if verbose:
+                print( "count: " + str( myct ) )
+            if myct == 1:
+                subdf = df.iloc[[x]]
+                hdf.index = subdf.index.copy()
+                print( hdf.index )
+                print( df.index )
+                df = pd.concat( [df,hdf], axis=1)
+            else:
+                commcols = list(set(hdf.columns).intersection(df.columns))
+                df.loc[locind, commcols] = hdf.loc[0, commcols]
+    badnames = get_names_from_data_frame( ['Unnamed'], df )
+    df=df.drop(badnames, axis=1)
+    return( df )
+
+

{antspymm-1.1.1.dist-info → antspymm-1.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: antspymm
-Version: 1.1.1
+Version: 1.1.3
 Summary: multi-channel/time-series medical image processing with antspyx
 Home-page: https://github.com/stnava/ANTsPyMM
 Author: Avants, Gosselin, Tustison, Reardon
@@ -18,10 +18,13 @@ Requires-Dist: dipy
 Requires-Dist: nibabel
 Requires-Dist: scipy
 Requires-Dist: siq
-Requires-Dist: sklearn
+Requires-Dist: scikit-learn
 
 # ANTsPyMM
 
+
+[mapping](https://imgur.com/a/XzpXI3i)
+
 ## processing utilities for timeseries/multichannel images - mostly neuroimaging
 
 the outputs of these processes can be used for data inspection/cleaning/triage
@@ -243,6 +246,13 @@ studycsv = antspymm.generate_mm_dataframe(
     rsf_filenames=[fns[2]])
 studycsv2 = studycsv.dropna(axis=1)
 mmrun = antspymm.mm_csv( studycsv2, mysep='_' )
+
+# aggregate the data after you've run on many subjects
+# studycsv_all would be the vstacked studycsv2 data frames
+zz=antspymm.aggregate_antspymm_results_sdf( studycsv_all, 
+    subject_col='subjectID', date_col='date', image_col='imageID',  base_path=bd, 
+    splitsep='_', idsep='-', wild_card_modality_id=True, verbose=True)
+
 ```
 
 ## NRG example
@@ -426,6 +436,15 @@ for f in folders:
 pdoc -o ./docs antspymm --html
 ```
 
+## ssl error 
+
+if you get an odd certificate error when calling `force_download`, try:
+
+```python
+import ssl
+ssl._create_default_https_context = ssl._create_unverified_context
+``
+
 ## to publish a release
 
 ```

antspymm-1.1.3.dist-info/RECORD

@@ -0,0 +1,7 @@
+antspymm/__init__.py,sha256=wck10IhO1ZFxNqz4bTIHFP7cqYosRKBI8bKIsZzRwfs,3166
+antspymm/mm.py,sha256=sYXNst5xri6vFDzvuqDB6avuwkKoq3GTEsvm0RLUuPU,347923
+antspymm-1.1.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+antspymm-1.1.3.dist-info/METADATA,sha256=bFp7u3Bd85Sq4Nc_TvvzxfYDhyU9Uxj4T9yzDcDK26I,14153
+antspymm-1.1.3.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+antspymm-1.1.3.dist-info/top_level.txt,sha256=iyD1sRhCKzfwKRJLq5ZUeV9xsv1cGQl8Ejp6QwXM1Zg,9
+antspymm-1.1.3.dist-info/RECORD,,

{antspymm-1.1.1.dist-info → antspymm-1.1.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

antspymm-1.1.1.dist-info/RECORD

@@ -1,7 +0,0 @@
-antspymm/__init__.py,sha256=BTNN_iisXoz2on4gVcAQWFB_qJYDM-yqZOJU1xho9sw,2931
-antspymm/mm.py,sha256=SX1eYoKPmZ6QId0GZBIiztywWJAjNX2lUy9uihyuFG4,308416
-antspymm-1.1.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-antspymm-1.1.1.dist-info/METADATA,sha256=wxrVbMj17gBEbDV9Nuh2Df8S-XPKTH35qS5iZcvPKic,13598
-antspymm-1.1.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-antspymm-1.1.1.dist-info/top_level.txt,sha256=iyD1sRhCKzfwKRJLq5ZUeV9xsv1cGQl8Ejp6QwXM1Zg,9
-antspymm-1.1.1.dist-info/RECORD,,