treesak 1.51.2__py3-none-any.whl

TreeSAK/PMSF.py

@@ -0,0 +1,106 @@
+import os
+import argparse
+from distutils.spawn import find_executable
+
+
+PMSF_usage = '''
+==================== PMSF example commands ====================
+
+# Dependency: iqtree2
+
+TreeSAK PMSF -i in.aln -o get_PMSF_tree_wd -t 12
+TreeSAK PMSF -i in.aln -o get_PMSF_tree_wd -t 12 -topo topo.tree
+
+# This is a wrapper for:
+iqtree2 -T 12 -B 1000 --alrt 1000 --quiet --seqtype AA -s in.aln --prefix guide_tree -m LG+F+G 
+iqtree2 -T 12 -B 1000 --alrt 1000 --quiet --seqtype AA -s in.aln --prefix PMSF -m LG+C60+F+G -ft guide_tree.treefile
+
+# more information: http://www.iqtree.org/doc/Complex-Models
+
+# Reference: The evolutionary origin of host association in the Rickettsiales
+Maximum likelihood phylogenetic reconstructions were done under the PMSF approximation 
+(with 100 non-parametric bootstraps; guidetree under LG+G+F) of the LG+C60+F+Γ4 model 
+(selected by ModelFinder) for both supermatrix alignments with IQTREE v1.6.5.
+
+===============================================================
+'''
+
+
+def PMSF(args):
+
+    msa_in                  = args['i']
+    iqtree_model_guide_tree = args['gm']
+    iqtree_model            = args['m']
+    op_dir                  = args['o']
+    tree_prefix             = args['p']
+    force_overwrite         = args['f']
+    num_of_threads          = args['t']
+    topo_constraint_txt     = args['topo']
+
+    guide_tree_wd  = '%s/guide_tree'           % op_dir
+    pwd_guide_tree = '%s/guide_tree.treefile'  % guide_tree_wd
+    pwd_cmd_txt    = '%s/cmds.txt'             % op_dir
+
+    iqtree_exe = ''
+    if find_executable('iqtree2'):
+        iqtree_exe = 'iqtree2'
+    elif find_executable('iqtree'):
+        iqtree_exe = 'iqtree'
+    else:
+        print('iqtree not detected, program exited!')
+        exit()
+
+    # check input file
+    if os.path.isfile(msa_in) is False:
+        print('MSA file not found, program exited!')
+        exit()
+
+    # create output dir
+    if os.path.isdir(op_dir) is True:
+        if force_overwrite is True:
+            os.system('rm -r %s' % op_dir)
+        else:
+            print('output folder already exist, program exited!')
+            exit()
+    os.system('mkdir %s' % op_dir)
+    os.system('mkdir %s' % guide_tree_wd)
+
+    guidetree_cmd = '%s -s %s --prefix %s/guide_tree --seqtype AA -m %s -T %s -B 1000 --alrt 1000 --quiet' % (iqtree_exe, msa_in, guide_tree_wd, iqtree_model_guide_tree, num_of_threads)
+    iqtree_cmd    = '%s -s %s --prefix %s/%s --seqtype AA -m %s -T %s -B 1000 --alrt 1000 --quiet -ft %s'  % (iqtree_exe, msa_in, op_dir, tree_prefix, iqtree_model, num_of_threads, pwd_guide_tree)
+    if topo_constraint_txt is not None:
+        guidetree_cmd = '%s -s %s --prefix %s/guide_tree --seqtype AA -m %s -T %s -B 1000 --alrt 1000 --quiet -g %s' % (iqtree_exe, msa_in, guide_tree_wd, iqtree_model_guide_tree, num_of_threads, topo_constraint_txt)
+        iqtree_cmd    = '%s -s %s --prefix %s/%s --seqtype AA -m %s -T %s -B 1000 --alrt 1000 --quiet -ft %s -g %s'  % (iqtree_exe, msa_in, op_dir, tree_prefix, iqtree_model, num_of_threads, pwd_guide_tree, topo_constraint_txt)
+
+    # write out commands
+    pwd_cmd_txt_handle = open(pwd_cmd_txt, 'w')
+    pwd_cmd_txt_handle.write(guidetree_cmd + '\n')
+    pwd_cmd_txt_handle.write(iqtree_cmd + '\n')
+    pwd_cmd_txt_handle.close()
+
+    # get guide tree
+    print('Building guide tree')
+    print(guidetree_cmd)
+    os.system(guidetree_cmd)
+
+    # get PMSF tree
+    print('Building PMSF tree with model %s' % iqtree_model)
+    print(iqtree_cmd)
+    os.system(iqtree_cmd)
+
+    print('Done!')
+
+
+if __name__ == '__main__':
+
+    # initialize the options parser
+    PMSF_parser = argparse.ArgumentParser()
+    PMSF_parser.add_argument('-i',       required=True,                          help='input MSA file')
+    PMSF_parser.add_argument('-gm',      required=False, default='LG+F+G',       help='iqtree model for guide tree, default: LG+F+G')
+    PMSF_parser.add_argument('-m',       required=False, default='LG+C60+F+G',   help='iqtree model, default: LG+C60+F+G')
+    PMSF_parser.add_argument('-o',       required=True,                          help='output plot')
+    PMSF_parser.add_argument('-p',       required=False, default='PMSF',         help='tree prefix, default: PMSF')
+    PMSF_parser.add_argument('-topo',    required=False, default=None,           help='topological constraint tree, pass to -g, default is None')
+    PMSF_parser.add_argument('-t',       required=False, type=int, default=1,    help='num of threads')
+    PMSF_parser.add_argument('-f',       required=False, action="store_true",    help='force overwrite')
+    args = vars(PMSF_parser.parse_args())
+    PMSF(args)

TreeSAK/PhyloBiAssoc.R

@@ -0,0 +1,84 @@
+suppressMessages(suppressWarnings(library("ape")))
+suppressMessages(suppressWarnings(library("phytools")))
+suppressMessages(suppressWarnings(library("optparse")))
+
+################################################################################
+
+option_list = list(
+  make_option(c("-t", "--treefile"), type="character", default=NULL, help="tree file"),
+  make_option(c("-d", "--datafile"), type="character", default=NULL, help="data file"));
+opt_parser = OptionParser(option_list=option_list);
+opt = parse_args(opt_parser);
+
+tree_file = opt$treefile
+data_file = opt$datafile
+
+# Rscript PhyloBiAssoc.R -t demo.tre -d demo.txt
+# phylosig        7.973475e-26    binaryPGLMM     0.03255813
+# The header of the first two columns has to be "ID" and "cate".
+
+################################################################################
+
+geotree <- read.tree(tree_file)
+geodata <- read.table(data_file, header = TRUE, sep = "\t")
+
+# sort rows in df according to the order of tips in the tree
+row.names(geodata) <- geodata[,1]
+row.names(geodata) <- geodata$ID
+geodata <- geodata[geotree$tip.label,]
+colnames_all = colnames(geodata[, ,])
+colnames_to_process = tail(colnames_all, -2)
+
+cat('ID', "phylosig", "binaryPGLMM", "chisq.test", "coefficient", "significance", '\n', fill=FALSE, sep = "\t")
+
+for (i in colnames_to_process){
+
+  # perform phylosig test
+  phylosig_test <- phylosig(tree = geotree, x = setNames(geodata[, i], geodata$ID), method = "lambda", test = TRUE)
+  phylosig_test_pvalue = phylosig_test$P
+
+  # perform binaryPGLMM test if phylosig P-value <= 0.05 (indicating significant phylogenetic signal)
+  # perform chi-squared test if phylosig P-value > 0.05  (indicating no phylogenetic signal)
+  # do nothing if phylosig returns NaN
+
+  association_test = ''
+  association_p_value = NA
+  do_nothing = FALSE
+  association_coefficient = 'na'
+  significant = 'n'
+  if (phylosig_test_pvalue == 'NaN') {
+    do_nothing = TRUE
+    significant = 'na'
+  } else if (phylosig_test_pvalue <= 0.05) {
+    binaryPGLMM_result <- binaryPGLMM(setNames(geodata[, i], geodata$ID) ~ geodata$cate, phy = geotree)
+    association_test = 'binaryPGLMM'
+    association_coefficient = binaryPGLMM_result$B[1]
+    association_p_value = binaryPGLMM_result$B.pvalue[2]
+    if (association_p_value <= 0.05) {
+      significant = 'y'
+    }
+
+  } else {
+    chisq_test <- chisq.test(table(geodata$cate, setNames(geodata[, i], geodata$ID)))
+    association_test = 'chisq.test'
+    association_p_value = chisq_test$p.value
+
+    #cor_test <- cor.test(geodata$cate, geodata[, i])
+    #association_coefficient = cor_test$estimate
+
+    if (association_p_value <= 0.05) {
+      significant = 'y'
+    }
+  }
+
+  # print to screen
+  if (do_nothing == FALSE) {
+
+  if (association_test == 'binaryPGLMM'){
+    cat(i, phylosig_test_pvalue, association_p_value, 'na', association_coefficient, significant, '\n', fill=FALSE, sep = "\t")
+  }
+  if (association_test == 'chisq.test'){
+    cat(i, phylosig_test_pvalue, 'na', association_p_value, association_coefficient, significant, '\n', fill=FALSE, sep = "\t")
+  }
+  }
+}

TreeSAK/PhyloBiAssoc.py

@@ -0,0 +1,167 @@
+import os
+import argparse
+import pandas as pd
+import multiprocessing as mp
+from statsmodels.stats.multitest import multipletests
+
+
+PhyloBiAssoc_usage = '''
+============================= PhyloBiAssoc example commands =============================
+
+BioSAK PhyloBiAssoc -i demo.tre -d demo.txt -o op_dir -t 10 -f
+
+# Note, header for the first two columns in -d has to be "ID" and "cate"!!!
+
+# It will perform:
+# 1) binaryPGLMM test if phylosig p-value <= 0.05 (significant phylogenetic signal)
+# 2) chi-squared test if phylosig p-value > 0.05  (no phylogenetic signal)
+# 3) do nothing if phylosig returns NaN (might due to the same value across all genomes)
+
+# https://www.rdocumentation.org/packages/ape/versions/5.7-1/topics/binaryPGLMM
+
+=========================================================================================
+'''
+
+
+def sep_path_basename_ext(file_in):
+
+    f_path, file_name = os.path.split(file_in)
+    if f_path == '':
+        f_path = '.'
+
+    f_base, f_ext = os.path.splitext(file_name)
+    return f_path, f_base, f_ext
+
+
+def subset_df(file_in, rows_to_keep, cols_to_keep, sep_symbol, row_name_pos, column_name_pos, file_out):
+
+    df = pd.read_csv(file_in, sep=sep_symbol, header=column_name_pos, index_col=row_name_pos)
+
+    if len(rows_to_keep) == 0:
+        if len(cols_to_keep) == 0:
+            subset_df = df.loc[:, :]
+        else:
+            subset_df = df.loc[:, cols_to_keep]
+    else:
+        if len(cols_to_keep) == 0:
+            subset_df = df.loc[rows_to_keep, :]
+        else:
+            subset_df = df.loc[rows_to_keep, cols_to_keep]
+
+    subset_df.to_csv(file_out, sep=sep_symbol)
+
+
+def PhyloBiAssoc(args):
+
+    tree_file           = args['i']
+    data_file           = args['d']
+    op_dir              = args['o']
+    num_threads         = args['t']
+    force_create_op_dir = args['f']
+
+    pwd_current_script  = os.path.realpath(__file__)
+    current_script_path = '/'.join(pwd_current_script.split('/')[:-1])
+    PhyloBiAssoc_R      = '%s/PhyloBiAssoc.R' % current_script_path
+
+    cmd_txt                     = '%s/cmds.txt'                         % op_dir
+    df_subset_dir               = '%s/df_subset'                        % op_dir
+    stats_op_dir                = '%s/stats_results'                    % op_dir
+    combined_stats_txt          = '%s/stats_results_all.txt'            % op_dir
+    combined_stats_txt_sig      = '%s/stats_results_0.05.txt'           % op_dir
+    combined_stats_txt_adjusted = '%s/stats_results_0.05_adjusted.txt'  % op_dir
+
+    # create op_dir
+    if os.path.isdir(op_dir) is True:
+        if force_create_op_dir is True:
+            os.system('rm -r %s' % op_dir)
+        else:
+            print('output directory exist, program exited!')
+            exit()
+    os.system('mkdir %s' % op_dir)
+    os.system('mkdir %s' % df_subset_dir)
+    os.system('mkdir %s' % stats_op_dir)
+
+    # read in dataframe
+    df = pd.read_csv(data_file, sep='\t', header=0, index_col=0)
+    col_header_list = list(df.columns.values)
+
+    subset_dict = dict()
+    for each_col in col_header_list[1:]:
+        subset_dict[each_col] = ['cate', each_col]
+
+    # subset dataframe
+    cmd_txt_handle = open(cmd_txt, 'w')
+    stats_cmd_list = []
+    op_stats_txt_set = set()
+    for each_subset in subset_dict:
+        cols_to_keep   = subset_dict[each_subset]
+        df_subset_file = '%s/%s.tab' % (df_subset_dir, each_subset)
+        stats_out_txt  = '%s/%s.txt' % (stats_op_dir, each_subset)
+        subset_df(data_file, set(), cols_to_keep, '\t', 0, 0, df_subset_file)
+        stats_cmd = 'Rscript %s -t %s -d %s > %s' % (PhyloBiAssoc_R, tree_file, df_subset_file, stats_out_txt)
+        cmd_txt_handle.write(stats_cmd + '\n')
+        stats_cmd_list.append(stats_cmd)
+        op_stats_txt_set.add(stats_out_txt)
+    cmd_txt_handle.close()
+
+    print('Processing %s objects with %s cores' % (len(stats_cmd_list), num_threads))
+    pool = mp.Pool(processes=num_threads)
+    pool.map(os.system, stats_cmd_list)
+    pool.close()
+    pool.join()
+    #os.system('cp /Users/songweizhi/Documents/Research/Sponge/12_PhyloBiAssoc_wd/PhyloBiAssoc_wd_backup/stats_results/*.txt %s/' % stats_op_dir)
+
+    # combine stats results
+    sig_list_id = []
+    sig_list_value = []
+
+    combined_stats_txt_handle = open(combined_stats_txt, 'w')
+    combined_stats_txt_handle.write('ID	phylosig	binaryPGLMM	chisq.test	coefficient	significance\n')
+    combined_stats_txt_sig_handle = open(combined_stats_txt_sig, 'w')
+    for each_file in sorted(list(op_stats_txt_set)):
+        f_path, f_base, f_ext = sep_path_basename_ext(each_file)
+        for each_line in open(each_file):
+            if not each_line.startswith('ID\tphylosig\tbinaryPGLMM\tchisq.test'):
+                each_line_split = each_line.strip().split('\t')
+                significance = each_line_split[5]
+                combined_stats_txt_handle.write('%s\t%s\n' % (f_base, '\t'.join(each_line_split[1:])))
+                if significance == 'y':
+                    combined_stats_txt_sig_handle.write(f_base + '\n')
+                    sig_bi = each_line_split[2]
+                    sig_chi = each_line_split[3]
+                    current_sig = ''
+                    if sig_bi == 'na':
+                        current_sig = sig_chi
+                    elif sig_chi == 'na':
+                        current_sig = sig_bi
+                    sig_list_id.append(f_base)
+                    sig_list_value.append(float(current_sig))
+    combined_stats_txt_handle.close()
+    combined_stats_txt_sig_handle.close()
+
+    # perform Bonferroni correction
+    sig_list_value_adjusted = list(multipletests(sig_list_value, alpha=0.1, method='bonferroni')[1])
+
+    # write out adjusted p values
+    combined_stats_txt_adjusted_handle = open(combined_stats_txt_adjusted, 'w')
+    combined_stats_txt_adjusted_handle.write('ID\tadjusted_p_value\n')
+    for (id, adjusted_p) in zip(sig_list_id, sig_list_value_adjusted):
+        if adjusted_p <= 0.05:
+            combined_stats_txt_adjusted_handle.write('%s\t%s\n' % (id, adjusted_p))
+    combined_stats_txt_adjusted_handle.close()
+
+    # Final report
+    print('Results exported to: \n%s\n%s' % (combined_stats_txt, combined_stats_txt_adjusted))
+    print('Done')
+
+
+if __name__ == "__main__":
+
+    PhyloBiAssoc_parser = argparse.ArgumentParser(usage=PhyloBiAssoc_usage)
+    PhyloBiAssoc_parser.add_argument('-i', required=True,                       help='tree file')
+    PhyloBiAssoc_parser.add_argument('-d', required=True,                       help='data file')
+    PhyloBiAssoc_parser.add_argument('-o', required=True,                       help='output directory')
+    PhyloBiAssoc_parser.add_argument('-t', required=False, type=int, default=1, help='number of threads, default: 1')
+    PhyloBiAssoc_parser.add_argument('-f', required=False, action="store_true", help='force overwrite')
+    args = vars(PhyloBiAssoc_parser.parse_args())
+    PhyloBiAssoc(args)

TreeSAK/PlotMCMC.py

@@ -0,0 +1,41 @@
+import matplotlib as mpl
+mpl.use('Agg')
+import matplotlib.pyplot as plt
+
+
+def PlotMCMC(num_list_x, num_list_y, err_range_x, err_range_y, pwd_figure):
+
+    x_err_l = []
+    x_err_r = []
+    y_err_l = []
+    y_err_u = []
+    n = 0
+    while n < len(num_list_x):
+        x_value = num_list_x[n]
+        y_value = num_list_y[n]
+        x_range = err_range_x[n]
+        y_range = err_range_y[n]
+        x_l_dist = abs(x_value - x_range[0])
+        x_r_dist = abs(x_range[1] - x_value)
+        y_l_dist = abs(y_value - y_range[0])
+        y_u_dist = abs(y_range[1] - y_value)
+        x_err_l.append(x_l_dist)
+        x_err_r.append(x_r_dist)
+        y_err_l.append(y_l_dist)
+        y_err_u.append(y_u_dist)
+        n += 1
+
+    plt.scatter(num_list_x, num_list_y, s=0)
+    plt.errorbar(num_list_x, num_list_y, xerr=[x_err_l, x_err_r], yerr=[y_err_l, y_err_u], ls='none', ecolor='black', elinewidth=1)
+    plt.tight_layout()
+    plt.savefig(pwd_figure)
+    plt.close()
+
+
+num_list_x = [1, 2, 3, 4, 5]
+num_list_y = [1, 2, 3, 4, 5]
+err_range_x = [[0.8, 1.1], [1.5, 2.1], [2.9, 3.7], [3.2, 4.1], [4.5, 5.5]]
+err_range_y = [[0.7, 1.3], [1.9, 2.5], [2.9, 3.7], [3.1, 4.4], [4.5, 5.5]]
+pwd_figure = '/Users/songweizhi/Desktop/aaa.png'
+
+PlotMCMC(num_list_x, num_list_y, err_range_x, err_range_y, pwd_figure)

TreeSAK/PlotMcmcNode.py

@@ -0,0 +1,152 @@
+import os
+import glob
+import argparse
+import pandas as pd
+import plotly.express as px
+
+
+PlotMcmcNode_usage = '''
+========================== PlotMcmcNode example commands ==========================
+
+TreeSAK PlotMcmcNode -i McmcTree_op_files -n topo123_nodes.txt -o topo123_age.pdf
+
+# txt file format (-i, tab separated, no header)
+# column 1: mcmc file
+# column 2: node id
+# column 3: mcmc file description
+# column 4: node description
+
+path/to/topo1_clock3_mcmc.txt	t_n171	Topo1	Symbiosis_event_1
+path/to/topo1_clock3_mcmc.txt	t_n151	Topo1	Symbiosis_event_2
+path/to/topo1_clock3_mcmc.txt	t_n131	Topo1	Symbiosis_event_3
+path/to/topo2_clock3_mcmc.txt	t_n171	Topo2	Symbiosis_event_1
+path/to/topo2_clock3_mcm.txtc	t_n171	Topo3	Symbiosis_event_1
+
+===================================================================================
+'''
+
+
+def sep_path_basename_ext(file_in):
+
+    # separate path and file name
+    f_path, file_name = os.path.split(file_in)
+    if f_path == '':
+        f_path = '.'
+
+    # separate file basename and extension
+    f_base, f_ext = os.path.splitext(file_name)
+
+    return f_path, f_base, f_ext
+
+
+def plot_distribution(df_txt, output_plot):
+
+    df = pd.read_table(df_txt, sep=',')
+    run_id_list = df['Setting'].unique()
+    node_id_list = df['Node'].unique()
+
+    # sort dataframe by run id
+    df = df.sort_values(by='Setting', ascending=False)
+
+    plot_width  = 900
+    plot_height = len(run_id_list)*100
+    if plot_height < 360:
+        plot_height = 360
+
+    fig = px.violin(df, x="Value", y="Setting", color="Node", points=False, orientation="h", width=plot_width, height=plot_height).update_layout(xaxis_title="Age", yaxis_title="Settings")
+    if len(node_id_list) == 1:
+        fig.update_traces(side="positive", fillcolor='lightblue', width=1.6, opacity=0.75)
+    else:
+        fig.update_traces(side="positive", fillcolor='rgba(0,0,0,0)', width=1.6)
+
+    fig.update_traces(showlegend=True)
+    fig.layout.template = "simple_white"
+    # fig.layout.width = 700
+    # fig.layout.height = 750
+    # fig.update_xaxes(range=[40, 0])
+    # fig.update_layout(margin_t=10, title_text='Demo', title_x=0.5)
+    fig.write_image(output_plot)
+
+
+def PlotMcmcNode(args):
+
+    node_txt    = args['i']
+    output_plot = args['o']
+
+    if output_plot[-4:] not in ['.PDF', '.pdf']:
+        output_plot = output_plot + '.pdf'
+    op_df_tmp = output_plot + '.txt'
+
+    # read in txt file
+    file_to_node_dict = dict()
+    file_rename_dict = dict()
+    node_rename_dict = dict()
+    not_found_file_set = set()
+    for each_node in open(node_txt):
+        line_split = each_node.strip().split()
+        mcmc_file           = line_split[0]
+        node_id             = line_split[1]
+
+        if os.path.isfile(mcmc_file) is False:
+            not_found_file_set.add(mcmc_file)
+
+        _, mcmc_f_base, _       = sep_path_basename_ext(mcmc_file)
+        node_id_with_file_name  = '%s_____%s' % (mcmc_file, node_id)
+
+        # get file_to_node_dict
+        if mcmc_file not in file_to_node_dict:
+            file_to_node_dict[mcmc_file] = set()
+        file_to_node_dict[mcmc_file].add(node_id)
+
+        # get file_rename_dict
+        if len(line_split) >= 3:
+            file_rename_on_plot = line_split[2]
+            file_rename_dict[mcmc_file] = file_rename_on_plot
+
+        # get node_rename_dict
+        if len(line_split) == 4:
+            node_rename_on_plot = line_split[3]
+            node_rename_dict[node_id_with_file_name] = node_rename_on_plot
+
+    if len(not_found_file_set) > 0:
+        print('The following files not found program exited!')
+        print('\n'.join(sorted(list(not_found_file_set))))
+        exit()
+
+    found_matched_node = False
+    op_df_tmp_handle = open(op_df_tmp, 'w')
+    op_df_tmp_handle.write('Value,Node,Setting\n')
+    for mcmc_file in file_to_node_dict:
+        current_node_set = file_to_node_dict.get(mcmc_file, set())
+        file_name_to_plot = file_rename_dict.get(mcmc_file, mcmc_file)
+        mcmc_df = pd.read_table(mcmc_file, index_col=0)
+        for each_col in mcmc_df:
+            if each_col in current_node_set:
+                node_desc_to_plot = node_rename_dict.get(('%s_____%s' % (mcmc_file, each_col)), each_col)
+                found_matched_node = True
+                value_list = mcmc_df[each_col].values
+                for each_value in value_list:
+                    op_df_tmp_handle.write('%s,%s,%s\n' % (each_value, node_desc_to_plot, file_name_to_plot))
+    op_df_tmp_handle.close()
+
+    if found_matched_node is False:
+        print('Provided node(s) not found, program exited!')
+        exit()
+
+    # plot distribution
+    plot_distribution(op_df_tmp, output_plot)
+
+    # remove tmp files
+    os.system('rm %s' % op_df_tmp)
+
+    # final report
+    print('Plot exported to %s, done!' % output_plot)
+
+
+if __name__ == '__main__':
+
+    PlotMcmcNode_parser = argparse.ArgumentParser()
+    PlotMcmcNode_parser.add_argument('-i',  required=True,                  help='input txt file')
+    PlotMcmcNode_parser.add_argument('-o',  required=True,                  help='output plot')
+    args = vars(PlotMcmcNode_parser.parse_args())
+    PlotMcmcNode(args)