hogprof 0.3.0__py3-none-any.whl

hogprof/extract_levels_hits.py

@@ -0,0 +1,518 @@
+#import profiler
+print("\nNote: this script requires the hogprof environment\n")
+import argparse
+import pandas as pd
+import os
+import sys
+from time import time
+from datetime import datetime
+import numpy as np
+import seaborn as sns
+import re
+from functools import lru_cache
+import csv
+from tqdm import tqdm
+from scipy.spatial.distance import squareform
+from scipy.cluster.hierarchy import linkage
+# test to see if this fixes heatmap issue
+sys.setrecursionlimit(10000)
+from ete3 import NCBITaxa, Tree
+ncbi = NCBITaxa()
+
+### get current time
+start_time_secs = time() 
+start_time = datetime.fromtimestamp(start_time_secs).strftime('%Y-%m-%d %H:%M:%S')
+### print start time human readable
+print(f"Script started at: {start_time}\n")
+
+def add_profiler_path(profiler_path):
+    """Add the directory containing profiler.py to Python's sys.path"""
+    if not os.path.exists(profiler_path):
+        raise FileNotFoundError(f"Profiler path does not exist: {profiler_path}")
+    profiler_dir = os.path.dirname(os.path.abspath(profiler_path))
+    if profiler_dir in sys.path:
+        sys.path.remove(profiler_dir)
+    sys.path.insert(0, profiler_dir)
+
+def create_directory(dirpath):
+    if not os.path.exists(dirpath):
+        os.mkdir(dirpath)
+    return dirpath
+
+'''https://stackoverflow.com/questions/46135839/auto-detect-the-delimiter-in-a-csv-file-using-pd-read-csv'''
+def get_delimiter(file_path, bytes = 4096):
+    sniffer = csv.Sniffer()
+    data = open(file_path, "r").read(bytes)
+    delimiter = sniffer.sniff(data).delimiter
+    return delimiter
+
+def get_hashids(subhogs_table, fam2orthoxml_file):
+    sep = get_delimiter(subhogs_table)
+    subhogs_df = pd.read_csv(subhogs_table, sep=sep)
+    if 'hashid' in subhogs_df.columns:
+        return subhogs_table
+    print(f"Extracting hashids from file: {fam2orthoxml_file}")
+    subhogid_col = "subhogid"
+    if subhogid_col not in subhogs_df.columns:
+        raise ValueError(f"Column '{subhogid_col}' not found in {subhogs_table}. Please ensure the input file has a column named '{subhogid_col}'.")
+    subhogs_df.set_index(subhogid_col, inplace=True)
+    ### read in the fam2orthoxml table generated by HogProf lshbuilder and the tree file 
+    fam2orthoxml_df = pd.read_csv(fam2orthoxml_file)
+    # build mapping: subhog_id -> hash row index
+    subhog_to_hashid = {subhog: idx for idx, subhog in enumerate(fam2orthoxml_df['subhog_id'])}
+    # attach hash ids
+    subhogs_df['hashid'] = subhogs_df.index.map(subhog_to_hashid)
+    # outputqueryfile
+    outputqueryfile = subhogs_table.replace(f"{os.path.splitext(subhogs_table)[1]}",
+                                            f"_queries{os.path.splitext(subhogs_table)[1]}")
+    # save new file with hash ids
+    subhogs_df.to_csv(outputqueryfile, sep=sep)
+    return outputqueryfile
+
+def get_hashids_by_subhogid(subhog_ids, fam2orthoxml_file):
+    fam2orthoxml_df = pd.read_csv(fam2orthoxml_file)
+    subhog_to_hashid = {subhog: idx for idx, subhog in enumerate(fam2orthoxml_df['subhog_id'])}
+    hashids = [subhog_to_hashid[subhog] for subhog in subhog_ids if subhog in subhog_to_hashid]
+    return hashids
+
+'''function to add some info to the extracted hits about how the origin and strength of coevolution'''
+def find_coevolution_origin(hits_df, treepath):
+    print("Finding coevolution origins...")
+    tree = Tree(treepath, format=1)
+    node_lookup = {int(n.name): n for n in tree.traverse() if n.name}
+    def edge_to_pair(row):
+        return frozenset({
+                re.search(r'_(HOG:[^_.]+)', row["query_hog"]).group(1),
+                re.search(r'_(HOG:[^_.]+)', row["target_hog"]).group(1)
+            })
+    
+    @lru_cache(maxsize=None)
+    def lineage(taxid):
+        node = node_lookup.get(int(taxid))
+        #print("taxid:", taxid, "node:", node)
+        if node is None:
+            return frozenset()
+        return frozenset(int(n.name) for n in node.get_ancestors()) | {int(taxid)}
+
+    def highest_non_overlapping_taxa(taxids):
+        taxids = sorted(set(map(int, taxids)))
+        keep = []
+        for t in taxids:
+            # Remove t if any other observed taxon is an ancestor of it
+            if not any(
+                other != t and other in lineage(t)
+                for other in taxids):
+                keep.append(t)
+        return keep
+    # 1 add column for checking 
+    hits_df['pair'] = hits_df.apply(edge_to_pair, axis=1)
+    # 2 group by pair so that each pair has a list of taxa it appears in
+    hits_grouped_df = hits_df.groupby('pair')['taxid'].apply(list).reset_index(name='taxid_list')
+    # add column with number of taxids per pair
+    hits_grouped_df['families_score'] = hits_grouped_df['taxid_list'].apply(len)
+    # 3 find highest rank taxon (or taxa) for each list 
+    hits_grouped_df["origin_taxids"] = hits_grouped_df["taxid_list"].apply(highest_non_overlapping_taxa)
+    # 4 add to hits_df column '1st_occurence' with True for the rows that have a taxid in origin_taxids and False otherwise
+    hits_df = hits_df.merge(hits_grouped_df[['pair', 'families_score','origin_taxids']], on='pair', how='left')
+    hits_df['1st_occurence'] = hits_df.apply(lambda row: row['taxid'] in row['origin_taxids'], axis=1)
+    # 5 remove 'pair' column and 'origin_taxids' column
+    hits_df.drop(columns=['pair', 'origin_taxids'], inplace=True)
+    return hits_df
+
+'''use hashes directly to extract hits'''
+def extract_hits_from_hashes(lshforestpath, hashes_h5, treepath, outputfile, profiler_path,
+                                   allvsall, k, subhogs_table, thresholds_df, treeroot=""):
+    # Add profiler directory to path and import
+    add_profiler_path(profiler_path)
+    import profiler
+    print("Imported profiler from:")
+    print(profiler.__file__)
+    subhogs_df = pd.read_csv(subhogs_table, sep=get_delimiter(subhogs_table), index_col=0)
+    ## index label is subhogid
+    subhogs_df.index.name = 'subhogid'
+    print(subhogs_df.head())
+    subhog_to_hashid = subhogs_df['hashid'].to_dict()
+    ### check that hashid were indeed extracted
+    if any(pd.isnull(list(subhog_to_hashid.values()))):
+        print("subhog_to_hashid:",subhog_to_hashid)
+        raise ValueError("Some subHOGs do not have hashid. Please check the queries file.")
+    hashid_to_subhog = {v: k for k, v in subhog_to_hashid.items()}
+    print("\nBuilding the profiler")
+    p = profiler.Profiler(lshforestpath = lshforestpath, 
+                        hashes_h5= hashes_h5, 
+                        oma = False , 
+                        nsamples = 256 ,
+                        mastertree = treepath,
+                        slicesubhogs = True
+                        )
+    p.slicesubhogs = False ### setting as False to use original hogid2fam function in profiler.py
+    # the reason for this is using family IDS vs hashids for querying. the original function used hashids,
+    # sometimes in levels we want to query everything within a family though.
+    print("Pulling hashes...")
+    raw_profiles = p.pull_hashes(subhog_to_hashid.values()) # returns hashid to hashsig dict
+    ### here check that the expected number of hashes was pulled
+    if len(raw_profiles)!= len(hashid_to_subhog):
+        print("Warning! Could not pull all the hashes!")
+    subhogs_without_hash = []
+    for subhog, hashid in subhog_to_hashid.items():
+        if hashid not in raw_profiles.keys():
+            subhogs_without_hash.append(subhog)
+    if len(subhogs_without_hash)>0:
+        print("Examples:", subhogs_without_hash[:5])
+    profiles = {
+        hashid_to_subhog[h]: {
+            'hashsig': raw_profiles[h],
+            'hashid': h
+        }
+        for h in raw_profiles
+        if h in hashid_to_subhog
+    }
+    profiles_df = pd.DataFrame.from_dict(profiles, orient='index')
+    profiles_num = len(profiles_df.index.unique())
+    ### checking for empty / degenerate /invalid signatures
+    profiles_df['sigsum'] = profiles_df.hashsig.map(lambda x : np.sum(x.digest()))
+    profiles_df = profiles_df[profiles_df.sigsum>0]
+    #print(f'{len(profiles.index.unique())} HOGs have valid signatures')
+    if profiles_num != len(profiles_df.index.unique()):
+        print("Warning: invalid hash signatures detected!")
+
+    def get_candidates(p, query_sig, k):
+        # query LSH forest for candidate hash ids - get top k
+        candidate_subhogids = [target_hog[target_hog.find('_')+1:] for target_hog in p.forest.query(query_sig, k)]
+        candidate_hashids = [p.subhog_to_hashids_dict[target_hog] for target_hog in candidate_subhogids]
+        local_hashid_to_subhog = {int(h): s for h, s in zip(candidate_hashids, candidate_subhogids)}
+        # pull candidate signatures (dict: hashid -> MinHash)
+        candidate_hashes = p.pull_hashes(candidate_hashids)
+        return candidate_hashids, local_hashid_to_subhog, candidate_hashes
+    
+    def get_scores(candidate_hashids,candidate_hashes, query_sig, level_threshold):
+        scores = {}
+        for hashid in candidate_hashids: ### candidate_hashids are the closest but they are not ordered
+            sig = candidate_hashes[hashid]
+            this_score = sig.jaccard(query_sig)
+            if this_score >= level_threshold:
+                scores[hashid] = this_score
+        return scores
+    
+    ### get jaccard scores too
+    rows = []
+    counter = 0
+    no_hits_subhogs = []
+    ### data structure_for all_vs_all
+    taxa_networks_dict = {}
+    network_id_to_hid = {}
+    level_to_T_dict = {}
+    taxid_to_taxname_dict = thresholds_df['taxname'].to_dict()
+    print("Querying each HOG...")
+    for query_subhog_id, querydict in profiles.items():
+        ### clean hogids e.g. 14_HOG:0004777_537 where 14 is the internal taxid (bin in thresholds) 
+        # and 537 is the internal hog id from the orthoxml transversal
+        query_level = int(query_subhog_id.split("_")[0])
+        hog_id_str = query_subhog_id.split("_")[1]
+        try:
+            level_threshold = thresholds_df['empirical_t'].loc[query_level]
+        except:
+            level_threshold = 0 ### take everything if no T is specified
+            print(f"Warning: no T defined for {query_level}")
+        level_to_T_dict[query_level] = level_threshold
+        current_taxon = thresholds_df['taxname'].loc[query_level]
+        query_sig = querydict['hashsig']
+        query_hashid = querydict['hashid']
+        # query LSH forest for candidate hash ids - get top k
+        candidate_hashids, local_hashid_to_subhog, candidate_hashes = get_candidates(p, query_sig, k)
+        scores = get_scores(candidate_hashids,candidate_hashes, query_sig, level_threshold)
+        ### it is possible that none pass the threshold
+        if len(scores) == 0:
+            no_hits_subhogs.append(query_subhog_id)
+            continue
+        ### it is also possible that all pass the threshold, so we need to check more (only check once more with double k)
+        if len(scores) == len(candidate_hashids):
+            candidate_hashids, local_hashid_to_subhog, candidate_hashes = get_candidates(p, query_sig, 2*k)
+            scores = get_scores(candidate_hashids,candidate_hashes, query_sig, level_threshold)
+        # sort candidate scores descending
+        scored_pairs = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+        '''
+        # save info for allvsall
+        if allvsall:
+            if query_level not in taxa_networks_dict.keys():
+                network_id_to_hid = {}
+            else:
+                network_id_to_hid = taxa_networks_dict[query_level]
+            # add new info
+            network_id_to_hid[query_subhog_id] = query_hashid
+            taxa_networks_dict[query_level] = network_id_to_hid
+        '''
+        # build rows: one row per query-target pair
+        for hid, jaccard in scored_pairs:
+            target_subhog = local_hashid_to_subhog[hid]
+
+            # target_subhog expected like "14_HOG:0004777_537" or "14_751313"
+            target_hog_clean = target_subhog  # already in desired form
+            try:
+                target_level = int(target_hog_clean.split("_")[0])
+            except Exception:
+                # if parsing fails, skip
+                print(f"Warning: cannot parse level from {target_hog_clean}; skipping")
+                continue
+            try:
+                target_hog_id_str = target_hog_clean.split("_")[1]
+            except Exception:
+                target_hog_id_str = ""
+            # don't keep self hits:
+            if query_subhog_id ==target_hog_clean:
+                continue
+            # only keep same-level hits so threshold makes sense
+            if target_level != query_level:
+                continue
+            # flag same-family (compare the id string parts)
+            same_fam = (hog_id_str == target_hog_id_str)
+            # append row
+            rows.append({
+                'query_hog': query_subhog_id,
+                'target_hog': target_hog_clean,
+                'jaccard': jaccard,
+                'empirical_t': level_threshold,
+                'taxname': current_taxon,
+                'taxid': query_level,
+                'same_fam': same_fam
+            })
+    taxa_hsig_dict ={}
+    '''
+    ### change info for allvsall
+    if allvsall:
+        for level, network_id_to_hid in taxa_networks_dict.items():
+            all_raw_profiles = p.pull_hashes(network_id_to_hid.values()) # returns hashid to hashsig dict
+            ### turn into subhog to hashsig
+            network_id_to_hsig = {subhogid:all_raw_profiles[hid] for subhogid, hid in  network_id_to_hid.items()}
+            taxa_hsig_dict[level] = [level_to_T_dict[level],network_id_to_hsig]
+        '''
+    hits_df = pd.DataFrame(rows)
+
+    if allvsall:
+        target_hogs_per_level = hits_df.groupby('taxid')['target_hog'].unique().to_dict()
+        for level, target_hogs in target_hogs_per_level.items():
+            query_hogs = hits_df[hits_df['taxid'] == level]['query_hog'].unique()
+            target_hogs = np.concatenate([target_hogs, query_hogs])
+            level_hashids = get_hashids_by_subhogid(list(target_hogs), fam2orthoxml_file)
+            taxa_hsig_dict[level] = [level_to_T_dict[level], p.pull_hashes(level_hashids)]
+    
+    if len(no_hits_subhogs)>0:
+        print(f"No hits found for {len(no_hits_subhogs)} subHOGs")
+        print("Examples:", no_hits_subhogs[:5])
+    else:
+        print("All subHOGs had hits")
+        print("Total hits found:", len(hits_df))
+        print(hits_df.head())
+
+    ### add column with coevolution origin
+    hits_df = find_coevolution_origin(hits_df, treepath)
+    # remove taxid column if it exists
+    if 'taxid' in hits_df.columns:
+        hits_df.drop(columns=['taxid'], inplace=True)
+    # replace nan taxname with treeroot if specified
+    if treeroot:
+        hits_df['taxname'] = hits_df['taxname'].replace(r'^\s*$', np.nan, regex=True)
+        hits_df['taxname'] = hits_df['taxname'].fillna(treeroot)
+        taxid_to_taxname_dict[0] = treeroot
+    print(hits_df.head())
+    ### save to file
+    hits_df.to_csv(outputfile, index=False)
+    print("Created file:", outputfile)
+    return taxa_hsig_dict, taxid_to_taxname_dict
+
+### from Dave's jupyter notebook, with changes
+def compare_all_vs_all(taxa_hsig_dict, outputfile):
+    print("Performing all vs all comparison...")
+    for level, [level_T, network_id_to_hsig] in taxa_hsig_dict.items():
+        out_eps = outputfile.replace(".csv",f"{level}.eps")
+        ids = list(network_id_to_hsig.keys())
+        sigs = [network_id_to_hsig[i] for i in ids]
+        jkern = np.array([
+            [sig1.jaccard(sig2) if i != j else 0
+            for j, sig2 in enumerate(sigs)]
+            for i, sig1 in enumerate(sigs)
+        ])
+        # convert to distance for clustering
+        dist = 1 - jkern
+        ### now apply threshold
+        jkern_plot = jkern.copy()
+        jkern_plot[jkern_plot < level_T] = 0
+        print(jkern)
+        g = sns.clustermap(
+            dist,
+            xticklabels=ids,
+            yticklabels=ids,
+            figsize=(20, 20),
+            cmap="viridis_r"  # reversed so small distance = high similarity
+        )
+        g.savefig(out_eps, format='eps')
+        print(f"Created file: {out_eps}")
+
+'''original idea to get the matrix'''
+def jaccard_similarity_matrix(sigs):
+    return np.array([[sig1.jaccard(sig2) for sig2 in sigs]
+            for sig1 in sigs])
+
+'''faster way to get the matrix'''
+def jaccard_similarity_matrix(sigs):
+    ### Get the hash values of all signatures as a 2D array
+    hashvalues = np.array([sig.hashvalues for sig in sigs])
+    #print("hashvalues shape:", hashvalues.shape)
+    ### Compare all pairs of signatures
+    # Vectorized pairwise Jaccard from matching MinHash values 
+    jkern = np.mean(hashvalues[:, np.newaxis, :, :] ==
+                    hashvalues[np.newaxis, :, :, :],
+                    axis=(2, 3))
+    #print("jkern shape:", jkern.shape)
+    return jkern
+
+'''visualize the similarity matrix as a heatmap with hierarchical clustering'''
+def make_similarity_heatmap(jkern, ids, taxname, outputfile,format="png"):
+    dist = 1 - jkern
+    # Convert square distance matrix to condensed form for scipy
+    condensed_dist = squareform(dist, checks=False)
+    # Hierarchical clustering from the precomputed distances
+    linkage_matrix = linkage(condensed_dist, method="average")
+    g = sns.clustermap(
+        jkern,
+        row_linkage=linkage_matrix,
+        col_linkage=linkage_matrix,
+        xticklabels=ids,
+        yticklabels=ids,
+        figsize=(20, 20),
+        cmap="viridis",
+        vmin=0,
+        vmax=1
+    )
+    ### only run eps for quite small datasets
+    if format == "eps":
+        outputfile = outputfile.replace(".csv", f"_{taxname}.eps")
+        g.savefig(outputfile, format="eps", bbox_inches='tight')
+    ### png is a good default for larger datasets
+    elif format == "png":
+        outputfile = outputfile.replace(".csv", f"_{taxname}.png")
+        g.savefig(outputfile, dpi=600, format="png", bbox_inches='tight')
+    else:
+        raise ValueError(f"Unsupported format: {format}")
+    print(f"Created file: {outputfile}")
+
+'''compare all vs all for each taxonomic level'''
+def compare_all_vs_all(taxa_hsig_dict, taxid_to_taxname_dict,outputfile, heatmap=False):
+    print("\nPerforming all vs all comparison (untested)...")
+    #print(taxa_hsig_dict)
+    print(f"Using {len(taxa_hsig_dict)} taxonomic levels for comparison")
+    n_profiles = sum(len(network_id_to_hsig)
+                     for level_T, network_id_to_hsig in taxa_hsig_dict.values())
+    print(f"Using {n_profiles} profiles for comparison")
+    for level, [level_T, network_id_to_hsig] in tqdm(taxa_hsig_dict.items(),
+                                                     desc="Taxonomic levels"):
+        taxname = taxid_to_taxname_dict.get(level, f"taxid_{level}")
+        ids = list(network_id_to_hsig.keys())
+        if len(ids) < 2:
+            print(f"Skipping {level}: only {len(ids)} profile(s)")
+            continue
+        sigs = [network_id_to_hsig[i] for i in ids]
+        # similarity matrix
+        jkern = jaccard_similarity_matrix(sigs)
+        np.fill_diagonal(jkern, 1.0)
+        # distance for clustering
+        #dist = 1 - jkern
+        # --- threshold ONLY for display ---
+        #jkern_plot = jkern.copy()
+        #jkern_plot[jkern_plot < level_T] = 0
+        # threshold only for exported similarity matrix
+        sim_df = pd.DataFrame(jkern, index=ids, columns=ids)
+        sim_df[sim_df < level_T] = 0
+        ### save to csv
+        sim_csv = outputfile.replace(".csv", f"_{level}_similarity_matrix.csv")
+        sim_df.to_csv(sim_csv)
+        print(f"Created similarity matrix file: {sim_csv}")
+        if heatmap:
+            make_similarity_heatmap(jkern, ids, taxname, outputfile)
+
+def main(lshforestpath, hashes_h5, treepath, outputfile, profiler_path, queries_file,thresholds_file,
+         allvsall = False, heatmap=False, k= 1000, treeroot=""):
+    ### print arguments
+    print("\n📋 Running with the following parameters:")
+    print(f"  • LSH Forest path: {lshforestpath}")
+    print(f"  • Hashes HDF5: {hashes_h5}")
+    print(f"  • Tree path: {treepath}")
+    print(f"  • Output file: {outputfile}")
+    print(f"  • Profiler path: {profiler_path}")
+    print(f"  • Queries file: {queries_file}")
+    print(f"  • All vs all: {allvsall}")
+    print(f"  • Heatmap: {heatmap}")
+    print(f"  • k: {k}")
+    print(f"  • thresholds table: {thresholds_file}")
+    print("\n")
+
+    ### if thresholds are given, take them into account
+    thresholds_df = pd.read_csv(thresholds_file)[['bin','empirical_t','taxname']]
+    thresholds_df.set_index('bin', inplace=True)
+
+    create_directory(os.path.dirname(outputfile))
+
+    ### get hits from hashes
+    taxa_hsig_dict, taxid_to_taxname_dict = extract_hits_from_hashes(lshforestpath, hashes_h5, treepath, outputfile, profiler_path,
+                                              allvsall, k, queries_file, thresholds_df, treeroot=treeroot)
+    if heatmap and not allvsall:
+        print("Warning: heatmap option is only relevant for allvsall mode. Ignoring heatmap option.")
+    if allvsall:
+        compare_all_vs_all(taxa_hsig_dict, taxid_to_taxname_dict, heatmap=heatmap, outputfile=outputfile.replace(".csv", "_allvsall.csv"))
+
+## Command line argument parsing'''
+def parse_args():
+    """
+    Parse command line arguments.
+    """
+    print("Parsing command line arguments")
+    parser = argparse.ArgumentParser(description="Extract hits from the profiler for levels mode, per taxon. Queries are subHOGs that can be found in the fam2orthoxml file.")
+    parser.add_argument("-i", "--input", required=True, help="HogProf output folder")
+    parser.add_argument("-o", "--outputfile", required=True, help="Output CSV file")
+    parser.add_argument("--queries_file", required=True, help="Query subHOG table")
+    parser.add_argument("--thresholds", required=True, help="Threshold CSV from Step 3")
+    parser.add_argument("--profilerpath", default="profiler.py")
+    parser.add_argument("--k", type=int, default=1000, help="Number of top hits to extract for each query.")
+    parser.add_argument("--allvsall",type=bool, help="Run all vs all mode additionally to the main hit extraction (may take a long time).", default=False)
+    parser.add_argument("--heatmap",type=bool, help="Generate a heatmap of the all vs all comparisons.", default=False)
+    parser.add_argument("--treeroot", default="", help="Root of the tree to be used in output table as a taxon name. " \
+    "The root is often called 'internal_0' so by default, if there is no taxon name it would get an empty string in " \
+    "the results table. Specify it here to avoid this")
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    # Parse command line arguments
+    args = parse_args()
+    #print("Are arguments fixed?")
+    inputfolder = args.input
+    lshforestpath = os.path.join(inputfolder, "newlshforest.pkl")
+    hashes_h5 = os.path.join(inputfolder, "hashes.h5")
+    treepath = os.path.join(inputfolder, "speciestree.nwk")
+    treepath = os.path.join(inputfolder, "reformatted_tree.nwk")
+    fam2orthoxml_file = os.path.join(inputfolder, "fam2orthoxml.csv")
+    queries_file = args.queries_file
+    # special case for unknown hashids (subhogid per line, identical to fam2orthoxml)
+    queries_file = get_hashids(queries_file, fam2orthoxml_file)
+    # Run the main function with the parsed arguments
+    main(
+        lshforestpath=lshforestpath,
+        hashes_h5=hashes_h5,
+        treepath=treepath,
+        outputfile=args.outputfile,
+        profiler_path = args.profilerpath,
+        allvsall=args.allvsall,
+        heatmap=args.heatmap,
+        k=args.k,
+        queries_file = queries_file,
+        thresholds_file=args.thresholds,
+        treeroot=args.treeroot
+    )
+
+### print end time
+end_time_secs = time()
+end_time = datetime.fromtimestamp(end_time_secs).strftime('%Y-%m-%d %H:%M:%S')
+print(f"Script finished at: {end_time}\n")
+execution_time = end_time_secs - start_time_secs
+### print execution time but reformat it to be human readable
+print(f"Total execution time: {round(execution_time, 2)} seconds\n")
+print("\nDONE!\n")