FunVIP 0.3.20__py3-none-any.whl

src/tool.py

@@ -0,0 +1,309 @@
+import time, re
+from Bio import SeqIO
+from Bio.Seq import Seq
+
+# from .logger import Mes
+import logging
+from functools import lru_cache
+import sys, os
+import shutil
+import platform
+from unidecode import unidecode
+
+
+# Maybe we should move this to /funvip/src/toolbox/ and split these to multiple files
+def initialize_path(path):
+    global genus_file
+    genus_file = path.genusdb
+
+
+def mkdir(path):
+    if os.path.exists(path) == False:
+        os.makedirs(path)
+
+
+def union_funinfo_list(funinfo_list1, funinfo_list2):
+    hash_list = [x.hash for x in funinfo_list1]
+    for funinfo in funinfo_list2:
+        if not funinfo.hash in hash_list:
+            funinfo_list1.append(funinfo)
+
+    return funinfo_list1
+
+
+# if string is not ascii, automatically solve it with unicode library
+def manage_unicode(string, column="", row=""):
+    try:
+        string.encode("ascii")
+        return string
+    except:
+        pass
+
+    if row != "":
+        row_string = f"{row}th row "
+    else:
+        row_string = ""
+
+    if column != "":
+        column_string = f"in {column} column, "
+    else:
+        column_string = ""
+
+    logging.info(
+        f"Illegal unicode character found {column_string}{row_string}: {unidecode(string)}. Trying flexible solve"
+    )
+
+    try:
+        return unidecode(string)
+    except:
+        logging.error(
+            f"Flexible solve failed to {string}. Please change the cell with available ascii strings"
+        )
+        raise Exception
+
+
+@lru_cache(maxsize=10000)
+def get_genus_species(
+    string,
+    endwords=[
+        "small",
+        "18S",
+        "ribosomal",
+        "internal",
+        "gene",
+        "ITS",
+        "5.8S",
+        "voucher",
+        "strain",
+        "beta",
+        "tubulin",
+    ],
+    genus_list=None,
+):
+    return_genus = ""
+    return_species = ""
+
+    # en for enumeratable object (splited string)
+    if genus_list is None:
+        with open(genus_file, "r") as f:
+            genus_list = f.read().splitlines()
+
+    en = string.replace(" ", "_").split("_")
+
+    for genus in genus_list:
+        for n, i in enumerate(en):
+            if i == genus:  # or i == genus[0] or i == genus[0]+".":
+                return_genus = i
+                try:
+                    if en[n + 2] in ["var", "var.", "f", "f.", "nom", "nom."]:
+                        try:
+                            en[n + 3]
+                            if not (en[n + 3] in endwords):
+                                return_species = " ".join(
+                                    [en[n + 1], en[n + 2], en[n + 3]]
+                                )
+                            else:
+                                return_species = " ".join([en[n + 1], en[n + 2]])
+                        except:
+                            return_species = " ".join([en[n + 1], en[n + 2]])
+
+                    else:
+                        if en[n + 1] in ["aff", "aff.", "cf", "cf."]:
+                            try:
+                                if not (en[n + 2] in endwords):
+                                    return_species = " ".join([en[n + 1], en[n + 2]])
+                                else:
+                                    return_species = en[n + 1]
+                            except:
+                                return_species = en[n + 1]
+
+                        elif en[n + 1] in ["sp", "sp."]:
+                            try:
+                                if not (en[n + 2] in endwords):
+                                    try:
+                                        int(en[n + 2])
+                                        return_species = " ".join(
+                                            [en[n + 1], en[n + 2]]
+                                        )
+                                    except:
+                                        return_species = en[n + 1]
+                                else:
+                                    return_species = en[n + 1]
+                            except:
+                                return_species = en[n + 1]
+
+                        else:
+                            if not (en[n + 1] in endwords):
+                                return_species = en[n + 1]
+                            else:
+                                return_species = "NaN"
+
+                except:
+                    try:
+                        if en[n + 1] in ["sp", "sp.", "aff", "aff."]:
+                            try:
+                                en[n + 2]
+                                if not (en[n + 2] in endwords):
+                                    return_species = " ".join([en[n + 1], en[n + 2]])
+                                else:
+                                    return_species = en[n + 1]
+                            except:
+                                return_species = en[n + 1]
+                        else:
+                            if not (en[n + 1] in endwords):
+                                return_species = en[n + 1]
+                            else:
+                                return_species = "NaN"
+                    except:
+                        return_species = "NaN"
+
+    return return_genus, return_species
+
+
+# get id from string by regex match
+# return message as error, not print it
+@lru_cache(maxsize=10000)
+def get_id(string, id_list):
+    id_set = set()
+    id_ = ""
+    for regex in id_list:
+        if re.search(regex, string):
+            id_set.add(re.search(regex, string).group(0))
+            # longest match as a best match
+            if len(re.search(regex, string).group(0)) > len(id_):
+                id_ = re.search(regex, string).group(0)
+        else:
+            pass
+
+    if len(id_set) >= 2:
+        logging.warning(
+            f"[Warning] Ambiguous regex match found. {id_} selected among {id_set}"
+        )
+
+    if id_ == "":
+        logging.warning(
+            f"[Warning] Cannot find regex match from {string}, using default fasta name"
+        )
+        id_ = string
+
+    id_ = str(id_)
+    return id_
+
+
+# select FI with specific datatype from FI_list
+def select(funinfo_list, datatype):
+    tmp_list = []
+    for funinfo in funinfo_list:
+        if funinfo.datatype == datatype:
+            tmp_list.append(funinfo)
+
+    return tmp_list
+
+
+def excel_sheetname_legal(string):
+    newick_illegal = ["'", "[", "]", ":", "*", "?", "/", "\\"]
+    for i in newick_illegal:
+        string = string.replace(i, "")
+
+    string = string.replace("  ", " ")
+    string = string.replace(" ", "_")
+
+    return string
+
+
+# moves newick files generated in tree building process to appropriate location
+def cleanup_tree(path):
+    # move result files to each of the folders
+    mkdir(f"{path.out_tree}/hash")
+    hash_files = [
+        f
+        for f in os.listdir(f"{path.out_tree}")
+        if f.startswith("hash_") and f.endswith(".nwk")
+    ]
+    for file in hash_files:
+        try:
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/hash/{file}")
+        except:
+            os.remove(f"{path.out_tree}/hash/{file}")
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/hash/{file}")
+
+    mkdir(f"{path.out_tree}/original")
+    hash_files = [
+        f for f in os.listdir(f"{path.out_tree}") if f.endswith("_original.nwk")
+    ]
+    for file in hash_files:
+        try:
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/original/{file}")
+        except:
+            os.remove(f"{path.out_tree}/original/{file}")
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/original/{file}")
+
+
+# moves svg files generated in CAT-V to appropriate location
+def cleanup_tree_image(path):
+    # move result files to each of the folders
+    mkdir(f"{path.out_tree}/hash")
+    hash_files = [
+        f
+        for f in os.listdir(f"{path.out_tree}")
+        if f.startswith("hash_") and f.endswith(".svg")
+    ]
+    for file in hash_files:
+        try:
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/hash/{file}")
+        except:
+            os.remove(f"{path.out_tree}/hash/{file}")
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/hash/{file}")
+
+    mkdir(f"{path.out_tree}/original")
+    original_files = [
+        f for f in os.listdir(f"{path.out_tree}") if f.endswith("_original.svg")
+    ]
+    for file in original_files:
+        try:
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/original/{file}")
+        except:
+            os.remove(f"{path.out_tree}/original/{file}")
+            shutil.move(f"{path.out_tree}/{file}", f"{path.out_tree}/original/{file}")
+
+
+# Change step string into numbered step
+def index_step(step):
+    # Declaring available steps
+    step_list = [
+        "setup",
+        "search",
+        "cluster",
+        "align",
+        "trim",
+        "concatenate",
+        "modeltest",
+        "tree",
+        "visualize",
+        "report",
+    ]
+
+    try:
+        return step_list.index(step.lower().strip())
+    except:
+        logging.error(f"DEVELOPMENTAL ERROR, INVALID STEP {step} USED")
+        raise Exception
+
+
+def check_avx():
+    return platform.machine() == "x86_64" or platform.machine() == "AMD64"
+
+
+# Return logging
+def get_level(level):
+    if level == 3:
+        return logging.DEBUG
+    elif level == 2:
+        return logging.INFO
+    elif level == 1:
+        return logging.WARNING
+    elif level == 0:
+        return logging.ERROR
+    else:
+        print(f"DEVELOPMENTAL ERROR ON MANAGING VERBOSE LEVEL {level}")
+        raise Exception

src/tree.py

@@ -0,0 +1,222 @@
+from funvip.src import ext, hasher
+from funvip.src.opt_generator import opt_generator
+from copy import deepcopy
+import multiprocessing as mp
+import logging
+import shutil
+import os
+
+
+def pipe_tree(V, path, opt, model_dict):
+    # for tree, use hash dict with genus and species information
+    tree_hash_dict = hasher.encode(V.list_FI, newick=True)
+
+    # remove tree files already exists to prevent error
+    try:
+        for file in [f for f in os.listdir(path.tmp) if f.endswith(".nwk")]:
+            os.remove(f"{path.tmp}/{file}")
+    except:
+        pass
+
+    try:
+        for file in [f for f in os.listdir(path.out_tree) if f.endswith(".nwk")]:
+            os.remove(f"{path.out_tree}/{file}")
+    except:
+        pass
+
+    try:
+        for file in [
+            f for f in os.listdir(f"{path.out_tree}/original/") if f.endswith(".nwk")
+        ]:
+            os.remove(f"{path.out_tree}/original/{file}")
+    except:
+        pass
+
+    try:
+        for file in [
+            f for f in os.listdir(f"{path.out_tree}/hash/") if f.endswith(".nwk")
+        ]:
+            os.remove(f"{path.out_tree}/hash/{file}")
+    except:
+        pass
+
+    fasttree_opt = []  # for multiprocessing on fasttree
+
+    tree_dataset = deepcopy(V.dict_dataset)
+
+    # Before drawing tree, finalize datasets
+    remove_dataset = []
+    for group in tree_dataset:
+        for gene in tree_dataset[group]:
+            # draw tree only when outgroup sequence exists
+            if tree_dataset[group][gene].list_og_FI == 0:
+                logging.warning(
+                    f"Passing tree construction of {group} {gene} dataset because no outgroup available"
+                )
+                # tree_dataset[group].pop(gene, None)
+                remove_dataset.append((group, gene))
+
+    for group, gene in remove_dataset:
+        tree_dataset[group].pop(gene, None)
+
+    # To indicate single genes for each group
+    singlegene_dict = {}
+
+    # Draw phylogenetic trees for each dataset
+    for group in tree_dataset:
+        singlegene_flag = 0
+        singlegene = ""
+        # For single gene, we don't have to draw tree multiple times
+        if len(tree_dataset[group].keys()) == 2:
+            singlegene_flag = 1
+            singlegene = list(set(tree_dataset[group].keys()) - {"concatenated"})[0]
+            singlegene_dict[group] = deepcopy(singlegene)
+
+        for gene in tree_dataset[group]:
+            # For single gene, skip concatenated
+            if gene == "concatenated" and singlegene_flag == 1:
+                logging.info(
+                    f"Passing tree construction of {group} {gene} because single gene detected. Will copy from concatenated"
+                )
+            # Else for each gene
+            elif gene != "concatenated":
+                # If not trimming use this
+                if opt.method.tree.lower() == "raxml":
+                    ext.RAxML(
+                        fasta=f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                        out=f"{opt.runname}_{group}_{gene}.nwk",
+                        hash_dict=tree_hash_dict,
+                        path=path,
+                        thread=opt.thread,
+                        bootstrap=opt.bootstrap,
+                        model=model_dict[group][gene],
+                    )
+
+                elif opt.method.tree.lower() == "iqtree":
+                    ext.IQTREE(
+                        fasta=f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                        out=f"{opt.runname}_{group}_{gene}.nwk",
+                        hash_dict=tree_hash_dict,
+                        path=path,
+                        memory=opt.memory,
+                        thread=opt.thread,
+                        bootstrap=opt.bootstrap,
+                        model=model_dict[group][gene],
+                    )
+
+                # if fasttree, append to opt to perform multiprocessing by each tree
+                else:
+                    if not (opt.method.tree.lower() == "fasttree"):
+                        logging.warning(
+                            "Tree construction method not selected, working for default option, FastTree"
+                        )
+                    fasttree_opt.append(
+                        (
+                            f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                            f"{opt.runname}_{group}_{gene}.nwk",
+                            tree_hash_dict,
+                            path,
+                            model_dict[group][gene],
+                        )
+                    )
+            # For concatenated datasets, use partition files
+            else:
+                if opt.method.tree.lower() == "raxml":
+                    ext.RAxML(
+                        fasta=f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                        out=f"{opt.runname}_{group}_{gene}.nwk",
+                        hash_dict=tree_hash_dict,
+                        path=path,
+                        thread=opt.thread,
+                        bootstrap=opt.bootstrap,
+                        partition=f"{path.out_alignment}/{opt.runname}_{group}.partition",
+                        model=model_dict[group][gene],
+                    )
+
+                elif opt.method.tree.lower() == "iqtree":
+                    ext.IQTREE(
+                        fasta=f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                        out=f"{opt.runname}_{group}_{gene}.nwk",
+                        hash_dict=tree_hash_dict,
+                        path=path,
+                        memory=opt.memory,
+                        thread=opt.thread,
+                        bootstrap=opt.bootstrap,
+                        partition=f"{path.out_alignment}/{opt.runname}_{group}.partition",
+                        model=model_dict[group][gene],
+                    )
+
+                else:
+                    if not (opt.method.tree.lower() == "fasttree"):
+                        logging.warning(
+                            "Tree method not selected, working for default option, FastTree"
+                        )
+                    fasttree_opt.append(
+                        (
+                            f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                            f"{opt.runname}_{group}_{gene}.nwk",
+                            tree_hash_dict,
+                            path,
+                            model_dict[group][gene],
+                        )
+                    )
+
+    # for fasttree, perform multiprocessing
+    if opt.method.tree.lower() == "fasttree":
+        # run multiprocessing start
+        if opt.verbose < 3:
+            p = mp.Pool(opt.thread)
+            fasttree_result = p.starmap(ext.FastTree, fasttree_opt)
+            p.close()
+            p.join()
+
+        else:
+            # non-multithreading mode for debugging
+            fasttree_result = []
+            for option in fasttree_opt:
+                fasttree_result.append(ext.FastTree(*option))
+
+    for group in singlegene_dict:
+        # copy concatenated result to single gene
+        shutil.copy(
+            f"{path.out_tree}/{opt.runname}_{group}_{singlegene_dict[group]}.nwk",
+            f"{path.out_tree}/{opt.runname}_{group}_concatenated.nwk",
+        )
+
+        shutil.copy(
+            f"{path.out_tree}/hash_{opt.runname}_{group}_{singlegene_dict[group]}.nwk",
+            f"{path.out_tree}/hash_{opt.runname}_{group}_concatenated.nwk",
+        )
+
+    ## Decode alignments and trimmed alignments for each gene after building tree
+    # If the code has been mature enough, move this to right after modeltest
+    for group in tree_dataset:
+        for gene in tree_dataset[group]:
+            try:
+                if gene != "concatenated":
+                    os.rename(
+                        f"{path.out_alignment}/{opt.runname}_MAFFT_{group}_{gene}.fasta",
+                        f"{path.out_alignment}/hash/{opt.runname}_hash_MAFFT_{group}_{gene}.fasta",
+                    )
+                    hasher.decode(
+                        tree_hash_dict,
+                        f"{path.out_alignment}/hash/{opt.runname}_hash_MAFFT_{group}_{gene}.fasta",
+                        f"{path.out_alignment}/{opt.runname}_MAFFT_{group}_{gene}.fasta",
+                    )
+
+                os.rename(
+                    f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                    f"{path.out_alignment}/hash/{opt.runname}_hash_trimmed_{group}_{gene}.fasta",
+                )
+
+                hasher.decode(
+                    tree_hash_dict,
+                    f"{path.out_alignment}/hash/{opt.runname}_hash_trimmed_{group}_{gene}.fasta",
+                    f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
+                )
+            except:
+                logging.warning(
+                    f"Tried decoding alignments of {group} {gene} but failed"
+                )
+
+    return V, path, opt