FunVIP 0.3.24__tar.gz → 0.3.24.2__tar.gz

{funvip-0.3.24 → funvip-0.3.24.2}/FunVIP.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: FunVIP
-Version: 0.3.24
+Version: 0.3.24.2
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0

{funvip-0.3.24 → funvip-0.3.24.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: FunVIP
-Version: 0.3.24
+Version: 0.3.24.2
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/main.py

@@ -52,9 +52,9 @@ def main():
     # Argument parsing in commandline mode
     args = CommandParser().get_args()
 
-    ############################################################################
-    #     Start of initializing blocks should not be moved for function!!!     #
-    ############################################################################
+    #############################################################################
+    #     Start of initializing blocks should not be moved for function!!!      #
+    #############################################################################
 
     # initialize working directories, and opts
     opt, path, list_info, list_warning, list_error = initialize.initialize(

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/cluster.py

@@ -15,6 +15,23 @@ import logging
 import gc
 from funvip.src.ext import mmseqs
 
+import sys
+
+
+def cluster_unpack(args):
+    return cluster(*args)
+
+
+def batched_generator(generator, batch_size):
+    batch = []
+    for item in generator:
+        batch.append(item)
+        if len(batch) == batch_size:
+            yield batch
+            batch = []
+    if batch:
+        yield batch
+
 
 # return list of original group of given FI
 def get_naive_group(V):
@@ -97,87 +114,6 @@ def assign_gene(result_dict, V, cutoff=0.99):
     return V
 
 
-# cluster each of FI object and assign group
-"""
-def cluster(FI, V_list_group, V_cSR, path, opt):
-    list_group = deepcopy(V_list_group)
-
-    # Reduce search df space by selecting only FI related ones
-    df_group = V_cSR.groupby(V_cSR["qseqid"])
-    list_id = list(set(V_cSR["qseqid"]))
-    df_search = df_group.get_group(FI.hash)
-
-    # If no evidence available, return it
-    if df_search is None:
-        # if confident is True, no adjusted_group for db is normal situation
-        if opt.confident is False and FI.datatype is "db":
-            logging.warning(f"No adjusted_group assigned to {FI}")
-        FI.adjusted_group = FI.group
-        return FI, None
-
-    # for db sequence with group, retain it
-    elif not (FI.group == "") and FI.datatype == "db":  # or type(FI.group) != str):
-        FI.adjusted_group = FI.group
-        return FI, FI.adjusted_group
-
-    # update group if sequence does not have group
-    else:
-        # sorting has peformed after split for better performance
-        sorted_df_search = df_search.sort_values(by=["bitscore"], ascending=False)
-
-        # Garbage collection
-        del df_search
-        gc.collect()
-
-        # reset index to easily get maximum
-        sorted_df_search.reset_index(inplace=True, drop=True)
-        # get result stasifies over cutoff
-        cutoff_df = sorted_df_search[
-            sorted_df_search["bitscore"]
-            > sorted_df_search["bitscore"][0] * opt.cluster.cutoff
-        ]
-
-        # Garbage collection
-        del sorted_df_search
-        gc.collect()
-
-        group_count = len(set(cutoff_df["subject_group"]))
-        list_group = list(set(cutoff_df["subject_group"]))
-
-        # if first time of group update
-        if FI.adjusted_group == "":
-            # if only 1 group available, take it
-            if group_count == 1:
-                FI.adjusted_group = list_group[0]
-            # if no group matched, warn it
-            elif group_count == 0:
-                logging.warning(
-                    f"Query seq in {FI.id} cannot be assigned to group. Check sequence"
-                )
-            elif group_count >= 2:
-                logging.warning(
-                    f"Query seq in {FI.id} has multiple matches to group, {list_group}."
-                )
-                FI.adjusted_group = list_group[0]
-            else:
-                logging.error("DEVELOPMENTAL ERROR IN GROUP ASSIGN")
-                raise Exception
-
-            logging.info(f"{FI.id} has clustered to {FI.adjusted_group}")
-
-        # if group already updated
-        else:
-            if not (FI.adjusted_group == ""):
-                if not (FI.adjusted_group in list_group):
-                    logging.warning(f"Clustering result colliding in {FI.id}")
-
-        if len(list_group) > 0:
-            return FI, list_group[0]
-        else:
-            return FI, None
-"""
-
-
 def cluster(FI, V_list_group, V_cSR, path, opt):
     # Reduce memory by focusing on relevant rows
     df_search = V_cSR[V_cSR["qseqid"] == FI.hash]
@@ -404,7 +340,7 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
 
 
 def group_cluster_opt_generator(V, opt, path):
-    opt_cluster = []
+    # opt_cluster = []
 
     # cluster(FO, df_search, V, path, opt)
     if len(V.list_qr_gene) == 0:
@@ -415,22 +351,24 @@ def group_cluster_opt_generator(V, opt, path):
 
     # For concatenated analysis
     else:
-        # cluster group by concatenated search result
-        list_id = list(set(V.cSR["qseqid"]))
-
-        for FI in V.list_FI:
-            if FI.hash in list_id:
-                opt_cluster.append(
-                    (
-                        FI,
-                        V.list_group,
-                        V.cSR[["qseqid", "bitscore", "subject_group"]],
-                        path,
-                        opt,
-                    )
-                )
-
-    return opt_cluster
+        # Use a set for faster membership testing
+        set_id = set(V.cSR["qseqid"])
+
+        # Pre-select the relevant DataFrame slice once
+        cSR_subset = V.cSR[["qseqid", "bitscore", "subject_group"]]
+
+        # Filter `V.list_FI` to relevant entries
+        relevant_FIs = (FI for FI in V.list_FI if FI.hash in set_id)
+
+        # Yield results as a generator
+        for FI in relevant_FIs:
+            yield (
+                FI,
+                V.list_group,  # Assuming V.list_group is small and static
+                cSR_subset,  # Pre-selected DataFrame slice
+                path,
+                opt,
+            )
 
 
 # opts ready for multithreading in outgroup append
@@ -468,16 +406,25 @@ def pipe_cluster(V, opt, path):
     if opt.method.search in ("blast", "mmseqs"):
         logging.info("group clustering")
 
+        rslt_cluster = []
+
         # cluster opt generation for multiprocessing
         # (FI, V, path, opt)
         opt_cluster = group_cluster_opt_generator(V, opt, path)
 
+        # print(f"opt_cluster : {sys.getsizeof(opt_cluster)}")
+
+        batch_size = opt.thread * 100
+        opt_cluster_batches = batched_generator(opt_cluster, batch_size)
+
         # run multiprocessing start
         if opt.verbose < 3:
-            p = mp.Pool(opt.thread)
-            rslt_cluster = p.starmap(cluster, opt_cluster)
-            p.close()
-            p.join()
+            for batch in opt_cluster_batches:
+                batch_list = list(batch)
+
+                with mp.Pool(opt.thread) as p:
+                    rslt_cluster.extend(p.starmap(cluster, batch_list))
+
         else:
             # non-multithreading mode for debugging
             rslt_cluster = [cluster(*o) for o in opt_cluster]
@@ -581,14 +528,6 @@ def pipe_append_outgroup(V, path, opt):
                 V.dict_dataset[group][gene].list_db_FI += ambiguous_group
             # Add outgroup and db in to dict_hash_FI
 
-            # Should find out why this does not work at the end
-            """
-            for FI in outgroup + ambiguous_group:
-                # If already in dict_hash_FI, they have priority
-                if not (FI.hash in V.dict_hash_FI):
-                    V.dict_hash_FI[FI.hash] = FI
-            """
-
     groups = deepcopy(list(V.dict_dataset.keys()))
 
     for group in groups:

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/ext.py

@@ -310,6 +310,7 @@ def RAxML(
     bootstrap=100,
     partition=None,
     model="-m GTRGAMMA",
+    version="old",
 ):
     if model == "skip":
         model = ""
@@ -328,7 +329,15 @@ def RAxML(
     elif platform == "darwin":
         CMD = f"raxmlHPC-PTHREADS -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}"
     else:
-        CMD = f"raxmlHPC-PTHREADS-AVX -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}"
+        if version == "old":
+            CMD = f"raxmlHPC-PTHREADS-AVX -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}"
+        elif version == "new":
+            CMD = f"raxmlHPC-PTHREADS-AVX2 -s '{fasta}' -n '{out}' -p 1 -T {thread} -f a -# {bootstrap} -x 1 {model}"
+        else:
+            logging.error(
+                f"DEVELOPMENTAL ERROR - unexpected RAxML version, {version} in ext.py"
+            )
+            raise Exception
 
     if not (partition is None):
         CMD += f" -q {partition}"

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/hasher.py

@@ -22,9 +22,19 @@ def encode(funinfo_list: list, newick: bool = False) -> dict:
     hash_dict = {}
 
     if newick is False:
+        return {funinfo.hash: f"{funinfo.id}" for funinfo in funinfo_list}
+
+        """
         for funinfo in funinfo_list:
             hash_dict[funinfo.hash] = f"{funinfo.id}"
+        """
     else:
+        return {
+            funinfo.hash: f"{funinfo.id}_{funinfo.genus}_{getattr(funinfo, 'species', funinfo.ori_species)}"
+            for funinfo in funinfo_list
+        }
+
+        """
         for funinfo in funinfo_list:
             try:
                 hash_dict[
@@ -34,11 +44,31 @@ def encode(funinfo_list: list, newick: bool = False) -> dict:
                 hash_dict[
                     funinfo.hash
                 ] = f"{funinfo.id}_{funinfo.genus}_{funinfo.ori_species}"
+        """
 
-    return hash_dict
+    # return hash_dict
 
 
 # Decode given file with given hash_dict
+
+
+def decode(hash_dict: dict, file: str, out: str, newick: bool = True) -> None:
+    with open(file, "rt") as fp:
+        content = fp.read()
+
+    hash_dict = {
+        re.escape(k): (newick_legal(v) if newick else v) for k, v in hash_dict.items()
+    }
+    pattern = re.compile("|".join(hash_dict.keys()))
+
+    # Perform the substitution
+    decoded_content = pattern.sub(lambda m: hash_dict[re.escape(m.group(0))], content)
+
+    with open(out, "w") as fw:
+        fw.write(decoded_content)
+
+
+"""
 def decode(hash_dict: dict, file: str, out: str, newick: bool = True) -> None:
     with open(file, "rt") as fp:
         line = fp.read()
@@ -54,6 +84,7 @@ def decode(hash_dict: dict, file: str, out: str, newick: bool = True) -> None:
 
         with open(out, "w") as fw:
             fw.write(line)
+"""
 
 
 # Decode given dataframe with given hash_dict

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/initialize.py

@@ -139,6 +139,9 @@ def check(
 # Use print instead of logging here, because logging has not been defined here
 class Path:
     def __init__(self, root):
+        # For RAxML updates
+        self.raxml_version = "old"
+
         # For universal data used in every run
         self.sys_path = os.path.abspath(f"{os.path.dirname(__file__)}/../")
 
@@ -174,7 +177,6 @@ class Path:
                 }
             else:
                 check_commands = {
-                    "RAxML": "raxmlHPC-PTHREADS-AVX -h",
                     "Modeltest-NG": "modeltest-ng --help",
                     "IQTREE": "iqtree -h",
                     "MMseqs2": "mmseqs -h",
@@ -192,6 +194,24 @@ class Path:
                     print(f"[ERROR] {program} not installed!")
                     install_flag = 1
 
+            # For RAxML, there are two versions
+            # I don't know why, but apt install RAxML can use raxmlHPC-PTHREADS-AVX,
+            # while conda install RAxML can use raxmlHPC -PTHREADS-AVX / raxmlHPC-PTHREADS-AVX2 (blank between raxmlHPC and -PTHREADS-AVX)
+            # However, if using 'raxmlHPC -PTHREADS-AVX', the computer cannot utilize full threads
+            return_code_1 = subprocess.call(
+                "raxmlHPC-PTHREADS-AVX -h", shell=True, stdout=open(os.devnull, "wb")
+            )
+            return_code_2 = subprocess.call(
+                "raxmlHPC-PTHREADS-AVX2 -h", shell=True, stdout=open(os.devnull, "wb")
+            )
+
+            if return_code_1 != 0 and return_code_2 != 0:
+                print(f"[ERROR] RAxML not installed!")
+                install_flag = 1
+
+            if return_code_2 == 0:
+                self.raxml_version = "new"
+
             if install_flag == 1:
                 if sys.platform == "darwin":
                     print(

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/tree.py

@@ -90,6 +90,7 @@ def pipe_tree(V, path, opt, model_dict):
                         thread=opt.thread,
                         bootstrap=opt.bootstrap,
                         model=model_dict[group][gene],
+                        version=path.raxml_version,
                     )
 
                 elif opt.method.tree.lower() == "iqtree":
@@ -131,6 +132,7 @@ def pipe_tree(V, path, opt, model_dict):
                         bootstrap=opt.bootstrap,
                         partition=f"{path.out_alignment}/{opt.runname}_{group}.partition",
                         model=model_dict[group][gene],
+                        version=path.raxml_version,
                     )
 
                 elif opt.method.tree.lower() == "iqtree":

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/tree_interpretation.py

@@ -640,8 +640,11 @@ class Tree_information:
                                     end = n
                                     break
 
+                            """
                             for n in range(start, end + 1):
                                 valid_index.append(n)
+                            """
+                            valid_index.extend(range(start, end + 1))
 
                             previous_index += len_dict[gene]
 

{funvip-0.3.24 → funvip-0.3.24.2}/funvip/src/tree_interpretation_pipe.py

@@ -19,27 +19,34 @@ def pipe_module_tree_interpretation(
     out,
     group,
     gene,
-    V,
+    V_tup_genus,
+    funinfo_dict,
+    funinfo_list,
+    hash_dict,
+    query_list,
+    outgroup,
+    partition,
     path,
     opt,
 ):
     # To reduce memory usage in multithreaded performance, copy necessary objects and then remove V
+
+    """
     funinfo_dict = V.dict_hash_FI
     funinfo_list = V.list_FI
     hash_dict = V.dict_hash_name
     query_list = V.dict_dataset[group][gene].list_qr_FI
     outgroup = V.dict_dataset[group][gene].list_og_FI
     partition = V.partition[group]
+    """
 
     # for unexpectively included sequence during clustering
     db_list = list(
-        set([FI for FI in V.list_FI if FI.datatype == "db"])
+        set([FI for FI in funinfo_list if FI.datatype == "db"])
         - set(outgroup)
         - set(query_list)
     )
-    genus_list = V.tup_genus
-
-    del V
+    genus_list = V_tup_genus
 
     # For get_genus_species
     initialize_path(path)
@@ -440,15 +447,18 @@ def synchronize(V, path, tree_info_list):
 ### Visualization after synchronization
 def pipe_module_tree_visualization(
     tree_info,
-    V,
+    V_tup_genus,
+    V_dict_hash_FI,
     path,
     opt,
 ):
     ######### Fix collapse_dict.keys()
+    # V.tup_genus
+    # V.dict_hash_FI
 
     group = tree_info.group
     gene = tree_info.gene
-    genus_list = list(V.tup_genus)
+    genus_list = list(V_tup_genus)
     genus_list.append("AMBIGUOUSGENUS")
     genus_list = tuple(genus_list)
 
@@ -487,9 +497,9 @@ def pipe_module_tree_visualization(
             # Get each of the leaf result to report
             for leaf in collapse_info.leaf_list:
                 report = Singlereport()
-                report.id = V.dict_hash_FI[leaf[0]].original_id
-                report.hash = V.dict_hash_FI[leaf[0]].hash
-                report.update_group(V.dict_hash_FI[leaf[0]].adjusted_group)
+                report.id = V_dict_hash_FI[leaf[0]].original_id
+                report.hash = V_dict_hash_FI[leaf[0]].hash
+                report.update_group(V_dict_hash_FI[leaf[0]].adjusted_group)
                 report.update_group_analysis(group)
                 report.update_gene(gene)
                 report.update_species_original(
@@ -508,9 +518,9 @@ def pipe_module_tree_visualization(
             for n, collapse_info in enumerate(tree_info.collapse_dict[taxon]):
                 for leaf in collapse_info.leaf_list:
                     report = Singlereport()
-                    report.id = V.dict_hash_FI[leaf[0]].original_id
-                    report.hash = V.dict_hash_FI[leaf[0]].hash
-                    report.update_group(V.dict_hash_FI[leaf[0]].adjusted_group)
+                    report.id = V_dict_hash_FI[leaf[0]].original_id
+                    report.hash = V_dict_hash_FI[leaf[0]].hash
+                    report.update_group(V_dict_hash_FI[leaf[0]].adjusted_group)
                     report.update_group_analysis(group)
                     report.update_gene(gene)
                     report.update_species_original(
@@ -536,8 +546,17 @@ def pipe_tree_interpretation(V, path, opt):
         for gene in V.dict_hash_FI[key].bygene_species:
             V.dict_hash_FI[key].bygene_species[gene] = V.dict_hash_FI[key].ori_species
 
+    # common variable preparation
+    funinfo_dict = V.dict_hash_FI
+    funinfo_list = V.list_FI
+    hash_dict = V.dict_hash_name
+
+    # make option variables
     for group in V.dict_dataset:
+        partition = V.partition[group]
         for gene in V.dict_dataset[group]:
+            query_list = V.dict_dataset[group][gene].list_qr_FI
+            outgroup = V.dict_dataset[group][gene].list_og_FI
             logging.debug(f"pipe_tree_interpretation {group} {gene}")
             # Condition 1 : draw all trees
             cond1 = opt.queryonly is False
@@ -566,7 +585,13 @@ def pipe_tree_interpretation(V, path, opt):
                             f"{opt.runname}_{group}_{gene}",
                             group,
                             gene,
-                            V,
+                            V.tup_genus,
+                            funinfo_dict,
+                            funinfo_list,
+                            hash_dict,
+                            query_list,
+                            outgroup,
+                            partition,
                             path,
                             opt,
                         )
@@ -604,7 +629,9 @@ def pipe_tree_interpretation(V, path, opt):
     # Generate visualization option to run
     tree_visualization_opt = []
     for tree_info in tree_info_list:
-        tree_visualization_opt.append((tree_info, V, path, opt))
+        tree_visualization_opt.append(
+            (tree_info, V.tup_genus, V.dict_hash_FI, path, opt)
+        )
 
     ## Tree visualization
     if opt.verbose < 3:

{funvip-0.3.24 → funvip-0.3.24.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "FunVIP"
-version = "0.3.24"
+version = "0.3.24.2"
 description = "Fungal Validation & Identification Pipeline"
 authors = [{name = "Changwan Seo", email = "wan101010@snu.ac.kr"}]
 urls = { "Homepage" = "https://github.com/Changwanseo/FunVIP" }