FunVIP 0.4.1__tar.gz → 0.5.0__tar.gz

{funvip-0.4.1 → funvip-0.5.0}/FunVIP.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: FunVIP
-Version: 0.4.1
+Version: 0.5.0
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0
@@ -11,7 +11,7 @@ Requires-Dist: biopython==1.84
 Requires-Dist: ete3==3.1.3
 Requires-Dist: Cython
 Requires-Dist: dendropy
-Requires-Dist: GenMine<1.2.0,>=1.1.0
+Requires-Dist: GenMine<1.4.0,>=1.3.0
 Requires-Dist: lxml
 Requires-Dist: matplotlib
 Requires-Dist: numpy<2.0.0
@@ -28,3 +28,4 @@ Requires-Dist: xlrd==2.0.1
 Requires-Dist: xlsxwriter
 Requires-Dist: xmltodict==0.12.0
 Requires-Dist: PyQt5>=5.15.0; sys_platform != "darwin"
+Dynamic: license-file

{funvip-0.4.1 → funvip-0.5.0}/FunVIP.egg-info/requires.txt

@@ -2,7 +2,7 @@ biopython==1.84
 ete3==3.1.3
 Cython
 dendropy
-GenMine<1.2.0,>=1.1.0
+GenMine<1.4.0,>=1.3.0
 lxml
 matplotlib
 numpy<2.0.0

{funvip-0.4.1 → funvip-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: FunVIP
-Version: 0.4.1
+Version: 0.5.0
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0
@@ -11,7 +11,7 @@ Requires-Dist: biopython==1.84
 Requires-Dist: ete3==3.1.3
 Requires-Dist: Cython
 Requires-Dist: dendropy
-Requires-Dist: GenMine<1.2.0,>=1.1.0
+Requires-Dist: GenMine<1.4.0,>=1.3.0
 Requires-Dist: lxml
 Requires-Dist: matplotlib
 Requires-Dist: numpy<2.0.0
@@ -28,3 +28,4 @@ Requires-Dist: xlrd==2.0.1
 Requires-Dist: xlsxwriter
 Requires-Dist: xmltodict==0.12.0
 Requires-Dist: PyQt5>=5.15.0; sys_platform != "darwin"
+Dynamic: license-file

{funvip-0.4.1 → funvip-0.5.0}/README.md

@@ -1,18 +1,31 @@
 
 # FunVIP [![DOI](https://zenodo.org/badge/588465720.svg)](https://zenodo.org/doi/10.5281/zenodo.10714946)
+
+
+FunVIP is now published please cite:
+#### Seo CW, Yoo S, Cho Y, Kim JS, Steinegger M, Lim YW. FunVIP: Fungal Validation and Identification Pipeline based on phylogenetic analysis. J. Microbiol. 2025;63(4):e2411017.
+<br><br/>
+
+
+
+
 ### **Fun**gal **V**alidation & **I**dentification **P**ipeline
 #### An automatic tree-based sequence identification and validation pipeline for fungal (or maybe other) species
 
+
+
 - Automatic tree-based identification
 - Works with multiple genetic marker
 - Database sequence validation algorithm implemented
 
 ![figure1 - ver17A](https://github.com/user-attachments/assets/22a50a62-14e8-41a7-87a0-8f5a1f9c3f62)
 
-This is Beta release. Bug reports are welcomed
+Bug reports are always welcomed
 <br><br/>
 
-### IMPORTANT NOTICE: The python dependency for Linux platform has changed from 3.12 to 3.11 for TCS inclusion. Please remake conda environment for FunVIP 0.3.25 update
+
+
+#### IMPORTANT NOTICE: The python dependency for Linux platform has changed from 3.12 to 3.11 for TCS inclusion. Please remake conda environment for FunVIP 0.3.25 update
 
 ## Tutorial
 * [Part 1 - Getting started!](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial.md)
@@ -88,7 +101,7 @@ tabular (```.xlsx```, ```.csv```,  ```.parquet```, ```.ftr```) form
 
 ## Scheduling
 1. ~~Beta release part 1 (2023 Feburary ~ As paper published, ver 0.3)
-Will be tested by our lab memebers to fix bugs and advance features~~ FunVIP article is currently accepted. Will be published soon.
+Will be tested by our lab memebers to fix bugs and advance features~~
 2. Beta release part 2 (As paper published ~ When pipeline gets stabled, ver 0.4)
 Will be tested by peer taxonomists
 3. Stable release (ver 1.0)

{funvip-0.4.1 → funvip-0.5.0}/funvip/main.py

@@ -410,3 +410,24 @@ def main():
             logging.info(f"Time report generation: {round(time_end-time_visualize,3)}s")
         except:
             logging.warning(f"Failed logging reoprt generation time")
+
+        # At the end of the run, print critical messages once again to be noticed
+        with open(path.criticallog, "r") as frclog:
+            critical_log_lines = frclog.readlines()
+
+            critical_logs = []
+            flag_critical = 0
+            for line in critical_log_lines:
+                if "[CRITICAL]" in line:
+                    flag_critical = 1
+                elif "[WARNING]" in line or "[ERROR]" in line:
+                    flag_critical = 0
+
+                if flag_critical == 1:
+                    critical_logs.append(line)
+
+            if len(critical_logs) > 0:
+                print("\n!!!<<<IMPORTANT NOTICE FROM FUNVIP RUN>>>!!!")
+
+            for line in critical_logs:
+                print(line)

{funvip-0.4.1 → funvip-0.5.0}/funvip/src/cluster.py

@@ -114,6 +114,7 @@ def assign_gene(result_dict, V, cutoff=0.99):
     return V
 
 
+# This function assigns group to each FI
 def cluster(FI, V_list_group, V_cSR, path, opt):
     # Reduce memory by focusing on relevant rows
     df_search = V_cSR[V_cSR["qseqid"] == FI.hash]
@@ -222,23 +223,23 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
     # split that same group to include all to alignment, and leave other groups for outgroup selection
     cutoff_df = cutoff_df[cutoff_df["subject_group"] != group]
 
-    ## For ambiugous database, mostly because of contaminated database
+    ## Add suspicious database, mostly because of contaminated database
+    # Use term "ambiguous" instead of "suspicious" because of previous compatibility
     # For each of the input, should use different cutoff
     ambiguous_db = set()
-    for qseqid, _df in cutoff_set_df.groupby(["qseqid"]):
-        # Select dataframe corresponding to current qseqid
-        df_qseqid = df_search[df_search["qseqid"] == qseqid]
-        """
-        print(
-            f"Ambiguous ingroup cutoff selected for query {qseqid} group {group} gene {gene} cutoff {min(list(_df['bitscore']))}"
-        )
-        """
-        # Get the list of subjects, which is closer than furtest ingroup
-        ambiguous_df = df_qseqid[df_qseqid["bitscore"] >= min(list(_df["bitscore"]))]
-        # Within the furthest match, get possible ingroups with ambiguous group
-        ambiguous_df = ambiguous_df[ambiguous_df["subject_group"] != group]
-        # Add inner ambiugities to ambiguous db
-        ambiguous_db.update([FI_dict[i] for i in list(ambiguous_df["sseqid"])])
+    if opt.suspicious is True:
+        for qseqid, _df in cutoff_set_df.groupby(["qseqid"]):
+            # Select dataframe corresponding to current qseqid
+            df_qseqid = df_search[df_search["qseqid"] == qseqid[0]]
+
+            # Get the list of subjects, which is closer than furtest ingroup
+            ambiguous_df = df_qseqid[
+                df_qseqid["bitscore"] >= min(list(_df["bitscore"]))
+            ]
+            # Within the furthest match, get possible ingroups with ambiguous group
+            ambiguous_df = ambiguous_df[ambiguous_df["subject_group"] != group]
+            # Add inner ambiugities to ambiguous db
+            ambiguous_db.update([FI_dict[i] for i in list(ambiguous_df["sseqid"])])
 
     ambiguous_db = list(ambiguous_db)
 
@@ -308,12 +309,20 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
                     f"Outgroup [{subject_group}] selected to [{group}]\n {text_outgroup_list}"
                 )
 
+                # Move outgroup within ambiguous db to outgroup
+                for FI in ambiguous_db:
+                    if FI.group == subject_group:
+                        ambiguous_db.remove(FI)
+                        outgroup_dict[subject_group].append(FI)
+
                 return (
                     group,
                     gene,
                     outgroup_dict[subject_group],
                     ambiguous_db,
                 )
+
+            # If not use the outgroup sequences with the maximum number
             else:
                 if len(outgroup_dict[subject_group]) > max_cnt:
                     max_cnt = len(outgroup_dict[subject_group])
@@ -330,6 +339,12 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
             f"Final outgroup selection for group {group} : {outgroup_dict[max_group]}"
         )
 
+        # Move outgroup within ambiguous db to outgroup
+        for FI in ambiguous_db:
+            if FI.group == max_group:
+                ambiguous_db.remove(FI)
+                outgroup_dict[max_group].append(FI)
+
         return (group, gene, outgroup_dict[max_group], ambiguous_db)
 
     # If outgroup cannot be selected
@@ -507,6 +522,7 @@ def pipe_append_outgroup(V, path, opt):
 
     # append outgroup by running result
     # (group, gene, outgroup, ambiguous_group)
+    critical_flag = 0
     for result in result_append_outgroup:
         # Parsing result
         group = result[0]
@@ -524,9 +540,10 @@ def pipe_append_outgroup(V, path, opt):
         print(f"query: {len(V.dict_dataset[group][gene].list_qr_FI)}")
 
         if len(outgroup) == 0 and len(ambiguous_group) == 0:
-            logging.warning(
+            logging.critical(
                 f"Removing {group} {gene} from analysis because outgroup cannot be selected"
             )
+            critical_flag = 1
             V.dict_dataset[group].pop(gene, None)
         elif (
             len(outgroup)
@@ -535,15 +552,17 @@ def pipe_append_outgroup(V, path, opt):
             + len(V.dict_dataset[group][gene].list_qr_FI)
             < 4
         ):
-            logging.warning(
+            logging.critical(
                 f"Removing {group} {gene} from analysis because not enough sequences are provided to infer phylogenetic tree"
             )
+            critical_flag = 1
             V.dict_dataset[group].pop(gene, None)
 
         else:
             V.dict_dataset[group][gene].list_og_FI = outgroup
+
             # Add ambiguous group to FI
-            if opt.ambiguous is True:
+            if opt.suspicious is True:
                 V.dict_dataset[group][gene].list_db_FI += ambiguous_group
             # Add outgroup and db in to dict_hash_FI
 
@@ -552,11 +571,16 @@ def pipe_append_outgroup(V, path, opt):
     for group in groups:
         try:
             if len(V.dict_dataset[group]) == 0:
-                logging.warning(
+                logging.critical(
                     f"Removing {group} from analysis because outgroup cannot be selected to all genes"
                 )
+                critical_flag = 1
                 V.dict_dataset.pop(group, None)
         except:
             pass
 
+    # Terminate if terminate option is given, and critical error occurs
+    if critical_flag == 1 and opt.terminate is True:
+        raise Exception
+
     return V, path, opt

{funvip-0.4.1 → funvip-0.5.0}/funvip/src/command.py

@@ -309,9 +309,14 @@ class CommandParser:
             type=int,
         )
         group_advanced.add_argument(
-            "--noambiguous",
+            "--nosuspicious",
             action="store_true",
-            help="Do not include ambiguous samples for sequence-set. Mostly for metabarcoding analysis. It may result wrong result with problematic database.",
+            help="Do not include suspicious samples for sequence-set. Mostly for metabarcoding analysis. May deduce inaccurate result with problematic database.",
+        )
+        group_advanced.add_argument(
+            "--terminate",
+            action="store_true",
+            help="Terminate FunVIP run when critical error detected",
         )
 
         # Cache

{funvip-0.4.1 → funvip-0.5.0}/funvip/src/dataset.py

@@ -462,6 +462,8 @@ class FunVIP_var:
         fail_list = []
         remove_dict = {}
         tree_hash_dict = hasher.encode(self.list_FI, newick=True)
+
+        critical_flag = 0
         for group in self.dict_dataset:
             remove_dict[group] = {}
             for gene in self.dict_dataset[group]:
@@ -545,9 +547,38 @@ class FunVIP_var:
                         ## If all value of vectors are zero, it means that all regions have at least one gap
                         ## Raise warning for this
                         if np.all((vector_products == 0)):
-                            logging.warning(
+                            logging.critical(
                                 f"Alignment for {group} {gene} does not have any overlapping regions! Removing from analysis"
                             )
+
+                            critical_flag = 1
+
+                            # Report one non_overlapping pair
+                            for i in range(len(seq_list) - 1):
+                                seq1 = seq_list[i]
+                                seq1_hash = seq1.id
+                                seq1_str = str(seq.seq)
+
+                                for j in range(i + 1, len(seq_list)):
+                                    seq2 = seq_list[j]
+                                    seq2_hash = seq2.id
+                                    seq2_str = str(seq.seq)
+
+                                    has_overlap = False
+
+                                    for k in range(len(seq1_str)):
+                                        if seq1_str[k] != "-" and seq2_str[k] != "-":
+                                            has_overlap = True
+                                            break
+
+                            seq1_id = self.dict_hash_FI[seq1_hash]
+                            seq2_id = self.dict_hash_FI[seq2_hash]
+
+                            logging.critical(
+                                f"At least one pair of sequence does not overlap, such as {seq1_id} and {seq2_id}"
+                            )
+                            critical_flag = 1
+
                             fail_list.append((group, gene))
                             # for tree, use hash dict with genus and species information
                             # Decoding process in done in tree building processes, so these alignments cannot be decoded. So decode them here
@@ -575,6 +606,10 @@ class FunVIP_var:
                     # If alignment corresponding to dataset does not exists, raise warning or error
                     pass
 
+        # Terminate if terminate option is given, and critical error occurs
+        if critical_flag == 1 and opt.terminate is True:
+            raise Exception
+
         # Remove bad datasets
         for fail in fail_list:
             group = fail[0]

{funvip-0.4.1 → funvip-0.5.0}/funvip/src/tree_interpretation.py

@@ -709,27 +709,6 @@ class Tree_information:
 
         if diff_min < self.zero:
             self.zero = diff_min - 0.00000001
-        """
-        print("Calculate zero")
-
-        for name, size in sorted(
-            ((name, sys.getsizeof(value)) for name, value in list(locals().items())),
-            key=lambda x: -x[1],
-        )[:10]:
-            # print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
-            pass
-
-        snapshot = tracemalloc.take_snapshot()
-        top_stats = snapshot.statistics("lineno")
-
-        # Print the top memory usage lines
-        
-        print("[ Top 10 ]")
-        for stat in top_stats[:10]:
-            print(stat)
-
-        print("===========================")
-        """
 
         if self.opt.verbose >= 3:
             print(f"[DEBUG] End of calculate zero")
@@ -763,12 +742,16 @@ class Tree_information:
         # reroot with outgroup_clade
         try:
             # For more than one outgroups, after rerooting, get_common_ancestor of outgroup again
+            # Before rerooting, unroot the tree to work properly
             if len(outgroup_leaves) >= 2:
+                print(outgroup_leaves)
+                self.t.unroot()
                 self.outgroup_clade = self.t.get_common_ancestor(outgroup_leaves)
                 self.t.set_outgroup(self.outgroup_clade)
                 self.t.ladderize(direction=1)
                 self.outgroup_clade = self.t.get_common_ancestor(outgroup_leaves)
             elif len(outgroup_leaves) == 1:
+                self.t.unroot()
                 self.outgroup_clade = outgroup_leaves[0]
                 self.t.set_outgroup(self.outgroup_clade)
                 self.t.ladderize(direction=1)
@@ -820,34 +803,27 @@ class Tree_information:
                 raise Exception
 
         self.Tree_style.ts.show_leaf_name = True
-        for node in self.t.traverse():
+
+        # Add bootstrap
+        # Before doing this, copy the tree to not interfere final tree
+        copied_tree = deepcopy(self.t)
+
+        for node in copied_tree.traverse():
             node.img_style["size"] = 0  # removing circles whien size is 0
+            if len(node) > 1:  # Prevent bootstrap on single branch
+                node.add_face(
+                    TextFace(
+                        f"{int(node.support)}",
+                        fsize=self.opt.visualize.fsize_bootstrap,
+                        fstyle="Arial",
+                    ),
+                    column=0,
+                    position="float",
+                )
+        copied_tree.render(f"{out}", tree_style=self.Tree_style.ts)
 
-        self.t.render(f"{out}", tree_style=self.Tree_style.ts)
         self.Tree_style.ts.show_leaf_name = False
 
-        """
-        print("reroot outgroup")
-       
-        for name, size in sorted(
-            ((name, sys.getsizeof(value)) for name, value in list(locals().items())),
-            key=lambda x: -x[1],
-        )[:10]:
-            print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
-           
-
-        snapshot = tracemalloc.take_snapshot()
-        top_stats = snapshot.statistics("lineno")
-
-        # Print the top memory usage lines
-        
-        print("[ Top 10 ]")
-        for stat in top_stats[:10]:
-            print(stat)
-       
-        print("===========================")
-        """
-
         if self.opt.verbose >= 3:
             print(f"[DEBUG] End of reroot outgroup")
             process = psutil.Process(os.getpid())
@@ -972,33 +948,9 @@ class Tree_information:
                 self.collapse_dict[taxon].append(collapse_info)
 
         ## start of tree_search
-        # at the last leaf
-        # tracemalloc.start()
 
         if len(clade.children) == 1:
             local_generate_collapse_information(clade, opt=opt)
-            """
-            print("tree search part 1")
-            
-            for name, size in sorted(
-                (
-                    (name, sys.getsizeof(value))
-                    for name, value in list(locals().items())
-                ),
-                key=lambda x: -x[1],
-            )[:10]:
-                print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
-
-            snapshot = tracemalloc.take_snapshot()
-            top_stats = snapshot.statistics("lineno")
-
-            # Print the top memory usage lines
-            print("[ Top 10 ]")
-            for stat in top_stats[:10]:
-                print(stat)
- 
-            print("===========================")
-            """
 
             if self.opt.verbose >= 3:
                 print(f"[DEBUG] End of Tree search with monophyletic branches")
@@ -1035,29 +987,6 @@ class Tree_information:
                 else:
                     self.tree_search(child_clade, gene, opt=opt)
 
-            """
-            print("tree search part 2")
-            
-            for name, size in sorted(
-                (
-                    (name, sys.getsizeof(value))
-                    for name, value in list(locals().items())
-                ),
-                key=lambda x: -x[1],
-            )[:10]:
-                print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
-
-            snapshot = tracemalloc.take_snapshot()
-            top_stats = snapshot.statistics("lineno")
-
-            # Print the top memory usage lines
-            print("[ Top 10 ]")
-            for stat in top_stats[:10]:
-                print(stat)
-
-            print("===========================")
-            """
-
             if self.opt.verbose >= 3:
                 print(f"[DEBUG] End of Tree search with bifurcated branches")
                 process = psutil.Process(os.getpid())
@@ -1151,14 +1080,6 @@ class Tree_information:
                         else:
                             taxon_dict[("", "")] = 1
 
-                    """
-                    for leaf in c:
-                        if t in taxon_dict:
-                            taxon_dict[t(leaf)] += 1
-                        else:
-                            taxon_dict[t(leaf)] = 1
-                    """
-
                     if len(taxon_dict) == 0:
                         print(
                             f"{bold_red}[DEVELOPMENTAL ERROR] Error in tree_interpretation.py line 912 {taxon_dict}\n {c}{reset}"
@@ -1388,21 +1309,6 @@ class Tree_information:
                 root_dist=clade.dist,
                 root_support=clade.support,
             ).copy("newick")
-            """
-            print(f"Reconstruct")
-            
-            for name, size in sorted(
-                (
-                    (name, sys.getsizeof(value))
-                    for name, value in list(locals().items())
-                ),
-                key=lambda x: -x[1],
-            )[:30]:
-                print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
-
-            print("==============================")
-            sys.stdout.flush()
-            """
 
             if self.opt.verbose >= 3:
                 print(f"[DEBUG] End of reconstruct")
@@ -1498,21 +1404,15 @@ class Tree_information:
 
                 clade.add_face(space_text, 2, position="branch-right")
                 clade.add_face(taxon_text, 3, position="branch-right")
-                clade.add_face(space_text, 4, position="branch-right")
-                clade.add_face(id_text, 5, position="branch-right")
+                # clade.add_face(space_text, 4, position="branch-right")
+
+                clade.add_face(id_text, 4, position="aligned")
 
                 # Get all tip names of the current working clade
                 collapse_leaf_name_list = [x[0] for x in collapse_info.leaf_list]
 
                 # Check if current working clade includes only outgroup sequences
-                """
-                if all(
-                    x in self.outgroup_leaf_name_list or x in collapse_info.query_list
-                    for x in collapse_leaf_name_list
-                ) and any(
-                    x in self.outgroup_leaf_name_list for x in collapse_leaf_name_list
-                ):
-                """
+
                 # Development, color unintended outgroup in outgroup color
                 if any(
                     self.funinfo_dict[x].adjusted_group in self.outgroup_group
@@ -1613,7 +1513,7 @@ class Tree_information:
             # relocate text position little bit for better visualization
             text.set("y", f'{int(float(text.get("y")))-2}')
 
-            # Problem here: taxon name with cff., aff. etc includes underscore
+            # Problem here: taxon name with cf., aff. etc includes underscore
             if text_type == "taxon":
                 genus = taxon_string_dict[text.text][0]
                 species = taxon_string_dict[text.text][1]
@@ -1719,19 +1619,6 @@ class Tree_information:
             encoding="utf-8",
             xml_declaration=True,
         )
-        """
-        print("In tree visualization")
-        
-        for name, size in sorted(
-            ((name, sys.getsizeof(value)) for name, value in list(locals().items())),
-            key=lambda x: -x[1],
-        )[:10]:
-            print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
-        
-        print("===========================")
-
-
-        """
 
         if self.opt.verbose >= 3:
             print(f"[DEBUG] End of Tree visualization")

{funvip-0.4.1 → funvip-0.5.0}/funvip/src/tree_interpretation_pipe.py

@@ -33,17 +33,6 @@ def pipe_module_tree_interpretation(
     path,
     opt,
 ):
-    # To reduce memory usage in multithreaded performance, copy necessary objects and then remove V
-
-    """
-    funinfo_dict = V.dict_hash_FI
-    funinfo_list = V.list_FI
-    hash_dict = V.dict_hash_name
-    query_list = V.dict_dataset[group][gene].list_qr_FI
-    outgroup = V.dict_dataset[group][gene].list_og_FI
-    partition = V.partition[group]
-    """
-
     # time_start = time()
 
     # for unexpectively included sequence during clustering