FunVIP 0.4.0__tar.gz → 0.5.0__tar.gz

{funvip-0.4.0 → funvip-0.5.0}/FunVIP.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: FunVIP
-Version: 0.4.0
+Version: 0.5.0
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0
@@ -11,7 +11,7 @@ Requires-Dist: biopython==1.84
 Requires-Dist: ete3==3.1.3
 Requires-Dist: Cython
 Requires-Dist: dendropy
-Requires-Dist: GenMine<1.2.0,>=1.1.0
+Requires-Dist: GenMine<1.4.0,>=1.3.0
 Requires-Dist: lxml
 Requires-Dist: matplotlib
 Requires-Dist: numpy<2.0.0
@@ -28,3 +28,4 @@ Requires-Dist: xlrd==2.0.1
 Requires-Dist: xlsxwriter
 Requires-Dist: xmltodict==0.12.0
 Requires-Dist: PyQt5>=5.15.0; sys_platform != "darwin"
+Dynamic: license-file

{funvip-0.4.0 → funvip-0.5.0}/FunVIP.egg-info/SOURCES.txt

@@ -100,6 +100,10 @@ funvip/external/mmseqs_Windows/matrices/blosum90.out
 funvip/external/mmseqs_Windows/matrices/blosum95.out
 funvip/external/mmseqs_Windows/matrices/nucleotide.out
 funvip/external/mmseqs_Windows/util/bash-completion.sh
+funvip/external/modeltest-ng_Windows/cyggcc_s-seh-1.dll
+funvip/external/modeltest-ng_Windows/cygstdc++-6.dll
+funvip/external/modeltest-ng_Windows/cygwin1.dll
+funvip/external/modeltest-ng_Windows/modeltest-ng.exe
 funvip/external/trimal.v1.4/trimAl/AUTHORS
 funvip/external/trimal.v1.4/trimAl/CHANGELOG
 funvip/external/trimal.v1.4/trimAl/LICENSE

{funvip-0.4.0 → funvip-0.5.0}/FunVIP.egg-info/requires.txt

@@ -2,7 +2,7 @@ biopython==1.84
 ete3==3.1.3
 Cython
 dendropy
-GenMine<1.2.0,>=1.1.0
+GenMine<1.4.0,>=1.3.0
 lxml
 matplotlib
 numpy<2.0.0

{funvip-0.4.0 → funvip-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: FunVIP
-Version: 0.4.0
+Version: 0.5.0
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0
@@ -11,7 +11,7 @@ Requires-Dist: biopython==1.84
 Requires-Dist: ete3==3.1.3
 Requires-Dist: Cython
 Requires-Dist: dendropy
-Requires-Dist: GenMine<1.2.0,>=1.1.0
+Requires-Dist: GenMine<1.4.0,>=1.3.0
 Requires-Dist: lxml
 Requires-Dist: matplotlib
 Requires-Dist: numpy<2.0.0
@@ -28,3 +28,4 @@ Requires-Dist: xlrd==2.0.1
 Requires-Dist: xlsxwriter
 Requires-Dist: xmltodict==0.12.0
 Requires-Dist: PyQt5>=5.15.0; sys_platform != "darwin"
+Dynamic: license-file

{funvip-0.4.0 → funvip-0.5.0}/README.md

@@ -1,29 +1,46 @@
 
 # FunVIP [![DOI](https://zenodo.org/badge/588465720.svg)](https://zenodo.org/doi/10.5281/zenodo.10714946)
+
+
+FunVIP is now published please cite:
+#### Seo CW, Yoo S, Cho Y, Kim JS, Steinegger M, Lim YW. FunVIP: Fungal Validation and Identification Pipeline based on phylogenetic analysis. J. Microbiol. 2025;63(4):e2411017.
+<br><br/>
+
+
+
+
 ### **Fun**gal **V**alidation & **I**dentification **P**ipeline
 #### An automatic tree-based sequence identification and validation pipeline for fungal (or maybe other) species
 
+
+
 - Automatic tree-based identification
-- Works with multigene
-- Data validation algorithm implemented
+- Works with multiple genetic marker
+- Database sequence validation algorithm implemented
 
 ![figure1 - ver17A](https://github.com/user-attachments/assets/22a50a62-14e8-41a7-87a0-8f5a1f9c3f62)
 
-This is Beta release. Bug reports are welcomed
+Bug reports are always welcomed
 <br><br/>
 
+
+
+#### IMPORTANT NOTICE: The python dependency for Linux platform has changed from 3.12 to 3.11 for TCS inclusion. Please remake conda environment for FunVIP 0.3.25 update
+
 ## Tutorial
 * [Part 1 - Getting started!](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial.md)
 * [Part 2 - Preparing database and query](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial2.md)
-* [Advacned tips](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/advanced.md)
+* [Advanced tips](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/advanced.md)
 <br><br/>
 ## Documentation
 * See [Documentation](https://github.com/Changwanseo/FunVIP/blob/main/Documentation.md) for advanced usage !
 <br><br/>
 ## Requirements
-- Conda environment
+- Conda or Mamba environment
 
-\* See [https://www.anaconda.com/products/individual](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) for how to install conda environment
+\* See [here](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) for how to install conda environment
+
+\* Recently, Mamba is a lot faster than conda. See [here](https://github.com/conda-forge/miniforge?tab=readme-ov-file) to how to install mamba environment
 <br><br/>
 ## Installation
 * [Windows](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Windows)
@@ -77,9 +94,14 @@ tabular (```.xlsx```, ```.csv```,  ```.parquet```, ```.ftr```) form
 * ```/Tree/{section}_{gene}.svg``` : Final collapsed tree in svg format. Can be edited in vector graphics programs, or in powerpoint (by ungroup)
 * ```/Tree/{section}_{gene}_original.svg ``` : Uncollapsed tree for inspection
 
+* Example output tree of FunVIP
+![image](https://github.com/user-attachments/assets/7291c990-62d0-4579-8ae7-adc5d39a7fed)
+
+
+
 ## Scheduling
-1. Beta release part 1 (2023 Feburary ~ As paper published, ver 0.3)
-Will be tested by our lab memebers to fix bugs and advance features
+1. ~~Beta release part 1 (2023 Feburary ~ As paper published, ver 0.3)
+Will be tested by our lab memebers to fix bugs and advance features~~
 2. Beta release part 2 (As paper published ~ When pipeline gets stabled, ver 0.4)
 Will be tested by peer taxonomists
 3. Stable release (ver 1.0)

{funvip-0.4.0 → funvip-0.5.0}/funvip/main.py

@@ -410,3 +410,24 @@ def main():
             logging.info(f"Time report generation: {round(time_end-time_visualize,3)}s")
         except:
             logging.warning(f"Failed logging reoprt generation time")
+
+        # At the end of the run, print critical messages once again to be noticed
+        with open(path.criticallog, "r") as frclog:
+            critical_log_lines = frclog.readlines()
+
+            critical_logs = []
+            flag_critical = 0
+            for line in critical_log_lines:
+                if "[CRITICAL]" in line:
+                    flag_critical = 1
+                elif "[WARNING]" in line or "[ERROR]" in line:
+                    flag_critical = 0
+
+                if flag_critical == 1:
+                    critical_logs.append(line)
+
+            if len(critical_logs) > 0:
+                print("\n!!!<<<IMPORTANT NOTICE FROM FUNVIP RUN>>>!!!")
+
+            for line in critical_logs:
+                print(line)

{funvip-0.4.0 → funvip-0.5.0}/funvip/src/cluster.py

@@ -114,6 +114,7 @@ def assign_gene(result_dict, V, cutoff=0.99):
     return V
 
 
+# This function assigns group to each FI
 def cluster(FI, V_list_group, V_cSR, path, opt):
     # Reduce memory by focusing on relevant rows
     df_search = V_cSR[V_cSR["qseqid"] == FI.hash]
@@ -222,23 +223,23 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
     # split that same group to include all to alignment, and leave other groups for outgroup selection
     cutoff_df = cutoff_df[cutoff_df["subject_group"] != group]
 
-    ## For ambiugous database, mostly because of contaminated database
+    ## Add suspicious database, mostly because of contaminated database
+    # Use term "ambiguous" instead of "suspicious" because of previous compatibility
     # For each of the input, should use different cutoff
     ambiguous_db = set()
-    for qseqid, _df in cutoff_set_df.groupby(["qseqid"]):
-        # Select dataframe corresponding to current qseqid
-        df_qseqid = df_search[df_search["qseqid"] == qseqid]
-        """
-        print(
-            f"Ambiguous ingroup cutoff selected for query {qseqid} group {group} gene {gene} cutoff {min(list(_df['bitscore']))}"
-        )
-        """
-        # Get the list of subjects, which is closer than furtest ingroup
-        ambiguous_df = df_qseqid[df_qseqid["bitscore"] >= min(list(_df["bitscore"]))]
-        # Within the furthest match, get possible ingroups with ambiguous group
-        ambiguous_df = ambiguous_df[ambiguous_df["subject_group"] != group]
-        # Add inner ambiugities to ambiguous db
-        ambiguous_db.update([FI_dict[i] for i in list(ambiguous_df["sseqid"])])
+    if opt.suspicious is True:
+        for qseqid, _df in cutoff_set_df.groupby(["qseqid"]):
+            # Select dataframe corresponding to current qseqid
+            df_qseqid = df_search[df_search["qseqid"] == qseqid[0]]
+
+            # Get the list of subjects, which is closer than furtest ingroup
+            ambiguous_df = df_qseqid[
+                df_qseqid["bitscore"] >= min(list(_df["bitscore"]))
+            ]
+            # Within the furthest match, get possible ingroups with ambiguous group
+            ambiguous_df = ambiguous_df[ambiguous_df["subject_group"] != group]
+            # Add inner ambiugities to ambiguous db
+            ambiguous_db.update([FI_dict[i] for i in list(ambiguous_df["sseqid"])])
 
     ambiguous_db = list(ambiguous_db)
 
@@ -308,12 +309,20 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
                     f"Outgroup [{subject_group}] selected to [{group}]\n {text_outgroup_list}"
                 )
 
+                # Move outgroup within ambiguous db to outgroup
+                for FI in ambiguous_db:
+                    if FI.group == subject_group:
+                        ambiguous_db.remove(FI)
+                        outgroup_dict[subject_group].append(FI)
+
                 return (
                     group,
                     gene,
                     outgroup_dict[subject_group],
                     ambiguous_db,
                 )
+
+            # If not use the outgroup sequences with the maximum number
             else:
                 if len(outgroup_dict[subject_group]) > max_cnt:
                     max_cnt = len(outgroup_dict[subject_group])
@@ -330,6 +339,12 @@ def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
             f"Final outgroup selection for group {group} : {outgroup_dict[max_group]}"
         )
 
+        # Move outgroup within ambiguous db to outgroup
+        for FI in ambiguous_db:
+            if FI.group == max_group:
+                ambiguous_db.remove(FI)
+                outgroup_dict[max_group].append(FI)
+
         return (group, gene, outgroup_dict[max_group], ambiguous_db)
 
     # If outgroup cannot be selected
@@ -507,6 +522,7 @@ def pipe_append_outgroup(V, path, opt):
 
     # append outgroup by running result
     # (group, gene, outgroup, ambiguous_group)
+    critical_flag = 0
     for result in result_append_outgroup:
         # Parsing result
         group = result[0]
@@ -524,9 +540,10 @@ def pipe_append_outgroup(V, path, opt):
         print(f"query: {len(V.dict_dataset[group][gene].list_qr_FI)}")
 
         if len(outgroup) == 0 and len(ambiguous_group) == 0:
-            logging.warning(
+            logging.critical(
                 f"Removing {group} {gene} from analysis because outgroup cannot be selected"
             )
+            critical_flag = 1
             V.dict_dataset[group].pop(gene, None)
         elif (
             len(outgroup)
@@ -535,15 +552,17 @@ def pipe_append_outgroup(V, path, opt):
             + len(V.dict_dataset[group][gene].list_qr_FI)
             < 4
         ):
-            logging.warning(
+            logging.critical(
                 f"Removing {group} {gene} from analysis because not enough sequences are provided to infer phylogenetic tree"
             )
+            critical_flag = 1
             V.dict_dataset[group].pop(gene, None)
 
         else:
             V.dict_dataset[group][gene].list_og_FI = outgroup
+
             # Add ambiguous group to FI
-            if opt.ambiguous is True:
+            if opt.suspicious is True:
                 V.dict_dataset[group][gene].list_db_FI += ambiguous_group
             # Add outgroup and db in to dict_hash_FI
 
@@ -552,11 +571,16 @@ def pipe_append_outgroup(V, path, opt):
     for group in groups:
         try:
             if len(V.dict_dataset[group]) == 0:
-                logging.warning(
+                logging.critical(
                     f"Removing {group} from analysis because outgroup cannot be selected to all genes"
                 )
+                critical_flag = 1
                 V.dict_dataset.pop(group, None)
         except:
             pass
 
+    # Terminate if terminate option is given, and critical error occurs
+    if critical_flag == 1 and opt.terminate is True:
+        raise Exception
+
     return V, path, opt

{funvip-0.4.0 → funvip-0.5.0}/funvip/src/command.py

@@ -309,9 +309,14 @@ class CommandParser:
             type=int,
         )
         group_advanced.add_argument(
-            "--noambiguous",
+            "--nosuspicious",
             action="store_true",
-            help="Do not include ambiguous samples for sequence-set. Mostly for metabarcoding analysis. It may result wrong result with problematic database.",
+            help="Do not include suspicious samples for sequence-set. Mostly for metabarcoding analysis. May deduce inaccurate result with problematic database.",
+        )
+        group_advanced.add_argument(
+            "--terminate",
+            action="store_true",
+            help="Terminate FunVIP run when critical error detected",
         )
 
         # Cache

{funvip-0.4.0 → funvip-0.5.0}/funvip/src/dataset.py

@@ -462,6 +462,8 @@ class FunVIP_var:
         fail_list = []
         remove_dict = {}
         tree_hash_dict = hasher.encode(self.list_FI, newick=True)
+
+        critical_flag = 0
         for group in self.dict_dataset:
             remove_dict[group] = {}
             for gene in self.dict_dataset[group]:
@@ -545,9 +547,38 @@ class FunVIP_var:
                         ## If all value of vectors are zero, it means that all regions have at least one gap
                         ## Raise warning for this
                         if np.all((vector_products == 0)):
-                            logging.warning(
+                            logging.critical(
                                 f"Alignment for {group} {gene} does not have any overlapping regions! Removing from analysis"
                             )
+
+                            critical_flag = 1
+
+                            # Report one non_overlapping pair
+                            for i in range(len(seq_list) - 1):
+                                seq1 = seq_list[i]
+                                seq1_hash = seq1.id
+                                seq1_str = str(seq.seq)
+
+                                for j in range(i + 1, len(seq_list)):
+                                    seq2 = seq_list[j]
+                                    seq2_hash = seq2.id
+                                    seq2_str = str(seq.seq)
+
+                                    has_overlap = False
+
+                                    for k in range(len(seq1_str)):
+                                        if seq1_str[k] != "-" and seq2_str[k] != "-":
+                                            has_overlap = True
+                                            break
+
+                            seq1_id = self.dict_hash_FI[seq1_hash]
+                            seq2_id = self.dict_hash_FI[seq2_hash]
+
+                            logging.critical(
+                                f"At least one pair of sequence does not overlap, such as {seq1_id} and {seq2_id}"
+                            )
+                            critical_flag = 1
+
                             fail_list.append((group, gene))
                             # for tree, use hash dict with genus and species information
                             # Decoding process in done in tree building processes, so these alignments cannot be decoded. So decode them here
@@ -575,6 +606,10 @@ class FunVIP_var:
                     # If alignment corresponding to dataset does not exists, raise warning or error
                     pass
 
+        # Terminate if terminate option is given, and critical error occurs
+        if critical_flag == 1 and opt.terminate is True:
+            raise Exception
+
         # Remove bad datasets
         for fail in fail_list:
             group = fail[0]

{funvip-0.4.0 → funvip-0.5.0}/funvip/src/ext.py

@@ -266,10 +266,15 @@ def Trimal(fasta, out, path, algorithm="gt", threshold=0.2):
 
 
 # Modeltest
-def Modeltest_ng(fasta, out, models, thread):
+def Modeltest_ng(fasta, out, path, models, thread):
+    path_modeltestng = f"{path.sys_path}/external/modeltest-ng_Windows/modeltest-ng.exe"
     if platform == "win32":
+        CMD = f"{path_modeltestng} -i '{fasta}' -o '{out}' -t ml -p {thread} --disable-checkpoint {models}"
+        """
         logging.error("Modeltest-NG is not available in windows. Try IQTREE modeltest")
         raise Exception
+        """
+
     else:
         CMD = f"modeltest-ng -i '{fasta}' -o '{out}' -t ml -p {thread} --disable-checkpoint {models}"
 
@@ -367,8 +372,10 @@ def FastTree(fasta, out, hash_dict, path, model=""):
     if model == "skip":
         model = ""
     if platform == "win32":
+        """
         if " " in model:
-            model = f'"{model}"'
+            model = f"{model}"
+        """
         if " " in fasta:
             fasta = f'"{fasta}"'
         if " " in path.tmp:

{funvip-0.4.0 → funvip-0.5.0}/funvip/src/modeltest.py

@@ -379,6 +379,7 @@ def modeltest(V, path, opt) -> dict:
                 ext.Modeltest_ng(
                     fasta=f"{path.out_alignment}/{opt.runname}_trimmed_{group}_{gene}.fasta",
                     models=models,
+                    path=path,
                     out=f"{path.out_modeltest}/{opt.runname}_{group}_{gene}",
                     thread=opt.thread,
                 )
@@ -455,7 +456,9 @@ def modeltest(V, path, opt) -> dict:
             logging.info(
                 "IQTREE will perform ModelFinder internally in tree construction step, skipping in modeltest step"
             )
-            model_dict[group][gene] = "skip"
+            for group in group_dict:
+                for gene in group_dict[group]:
+                    model_dict[group][gene] = "skip"
 
     else:  # including opt.model_method.lower() == "none":
         for group in group_dict: