FunVIP 0.3.23.4__tar.gz → 0.3.24__tar.gz

{funvip-0.3.23.4 → funvip-0.3.24}/FunVIP.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: FunVIP
-Version: 0.3.23.4
+Version: 0.3.24
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0
@@ -28,4 +28,4 @@ Requires-Dist: unidecode==1.2.0
 Requires-Dist: xlrd==2.0.1
 Requires-Dist: xlsxwriter
 Requires-Dist: xmltodict==0.12.0
-Requires-Dist: PyQt5>=5.9.2; sys_platform != "darwin"
+Requires-Dist: PyQt5>=5.15.0; sys_platform != "darwin"

{funvip-0.3.23.4 → funvip-0.3.24}/FunVIP.egg-info/SOURCES.txt

@@ -182,6 +182,7 @@ funvip/src/validate_input.py
 funvip/src/validate_option.py
 funvip/src/validation.py
 funvip/src/version.py
+funvip/src/visualize.py
 funvip/src/__pycache__/CATV_pipe.cpython-310.pyc
 funvip/src/__pycache__/CAT_V.cpython-310.pyc
 funvip/src/__pycache__/CAT_V.cpython-39.pyc
@@ -212,6 +213,9 @@ funvip/test_dataset/penicillium/Options.config
 funvip/test_dataset/penicillium/preset.yaml
 funvip/test_dataset/penicillium/DB/DB_Penicillium.xlsx
 funvip/test_dataset/penicillium/Query/Query.xlsx
+funvip/test_dataset/sanghuangporus/preset.yaml
+funvip/test_dataset/sanghuangporus/DB/FunVIP_Sanghuangporus_db.xlsx
+funvip/test_dataset/sanghuangporus/Query/FunVIP_Sanghuangporus_query.xlsx
 funvip/test_dataset/terrei/preset.yaml
 funvip/test_dataset/terrei/DB/FunVIP_Aspergillus_db.xlsx
 funvip/test_dataset/terrei/Query/FunVIP_Aspergillus_query.xlsx

{funvip-0.3.23.4 → funvip-0.3.24}/FunVIP.egg-info/requires.txt

@@ -21,4 +21,4 @@ xlsxwriter
 xmltodict==0.12.0
 
 [:sys_platform != "darwin"]
-PyQt5>=5.9.2
+PyQt5>=5.15.0

{funvip-0.3.23.4 → funvip-0.3.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: FunVIP
-Version: 0.3.23.4
+Version: 0.3.24
 Summary: Fungal Validation & Identification Pipeline
 Author-email: Changwan Seo <wan101010@snu.ac.kr>
 License: GPL-3.0
@@ -28,4 +28,4 @@ Requires-Dist: unidecode==1.2.0
 Requires-Dist: xlrd==2.0.1
 Requires-Dist: xlsxwriter
 Requires-Dist: xmltodict==0.12.0
-Requires-Dist: PyQt5>=5.9.2; sys_platform != "darwin"
+Requires-Dist: PyQt5>=5.15.0; sys_platform != "darwin"

{funvip-0.3.23.4 → funvip-0.3.24}/README.md

@@ -1,107 +1,52 @@
-[![DOI](https://zenodo.org/badge/588465720.svg)](https://zenodo.org/doi/10.5281/zenodo.10714946)
 
-#### This is Beta release. Bug reports are welcomed
-
-## Scheduling
-### Beta release part 1 (2023 Feburary ~ As paper published, ver 0.3)
-- Will be tested by our lab memebers to fix bugs and advance features
-
-### Beta release part 2 (As paper published ~ When pipeline gets stabled, ver 0.4)
-- Will be tested by peer taxonomists
-
-### Stable release (ver 1.0)
-
-
-# FunVIP
-"Fun"gal "V"alidation & "I"dentification "P"ipeline
-
-An automatic tree-based sequence identification and validation pipeline for fungal species
+# FunVIP [![DOI](https://zenodo.org/badge/588465720.svg)](https://zenodo.org/doi/10.5281/zenodo.10714946)
+### **Fun**gal **V**alidation & **I**dentification **P**ipeline
+#### An automatic tree-based sequence identification and validation pipeline for fungal (or maybe other) species
 
 - Automatic tree-based identification
 - Works with multigene
 - Data validation algorithm implemented
 
+![figure1 - ver17A](https://github.com/user-attachments/assets/22a50a62-14e8-41a7-87a0-8f5a1f9c3f62)
 
-## See [tutorial](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial.md) for step by step tutorial
-## See [documentation](https://github.com/Changwanseo/FunVIP/blob/main/Documentation.md) for advanced usage
-
+This is Beta release. Bug reports are welcomed
+<br><br/>
 
+## Tutorial
+* [Part 1 - Getting started!](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial.md)
+* [Part 2 - Preparing database and query](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/tutorial2.md)
+* [Advacned tips](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/advanced.md)
+<br><br/>
+## Documentation
+* See [Documentation](https://github.com/Changwanseo/FunVIP/blob/main/Documentation.md) for advanced usage !
+<br><br/>
 ## Requirements
-- Conda environment (See [https://www.anaconda.com/products/individual](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) for how to install conda environment)
-
-<!--
-## Installation with conda (May not work with Linux or Mac)
-1. ```conda create -n FunVO{ python=3.10```
-2. ```conda activate FunVIP```
-3. ```conda install -c cwseo FunVIP```
-4. run ```FunVIP --test Terrei --email [your email] ``` to check installation
-If this one fails, use next one
--->
+- Conda environment
 
+\* See [https://www.anaconda.com/products/individual](https://conda.io/projects/conda/en/latest/user-guide/install/index.html) for how to install conda environment
+<br><br/>
 ## Installation
-### Windows
-1. Install visual c++ [here](https://visualstudio.microsoft.com/visual-cpp-build-tools/)
-2. ```conda create -n FunVIP python>=3.8```
-3. ```conda activate FunVIP```
-4. ```pip install FunVIP```
-5. run ```FunVIP --test Terrei --email [your email] ``` to check installation
-
-* For upgrade use this command
-``` pip install FunVIP --upgrade ```
-
-### Linux
-1. ```conda create -n FunVIP python>=3.8```
-2. ```conda activate FunVIP```
-3. ```pip install FunVIP```
-4. ```conda config --add channels conda-forge```
-5. ```conda install -c bioconda raxml iqtree "modeltest-ng>=0.1.7" mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree```
-6. run ```FunVIP --test Terrei --email [your email] ``` to check installation
-
-
-* For intel mac system, this method probably work, but we couldn't test it because we don't have any intel mac device. We're looking for feedbacks in intel mac
-
-### Apple Silicon Mac
-1. ```CONDA_SUBDIR=osx-64 conda create -n FunVIP python>=3.8```
-2. ```conda activate FunVIP```
-3. ```conda config --env --set subdir osx-64```
-4. ```conda install pyqt```
-5. ```pip install FunVIP```
-6. ```conda install -c bioconda raxml iqtree mmseqs2 "blast>=2.12" mafft trimal gblocks fasttree```
-7. run ```FunVIP --test Terrei --email [your email] ``` to check installation
-
-### Installation from source (For developers and core users)
-* this is for developmental steps
-1. ```git clone https://github.com/Changwanseo/FunVIP.git```
-2. Move to ```~/FunVIP```
-3. ```conda create -n FunVIP python=3.10```
-4. ```conda activate FunVIP```
-5. ```pip install ./```
-6. run ```FunVIP --test Terrei --email [your email]``` to check installation
-
-
+* [Windows](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Windows)
+* [Mac - apple silicon](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Apple )
+* [Linux](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Linux)
+* [from source](https://github.com/Changwanseo/FunVIP/blob/main/tutorial/installation.md##Installation)
+<br><br/>
 ## Usage
 ```FunVIP --db {Your database file} --query {Your query file} --email {Your email} --gene {Your genes} --preset {fast or accurate}```
-
+<br><br/>
 ### Example
-```FunVIP --db Penicillium.xlsx --query Query.xlsx --email {Your email} --gene ITS BenA RPB2 CaM --preset fast```
-
+```FunVIP --db Penicillium.xlsx --query Query.xlsx --email {Your email} --thread 8 --gene ITS BenA RPB2 CaM --preset fast```
 
 \* See documentation for detailed usage
+<br><br/>
 
 
 
-<!--### GUI mode (\*Currently under development)
-1. Go to ~/FunID-dev
-2. ```streamlit run FunID_GUI.py```
-* GUI run is on experimental
-* If you want to edit GUI options, edit ```Option_manager.xlsx``` and variables in ```FunID_GUI.py```
-
-### Server mode (\* Currently under development)-->
 
 
 
 ## How to make database?
-![Fig 2 Database and command configuration of FunID (ver2) ](https://github.com/Changwanseo/FunVIP/assets/64393882/9ba71eb9-91e9-4c0b-ac60-b9b7be993694)
+![figure1 - ver17B](https://github.com/user-attachments/assets/0043e6f6-7470-4c2b-bc96-b51f41c43ee4)
 
 
 
@@ -132,11 +77,28 @@ tabular (```.xlsx```, ```.csv```,  ```.parquet```, ```.ftr```) form
 * ```/Tree/{section}_{gene}.svg``` : Final collapsed tree in svg format. Can be edited in vector graphics programs, or in powerpoint (by ungroup)
 * ```/Tree/{section}_{gene}_original.svg ``` : Uncollapsed tree for inspection
 
-## How does FunVIP work?
-![figure1 - ver4](https://github.com/Changwanseo/FunID/assets/64393882/6a366d32-6aaf-4d0c-8102-8c7dd5fda4c2)
-
+## Scheduling
+1. Beta release part 1 (2023 Feburary ~ As paper published, ver 0.3)
+Will be tested by our lab memebers to fix bugs and advance features
+2. Beta release part 2 (As paper published ~ When pipeline gets stabled, ver 0.4)
+Will be tested by peer taxonomists
+3. Stable release (ver 1.0)
 
+## License
+[GPL 3.0](https://github.com/Changwanseo/FunVIP/blob/main/LICENSE)
 
 
-## License
-GPL 3.0
+<!--
+## Installation with conda (May not work with Linux or Mac)
+1. ```conda create -n FunVO{ python=3.10```
+2. ```conda activate FunVIP```
+3. ```conda install -c cwseo FunVIP```
+4. run ```FunVIP --test Terrei --email [your email] ``` to check installation
+If this one fails, use next one
+-->
+<!--### GUI mode (\*Currently under development)
+1. Go to ~/FunID-dev
+2. ```streamlit run FunID_GUI.py```
+* GUI run is on experimental
+* If you want to edit GUI options, edit ```Option_manager.xlsx``` and variables in ```FunID_GUI.py```
+### Server mode (\* Currently under development)-->

{funvip-0.3.23.4 → funvip-0.3.24}/funvip/main.py

@@ -121,7 +121,6 @@ def main():
         )
 
         # logging works well from here
-
         save.save_session(opt=opt, path=path, global_var=locals(), var=vars())
 
         time_setup = time()
@@ -173,7 +172,7 @@ def main():
         V.generate_dataset(opt)
 
         # Appending outgroup
-        logging.info("Appending outgroup")
+        logging.info("APPENDING OUTGROUP")
         # For non-concatenated outgroup
         # ready for multiprocessing run
         # Pushing all v to multiprocessing requires too much memory
@@ -181,6 +180,7 @@ def main():
 
         # remove invalid dataset from downstream analysis
         V.remove_invalid_dataset()
+
         # Save dataset to outgroups
         V.save_dataset(path, opt)
 
@@ -316,6 +316,11 @@ def main():
         # Running visualization
         V, path, opt = tree_interpretation_pipe.pipe_tree_interpretation(V, path, opt)
 
+        """
+        # For final visualization
+        V, path, opt = visualize.visualize(V, path, opt)
+        """
+
         # move tree image files
         tool.cleanup_tree(path)
         tool.cleanup_tree_image(path)

{funvip-0.3.23.4 → funvip-0.3.24}/funvip/src/align.py

@@ -4,6 +4,7 @@ import logging
 from Bio import SeqIO
 import multiprocessing as mp
 import math
+import os
 
 
 # Each module to be run in alignment multiprocessing
@@ -50,6 +51,25 @@ def module_alignment(
 def pipe_alignment(V, path, opt):
     alignment_opt = opt_generator(V, opt, path, step="alignment")
 
+    # Before running alignment, clear previous files
+    for file in os.listdir(path.out_alignment):
+        if any(file.endswith(x) for x in [".fasta", ".partition", ".colnumbering"]):
+            os.remove(
+                f"{path.out_alignment}/{file}",
+            )
+
+    for file in os.listdir(f"{path.out_alignment}/failed/"):
+        if any(file.endswith(x) for x in [".fasta", ".partition", ".colnumbering"]):
+            os.remove(
+                f"{path.out_alignment}/failed/{file}",
+            )
+
+    for file in os.listdir(f"{path.out_alignment}/hash/"):
+        if any(file.endswith(x) for x in [".fasta", ".partition", ".colnumbering"]):
+            os.remove(
+                f"{path.out_alignment}/hash/{file}",
+            )
+
     # Multiprocessing start
     # Thread optimizations
     # Using O(N^2L^2) of MAFFT-G-ins-i

{funvip-0.3.23.4 → funvip-0.3.24}/funvip/src/cluster.py

@@ -6,6 +6,7 @@ import os, sys, subprocess
 from copy import deepcopy
 from functools import lru_cache
 from time import sleep
+from time import time
 
 # from Bio.Blast import NCBIXML
 from Bio import SeqIO
@@ -29,6 +30,8 @@ def append_query_group(V):
 
     V.cSR["query_group"] = V.cSR["qseqid"].apply(lambda x: group_dict.get(x))
 
+    logging.debug(V.cSR["query_group"])
+
     # Indicate queries without any corresponding group
     for FI in V.list_FI:
         if FI.adjusted_group == "" and not ("noseq" in FI.issues):
@@ -95,18 +98,15 @@ def assign_gene(result_dict, V, cutoff=0.99):
 
 
 # cluster each of FI object and assign group
-def cluster(FI, V, path, opt):
-    list_group = deepcopy(V.list_group)
+"""
+def cluster(FI, V_list_group, V_cSR, path, opt):
+    list_group = deepcopy(V_list_group)
 
     # Reduce search df space by selecting only FI related ones
-    df_group = V.cSR.groupby(V.cSR["qseqid"])
-    list_id = list(set(V.cSR["qseqid"]))
+    df_group = V_cSR.groupby(V_cSR["qseqid"])
+    list_id = list(set(V_cSR["qseqid"]))
     df_search = df_group.get_group(FI.hash)
 
-    # delete V to reduce memory assumption
-    del V
-    gc.collect()
-
     # If no evidence available, return it
     if df_search is None:
         # if confident is True, no adjusted_group for db is normal situation
@@ -124,6 +124,11 @@ def cluster(FI, V, path, opt):
     else:
         # sorting has peformed after split for better performance
         sorted_df_search = df_search.sort_values(by=["bitscore"], ascending=False)
+
+        # Garbage collection
+        del df_search
+        gc.collect()
+
         # reset index to easily get maximum
         sorted_df_search.reset_index(inplace=True, drop=True)
         # get result stasifies over cutoff
@@ -133,7 +138,6 @@ def cluster(FI, V, path, opt):
         ]
 
         # Garbage collection
-        del df_search
         del sorted_df_search
         gc.collect()
 
@@ -141,7 +145,7 @@ def cluster(FI, V, path, opt):
         list_group = list(set(cutoff_df["subject_group"]))
 
         # if first time of group update
-        if FI.adjusted_group == "" or FI.adjusted_group == "":
+        if FI.adjusted_group == "":
             # if only 1 group available, take it
             if group_count == 1:
                 FI.adjusted_group = list_group[0]
@@ -171,16 +175,80 @@ def cluster(FI, V, path, opt):
             return FI, list_group[0]
         else:
             return FI, None
+"""
+
+
+def cluster(FI, V_list_group, V_cSR, path, opt):
+    # Reduce memory by focusing on relevant rows
+    df_search = V_cSR[V_cSR["qseqid"] == FI.hash]
+
+    if df_search.empty:
+        # If confident is False and FI datatype is "db"
+        if opt.confident is False and FI.datatype is "db":
+            logging.warning(f"No adjusted_group assigned to {FI}")
+        FI.adjusted_group = FI.group
+        return FI
+
+    # For db sequence with group, retain it
+    elif not (FI.group == "") and FI.datatype == "db":
+        FI.adjusted_group = FI.group
+        return FI
+
+    # Update group if sequence doesn't have one
+    else:
+        df_search = df_search.sort_values(by=["bitscore"], ascending=False)
+
+        # Apply cutoff filter to reduce DataFrame size
+        cutoff = df_search["bitscore"].iloc[0] * opt.cluster.cutoff
+        cutoff_df = df_search[df_search["bitscore"] > cutoff]
+
+        # Clear unused data
+        del df_search
+        gc.collect()
+
+        # Extract group information
+        unique_groups = set(cutoff_df["subject_group"])
+        group_count = len(unique_groups)
+
+    if FI.adjusted_group == "":
+        if group_count == 1:
+            FI.adjusted_group = unique_groups.pop()
+        elif group_count == 0:
+            logging.warning(
+                f"Query seq in {FI.id} cannot be assigned to group. Check sequence."
+            )
+        elif group_count >= 2:
+            logging.warning(
+                f"Query seq in {FI.id} has multiple matches to groups: {list(unique_groups)}"
+            )
+            FI.adjusted_group = next(
+                iter(unique_groups)
+            )  # Pick one (deterministic for testing)
+        else:
+            logging.error("DEVELOPMENTAL ERROR IN GROUP ASSIGN")
+            raise Exception
+
+        logging.info(f"{FI.id} has clustered to {FI.adjusted_group}")
+
+    # if group already updated
+    else:
+        if not (FI.adjusted_group in unique_groups):
+            logging.warning(f"Clustering result collides for {FI.id}")
+
+    if unique_groups:
+        return FI
+    else:
+        return FI
 
 
 ### Append outgroup to given group-gene dataset by search matrix
-def append_outgroup(V, df_search, gene, group, path, opt):
+def append_outgroup(V_list_FI, df_search, gene, group, path, opt):
     logging.info(f"Appending outgroup on group: {group}, Gene: {gene}")
-    list_FI = deepcopy(V.list_FI)
+    list_FI = V_list_FI
 
     # In multiprocessing, delete V to reduce memory consumption
-    del V
-    gc.collect()
+    # del V
+    # gc.collect()
 
     # ready for by sseqid hash, which group to append
     # this time, append adjusted group
@@ -336,6 +404,8 @@ def append_outgroup(V, df_search, gene, group, path, opt):
 
 
 def group_cluster_opt_generator(V, opt, path):
+    opt_cluster = []
+
     # cluster(FO, df_search, V, path, opt)
     if len(V.list_qr_gene) == 0:
         logging.error(
@@ -347,11 +417,20 @@ def group_cluster_opt_generator(V, opt, path):
     else:
         # cluster group by concatenated search result
         list_id = list(set(V.cSR["qseqid"]))
+
         for FI in V.list_FI:
             if FI.hash in list_id:
-                V.opt_cluster.append((FI, V, path, opt))
+                opt_cluster.append(
+                    (
+                        FI,
+                        V.list_group,
+                        V.cSR[["qseqid", "bitscore", "subject_group"]],
+                        path,
+                        opt,
+                    )
+                )
 
-    return V
+    return opt_cluster
 
 
 # opts ready for multithreading in outgroup append
@@ -371,7 +450,9 @@ def outgroup_append_opt_generator(V, path, opt):
                 df_group_ = df_group.get_group(group)
                 # Generating outgroup opt for multiprocessing
                 for gene in V.dict_dataset[group]:
-                    opt_append_outgroup.append((V, df_group_, gene, group, path, opt))
+                    opt_append_outgroup.append(
+                        (V.list_FI, df_group_, gene, group, path, opt)
+                    )
 
             except:
                 logging.warning(
@@ -389,36 +470,47 @@ def pipe_cluster(V, opt, path):
 
         # cluster opt generation for multiprocessing
         # (FI, V, path, opt)
-        V = group_cluster_opt_generator(V, opt, path)
+        opt_cluster = group_cluster_opt_generator(V, opt, path)
 
         # run multiprocessing start
         if opt.verbose < 3:
             p = mp.Pool(opt.thread)
-            V.rslt_cluster = p.starmap(cluster, V.opt_cluster)
+            rslt_cluster = p.starmap(cluster, opt_cluster)
             p.close()
             p.join()
         else:
             # non-multithreading mode for debugging
-            V.rslt_cluster = [cluster(*o) for o in V.opt_cluster]
+            rslt_cluster = [cluster(*o) for o in opt_cluster]
         # gather cluster result
-        for cluster_result in V.rslt_cluster:
-            FI = cluster_result[0]
+        for cluster_result in rslt_cluster:
+            FI = cluster_result
             logging.debug((FI.id, FI.datatype, FI.group, FI.adjusted_group))
 
         # replace group assigning result
         # collect FI from cluster result
-        replace_FI = [r[0] for r in V.rslt_cluster]
+        # somethings been duplicated here
+        replace_FI = [r for r in rslt_cluster]
+
         # collect hash
         replace_hash_FI = [FI.hash for FI in replace_FI]
+
         # maintain not clustered result and append clustered result
-        V.list_FI = [
-            FI for FI in V.list_FI if not (FI.hash in replace_hash_FI)
-        ] + replace_FI
+        V.list_FI = [FI for FI in V.list_FI if not (FI.hash in replace_hash_FI)]
+        V.list_FI += replace_FI
+
         # For syncyhronizing FI in dict_hash_FI to prevent error
         for FI in replace_FI:
             V.dict_hash_FI[FI.hash] = FI
 
-        V.list_group = list(set([r[1] for r in V.rslt_cluster if (not (r[1] is None))]))
+        V.list_group = list(
+            set(
+                [
+                    r.adjusted_group
+                    for r in rslt_cluster
+                    if (not (r.adjusted_group == ""))
+                ]
+            )
+        )
 
         if opt.queryonly is True:
             for FI in V.list_FI:
@@ -429,6 +521,7 @@ def pipe_cluster(V, opt, path):
             for FI in V.list_FI:
                 V.dict_hash_FI[FI.hash] = FI
 
+        # For debugging
         for FI in V.list_FI:
             logging.debug((FI.id, FI.datatype, FI.group, FI.adjusted_group))
 

{funvip-0.3.23.4 → funvip-0.3.24}/funvip/src/command.py

@@ -54,7 +54,7 @@ class CommandParser:
         )
         group_test.add_argument(
             "--test",
-            help="Use test dataset, [Penicillium]",
+            help="Use test dataset, [Penicillium, Terrei, Sanghuangporus]",
             type=str,
         )
 

{funvip-0.3.23.4 → funvip-0.3.24}/funvip/src/concatenate.py

@@ -37,25 +37,13 @@ def combine_alignment(V, opt, path):
                         )
                         len_dict[gene] = len(fasta_list[0].seq)
                         seq_dict[gene] = {}
+
+                        V_concat = V.dict_dataset[group]["concatenated"]
+
                         total_dataset = (
-                            [
-                                FI.hash
-                                for FI in V.dict_dataset[group][
-                                    "concatenated"
-                                ].list_qr_FI
-                            ]
-                            + [
-                                FI.hash
-                                for FI in V.dict_dataset[group][
-                                    "concatenated"
-                                ].list_db_FI
-                            ]
-                            + [
-                                FI.hash
-                                for FI in V.dict_dataset[group][
-                                    "concatenated"
-                                ].list_og_FI
-                            ]
+                            [FI.hash for FI in V_concat.list_qr_FI]
+                            + [FI.hash for FI in V_concat.list_db_FI]
+                            + [FI.hash for FI in V_concat.list_og_FI]
                         )
                         for seq in fasta_list:
                             # if available hash
@@ -155,7 +143,30 @@ def concatenate_df(V, path, opt):
         # Leave non-empty dataframes
         if isinstance(V.dict_gene_SR[gene], pd.DataFrame):
             gene_list.append(gene)
+
             df = deepcopy(V.dict_gene_SR[gene].set_index(["qseqid", "sseqid"]))
+
+            """
+            df = deepcopy(
+                V.dict_gene_SR[gene]
+                .set_index(["qseqid", "sseqid"])
+                .drop(
+                    columns=[
+                        "pident",
+                        "length",
+                        "mismatch",
+                        "gaps",
+                        "qstart",
+                        "qend",
+                        "sstart",
+                        "send",
+                        "evalue",
+                        "bitscore",
+                        "subject_group",
+                    ]
+                )
+            )
+            """
             df_list.append(df)
 
     if len(df_list) <= 0:
@@ -179,6 +190,7 @@ def concatenate_df(V, path, opt):
         df_multigene_regression_ori = pd.concat(df_list, axis=1)
 
         # Drop unnecessary columns for processing
+
         df_multigene_regression_ori.drop(
             columns=[
                 "pident",

{funvip-0.3.23.4 → funvip-0.3.24}/funvip/src/dataset.py

@@ -320,16 +320,18 @@ class FunVIP_var:
         for FI in self.list_FI:
             if FI.hash in self.dict_hash_FI:
                 h = FI.hash
-
                 # final species
                 if FI.final_species != self.dict_hash_FI[h].final_species:
+                    # If final species was empty
                     if FI.final_species == "":
                         FI.final_species = self.dict_hash_FI[h].final_species
+                    # If final species in hash dict was empty
                     elif self.dict_hash_FI[h].final_species == "":
                         self.dict_hash_FI[h].final_species = FI.final_species
+                    # If they collides, it is error
                     else:
                         logging.error(
-                            f"DEVELOPMNETAL ERROR Both list_FI and dict_hash_FI have conflicting final species, {FI.final_species} and {self.dict_hash_FI[h].final_species}"
+                            f"DEVELOPMNETAL ERROR Both list_FI and dict_hash_FI have conflicting final species, {FI.final_species} and {self.dict_hash_FI[h].final_species} for hash {h}"
                         )
                         raise Exception
 
@@ -403,6 +405,15 @@ class FunVIP_var:
 
     # save fasta for outgroup adjusted fasta
     def save_dataset(self, path, opt):
+        # Before save_dataset, remove pre-existing results
+        for file in os.listdir(path.out_adjusted):
+            if file.endswith(".fasta"):
+                os.remove(f"{path.out_adjusted}/{file}")
+
+        for file in os.listdir(f"{path.out_adjusted}/hash/"):
+            if file.endswith(".fasta"):
+                os.remove(f"{path.out_adjusted}/hash/{file}")
+
         for group in self.dict_dataset:
             for gene in self.dict_dataset[group]:
                 if not (gene == "concatenated"):