FunVIP 0.3.20__py3-none-any.whl

FunVIP-0.3.20.dist-info/METADATA

@@ -0,0 +1,32 @@
+Metadata-Version: 2.1
+Name: FunVIP
+Version: 0.3.20
+Summary: Fungal Validation & Identification Pipeline
+Author-email: Changwan Seo <wan101010@snu.ac.kr>
+License: GPL-3.0
+Project-URL: Homepage, https://github.com/Changwanseo/FunVIP
+Requires-Python: <3.13,>3.8
+License-File: LICENSE
+Requires-Dist: biopython ==1.78
+Requires-Dist: ete3 ==3.1.3
+Requires-Dist: Cython
+Requires-Dist: dendropy
+Requires-Dist: GenMine >=1.0.13
+Requires-Dist: lxml
+Requires-Dist: matplotlib
+Requires-Dist: numpy
+Requires-Dist: openpyxl ==3.0.9
+Requires-Dist: pandas ==1.4.2
+Requires-Dist: plotly ==5.9.0
+Requires-Dist: psutil
+Requires-Dist: pyyaml
+Requires-Dist: sip >=4.19.4
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
+Requires-Dist: tabulate
+Requires-Dist: unidecode ==1.2.0
+Requires-Dist: xlrd ==2.0.1
+Requires-Dist: xlsxwriter
+Requires-Dist: xmltodict ==0.12.0
+Requires-Dist: PyQt5 >=5.9.2 ; sys_platform != "darwin"
+

FunVIP-0.3.20.dist-info/RECORD

@@ -0,0 +1,36 @@
+data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+external/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+external/BLAST_Windows/bin/cleanup-blastdb-volumes.py,sha256=RhlY30loTnSWp_xGc43GPyv1My9QD2NaxWWqy_fwHEk,6243
+src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+src/align.py,sha256=HVHwur7q6JS5lIvGCRCSUe7fzaCwvqAqcpA9SvMYCmQ,4100
+src/cluster.py,sha256=ZNbVJSzRAeDhW9K76fiLT0O0giTqdSGBrC0KhZeAqHM,19438
+src/command.py,sha256=nAFIB4IAGVKqGkuOGYx98tIOM4T80lyDzfNI3Swxx1U,13840
+src/concatenate.py,sha256=1GA_akqcEKXrsqC0nkNFmrRo49hWH5tcngGFcRLawis,13871
+src/dataset.py,sha256=sbxrmZBQZ5Zf8sCwowtJRFKO4p_p_CDXCTpdIs-M_h4,31797
+src/ext.py,sha256=CQ_8LKjGgZIq5p_tWzmSxgw1NmB0ULMVAlLJXpZSMcw,14722
+src/hasher.py,sha256=6GV8b_sY8nP5_7diV1UoFMDEjMpINpdCynw9cS_-ni8,2810
+src/initialize.py,sha256=13ci7eudMN0JjPZ4u-ChUYfKQ7ziyjrgX4VmbA0uUtc,11522
+src/logger.py,sha256=ssUoj-MOK1E_M3fYalfVY7vdHxqteLw7sGxYbXI6uVg,2089
+src/logics.py,sha256=o6_buuuy9qCeqj5sWR56NVAWRI14i5lOd6ui6yprm9U,2785
+src/modeltest.py,sha256=e6tIuALrziVbqWCErJBNhKbvxv9ELZ3zTliTjI_oF34,16418
+src/ncbi.py,sha256=Ed14A519oalurEVCckaoFNV6dIqHZU55gPllOtJgHVo,5072
+src/opt_generator.py,sha256=Ts1IIMrd3xV9lKwtgGp-StzVU37NKu_xu_njRTKQTss,3088
+src/patch.py,sha256=Mt5OzPrT-voxOVm5m9sUUuoVZQAY2zjPmTvzFBgYiUY,7674
+src/reporter.py,sha256=MO9Q8wXpTksw4hI7xGDAntMX7vu3IXz92KLPrvz7lSg,40744
+src/save.py,sha256=k40XRmawNFJH7w2GK44WubnT3TS7tE9UmpmKqyVwjbk,5678
+src/search.py,sha256=KWy7JPlfsvJoD-JbPyTZpaeA-_J4AOiJKX-hTuMv2cI,15087
+src/tool.py,sha256=4rAYeEmHIxgztDgCrqfRXFumactCcxrHe0HTmDd1L7w,9781
+src/tree.py,sha256=6RqcfrVM3_EITzMACsKzo62epwQpGtFeF6XiLSa3bsc,9266
+src/tree_interpretation.py,sha256=DA7jxbayCWm6AbDEKo-KNbnExjF1q8FxBcCp9kF4quI,54919
+src/tree_interpretation_pipe.py,sha256=IypkGyYW6pnJx5grtTGvMpaSzn0K9R_QJd7XQXBxoJ4,26991
+src/trim.py,sha256=XAKywgv_d4pHpqs7V6_xmMRG2M_RxNHHUbGRHigX2Ys,4238
+src/validate_input.py,sha256=8DUkCn0_WfB4BJBkUF2BnnsZenzpoOgOLSnURzxzfTQ,31610
+src/validate_option.py,sha256=mOfF61uCRGw9kRD56prvjQ9UERYcDK1_20bCbR1NYhY,60353
+src/validation.py,sha256=sUuzvhiq3FpQbU6BIVwJsLgiCBlpYnMBZXZuj_zN5R4,1040
+src/version.py,sha256=TRisvZ5lVreWU87DPgDgMB4tEIefjWZyJOkCbTX2d-Q,12275
+FunVIP-0.3.20.dist-info/LICENSE,sha256=IwGE9guuL-ryRPEKi6wFPI_zOhg7zDZbTYuHbSt_SAk,35823
+FunVIP-0.3.20.dist-info/METADATA,sha256=8qfvnXHDYfhk9a8ABOIM5HI89bPlQA7aAWVcg_b2uyo,901
+FunVIP-0.3.20.dist-info/WHEEL,sha256=MxOBbn7YEn8Le9UJ_3pkKUGGC2zJe2Y86VEFkJgyT7E,115
+FunVIP-0.3.20.dist-info/entry_points.txt,sha256=32U6s3fPYXUVP4xRuBkownB-gzt-QAUxAEVnD8V4F9E,69
+FunVIP-0.3.20.dist-info/top_level.txt,sha256=Bc9RcY5o1hc_9FJtBHbyryHn86AjoeyB4Cuw-YHOQgg,18
+FunVIP-0.3.20.dist-info/RECORD,,

FunVIP-0.3.20.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.43.0) + setuptools-ext (0.9)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

FunVIP-0.3.20.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+FunID = funvip.main:main
+FunVIP = funvip.main:main

FunVIP-0.3.20.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+data
+external
+src

external/BLAST_Windows/bin/cleanup-blastdb-volumes.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""
+# $Id: cleanup-blastdb-volumes.py 590894 2019-08-07 14:59:53Z camacho $
+# ===========================================================================
+# 
+#                            PUBLIC DOMAIN NOTICE
+#               National Center for Biotechnology Information
+# 
+#  This software/database is a "United States Government Work" under the
+#  terms of the United States Copyright Act.  It was written as part of
+#  the author's official duties as a United States Government employee and
+#  thus cannot be copyrighted.  This software/database is freely available
+#  to the public for use. The National Library of Medicine and the U.S.
+#  Government have not placed any restriction on its use or reproduction.
+# 
+#  Although all reasonable efforts have been taken to ensure the accuracy
+#  and reliability of the software and data, the NLM and the U.S.
+#  Government do not and cannot warrant the performance or results that
+#  may be obtained by using this software or data. The NLM and the U.S.
+#  Government disclaim all warranties, express or implied, including
+#  warranties of performance, merchantability or fitness for any particular
+#  purpose.
+# 
+#  Please cite the author in any work or product based on this material.
+# 
+# ===========================================================================
+# 
+# Author:  Christiam Camacho
+# 
+# File Description:
+#   Script to remove needless BLAST database files.
+# 
+# ===========================================================================
+"""
+import argparse, os, configparser
+import unittest, tempfile
+from pathlib import Path
+from glob import glob
+
+VERSION = '1.0'
+DESC = r""" Remove needless BLAST database volumes.  """
+
+
+class Tester(unittest.TestCase):
+    """ Testing class for this script. """
+
+    def test_blastdb_config_invalid(self):
+        rv = get_blastdb_from_ncbi_config("/dev/null")
+        self.assertIsNone(rv)
+
+    def test_blastdb_config(self):
+        config = configparser.ConfigParser()
+        expected = "/blast/db/blast"
+        config['BLAST'] = {'BLASTDB': expected}
+        tf = tempfile.NamedTemporaryFile(mode="wt")
+        config.write(tf)
+        tf.flush()
+        rv = get_blastdb_from_ncbi_config(tf.name)
+        self.assertEqual(expected, rv)
+
+    def test_blastdb_finder(self):
+        tal = tempfile.NamedTemporaryFile(suffix=".pin")
+        dbname = find_blastdb(tal.name[:-4], True)
+        self.assertEqual(dbname, tal.name[:-4])
+
+
+def find_blastdb(name: str, is_prot: bool) -> str:
+    """ Returns full path to BLAST database or None. """
+    alias_file = "{}.{}al".format(name, "p" if is_prot else "n")
+    index_file = "{}.{}in".format(name, "p" if is_prot else "n")
+    if os.path.exists(alias_file) or os.path.exists(index_file):
+        return name
+
+    if "BLASTDB" in os.environ:
+        alf = os.path.join(os.environ["BLASTDB"], alias_file)
+        idxf = os.path.join(os.environ["BLASTDB"], index_file)
+        if os.path.exists(alf) or os.path.exists(idxf):
+            return os.path.join(os.environ["BLASTDB"], name)
+
+    paths = [ os.getcwd(), str(Path.home()) ]
+    if "NCBI" in os.environ:
+        paths.append(os.path.join(os.environ["NCBI"]))
+
+    for path in paths:
+        for fname in [ ".ncbirc", "ncbi.ini" ]:
+            ncbirc = os.path.join(path, fname)
+            if os.path.exists(ncbirc):
+                blastdb = get_blastdb_from_ncbi_config(ncbirc)
+                if blastdb is not None:
+                    alf = os.path.join(blastdb, alias_file)
+                    idxf = os.path.join(blastdb, index_file)
+                    if os.path.exists(alf) or os.path.exists(idxf):
+                        return os.path.join(blastdb, name)
+
+
+def get_blastdb_from_ncbi_config(config_file: str) -> str:
+    """ Return the BLASTDB setting from the NCBI configuration file or None. """
+    config = configparser.ConfigParser()
+    config.read(config_file)
+    if 'BLAST' in config and 'BLASTDB' in config['BLAST']:
+        return config['BLAST']['BLASTDB']
+
+
+def main():
+    """ Entry point into this program. """
+    parser = create_arg_parser()
+    args = parser.parse_args()
+
+    ext = args.dbtype[0]
+    db = find_blastdb(args.db, ext == 'p')
+    if db == None:
+        print("Cannot find {} {} BLAST database".
+              format("protein" if ext == 'p' else "nucleotide", args.db),
+              file=sys.stderr)
+        return 1
+
+    alias_file = "{}.{}al".format(db, ext)
+    if not os.path.exists(alias_file):
+        return 1
+
+    with open(alias_file, "rt") as al:
+        for line in al:
+            if not line.startswith("DBLIST"):
+                continue
+            vols = list(map(lambda x: x.replace('"', ''), line.split()[1:]))
+            for existing_vols in sorted(glob("{}.*.{}in".format(db, ext))):
+                vol_name = os.path.basename(existing_vols)[:-4]
+                if vol_name in vols:
+                    continue
+                if args.dry_run:
+                    print("Will remove extra volume {}".format(existing_vols[:-4]))
+                to_rm = glob("{}??".format(existing_vols[:-2]))
+                to_rm += glob("{}.tar.gz.md5".format(existing_vols[:-4]))
+                for f in to_rm:
+                    if not args.dry_run:
+                        os.unlink(f)
+                        print("Removed {}".format(f))
+                    elif args.verbose > 0:
+                        print("Will remove {}".format(f))
+
+    return 0
+
+
+def create_arg_parser():
+    """ Create the command line options parser object for this script. """
+    parser = argparse.ArgumentParser(description=DESC)
+    parser.add_argument("-db", required=True, help="BLAST database name")
+    parser.add_argument("-dbtype", help="Molecule type", required=True,
+                        choices=["prot", "nucl"])
+    parser.add_argument("-dry-run", action='store_true',
+                        help="Do not delete any files, just list them")
+    parser.add_argument('-version', action='version',
+                        version='%(prog)s ' + VERSION)
+    parser.add_argument("-verbose", action="count", default=0,
+                        help="Increase output verbosity")
+    return parser
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main())
+

src/align.py

@@ -0,0 +1,124 @@
+from funvip.src import ext
+from funvip.src.opt_generator import opt_generator
+import logging
+from Bio import SeqIO
+import multiprocessing as mp
+import math
+
+
+# Each module to be run in alignment multiprocessing
+def module_alignment(
+    in_fasta,
+    out_fasta,
+    path,
+    thread,
+    mafft_algorithm,
+    adjustdirection,
+    op,
+    ep,
+):
+    # Running MAFFT
+    ext.MAFFT(
+        fasta=in_fasta,
+        out=out_fasta,
+        path=path,
+        thread=thread,
+        algorithm=mafft_algorithm,
+        adjust=adjustdirection,
+        maxiterate=1000,
+        op=op,
+        ep=ep,
+    )
+
+    # to prevent reversed sequence making error
+    fasta_list = list(SeqIO.parse(out_fasta, "fasta"))
+    for seq in fasta_list:
+        if seq.description.startswith("_R_"):
+            seq.id = ""
+            seq.description = seq.description[3:]
+    SeqIO.write(fasta_list, out_fasta, "fasta")
+
+    # Fix unexpected spaces on mafft
+    with open(out_fasta, "r") as fr:
+        alignment = fr.read()
+
+    with open(out_fasta, "w") as fw:
+        fw.write(alignment.replace(" HS", "HS"))
+
+
+# Alignment pipeline
+def pipe_alignment(V, path, opt):
+    alignment_opt = opt_generator(V, opt, path, step="alignment")
+
+    # Multiprocessing start
+    # Thread optimizations
+    # Using O(N^2L^2) of MAFFT-G-ins-i
+    # Use multithreading optimizations in non-verbose mode
+    if opt.verbose < 3:
+        time_consumptions = []
+        ## Simulate time comsumption
+        # Parsing probable time_consumption
+        for aln_opt in alignment_opt:
+            before_aln_file = aln_opt[0]
+            before_seq_list = list(SeqIO.parse(before_aln_file, "fasta"))
+            N = len(before_seq_list)
+            L = max([len(seq.seq) for seq in before_seq_list])
+            time_consumptions.append(N * N * L * L)
+
+        time_consumptions = sorted(time_consumptions, reverse=True)
+        best_distribute = 1
+        best_thread = opt.thread
+        best_time = 99999999999999999999
+        # Optimize thread numbers
+        for distribute_num in range(opt.thread):
+            time_buffers = {}
+
+            # Threads to be distributed
+            each_thread_num = int(opt.thread / (distribute_num + 1))
+            # Efficiency multipler estimated from
+            # Rubio-Largo, A., Castelli, M., Vanneschi, L., & Vega-Rodríguez, M. A. (2018). A parallel multiobjective metaheuristic for multiple sequence alignment. Journal of Computational Biology, 25(9), 1009-1022.
+
+            efficiency_multiplier = 1.0258 * math.log(each_thread_num) + 1
+
+            for i in range(distribute_num + 1):
+                time_buffers[i + 1] = 0
+
+            for t in time_consumptions:
+                time_buffers[min(time_buffers, key=time_buffers.get)] += int(
+                    t / efficiency_multiplier
+                )
+
+            total_time_consumption = max(time_buffers.values())
+            logging.debug(
+                f"Time buffers for {distribute_num+1} workers: {time_buffers}"
+            )
+            logging.debug(total_time_consumption)
+
+            if total_time_consumption < best_time:
+                best_thread = each_thread_num
+                best_distribute = distribute_num + 1
+                best_time = total_time_consumption
+
+        # logging.info(f"time_consumption_simulation: {time_buffers}")
+
+        logging.info(
+            f"Using {best_thread} threads for {best_distribute} workers in alignment by optimization"
+        )
+
+        # Update thread numbers by simulation
+        alignment_opt = opt_generator(
+            V, opt, path, step="alignment", thread=best_thread
+        )
+        p = mp.Pool(best_distribute)
+
+        alignment_result = p.starmap(module_alignment, alignment_opt)
+        p.close()
+        p.join()
+
+    else:
+        # non-multithreading mode for debugging
+        alignment_result = []
+        for option in alignment_opt:
+            alignment_result.append(module_alignment(*option))
+
+    return V, path, opt