bscampp 1.0.1a0__py3-none-any.whl

bscampp/init_configs.py

@@ -0,0 +1,93 @@
+import os, sys, shutil
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+from argparse import ArgumentParser, Namespace
+from platform import platform
+
+def find_main_config(homepath):
+    with open(homepath, 'r') as f:
+        _root_dir = f.read().strip()
+        main_config_path = os.path.join(_root_dir, 'main.config')
+        if os.path.exists(main_config_path):
+            return _root_dir, main_config_path
+        else:
+            return None, None
+
+'''
+Initialize the config file at ~/.bscampp/main.config
+By default will prioritize existing software from the user environment
+'''
+def init_config_file(homepath, rerun=False, prioritize_user_software=True):
+    if not rerun:
+        # make sure home.path exists
+        if not os.path.exists(homepath):
+            print(f'Cannot find home.path: {homepath}, regenerating...')
+        else:
+            _root_dir, main_config_path = find_main_config(homepath)
+            if _root_dir is not None:
+                return _root_dir, main_config_path
+            else:
+                print(f'Cannot find main.config, regenerating...')
+
+    _root_dir = os.path.expanduser('~/.bscampp')
+    main_config_path = os.path.join(_root_dir, 'main.config')
+    print(f'Initializing the config file at: {main_config_path}')
+
+    # write to local home.path and _root_dir
+    if not os.path.isdir(_root_dir):
+        os.mkdir(_root_dir)
+    with open(homepath, 'w') as f:
+        f.write(_root_dir)
+
+    # create main.config based on the default.config at this file's location
+    _config_path = os.path.join(os.path.dirname(__file__), 'default.config')
+    cparser = configparser.ConfigParser()
+    cparser.optionxform = str
+    assert os.path.exists(_config_path), \
+            'default config file missing! Please redownload from GitHub\n'
+
+    if os.path.exists(main_config_path):
+        print(f'Main configuration file {main_config_path} exists...')
+        print('Overwriting the existing config file...')
+    print('\n')
+
+    with open(_config_path, 'r') as f:
+        cparser.read_file(f)
+
+    # check platform, e.g., macOS or linux, etc.
+    platform_name = platform()
+    print(f'System is: {platform_name}')
+
+    tools_dir = os.path.join(os.path.dirname(__file__), 'tools')
+    set_sections = ['basic']
+
+    # default path to all potential binaries
+    cparser.set('basic', 'pplacer_path',
+            os.path.join(tools_dir, 'pplacer'))
+    cparser.set('basic', 'epang_path',
+            os.path.join(tools_dir, 'epa-ng'))
+    cparser.set('basic', 'hamming_distance_dir',
+            os.path.join(tools_dir, 'hamming_distance'))
+    
+    # macOS TODO: need to recompile the binaries 
+    if 'macos' in platform_name.lower():
+        cparser.set('basic', 'hamming_distance_dir',
+                os.path.join(tools_dir, 'macOS', 'hamming_distance'))
+    
+    # prioritize user's software
+    if prioritize_user_software:
+        print('Detecting existing software from user\'s environment...')
+        softwares = ['pplacer', 'epa-ng', 'taxit']
+        for software in softwares:
+            sname = software.replace('-', '')
+            software_path = shutil.which(software)
+            if software_path:
+                print('\t{}: {}'.format(software, software_path)) 
+                cparser.set('basic', f'{sname}_path', software_path)
+    with open(main_config_path, 'w') as f:
+        cparser.write(f)
+    print(f'\n(Done) main.config was written to: {main_config_path}') 
+    print(f'If you want to make changes, please directly edit {main_config_path}') 
+    return _root_dir, main_config_path

bscampp/jobs.py

@@ -0,0 +1,198 @@
+import os, shutil, subprocess, stat, re, traceback, shlex
+from subprocess import Popen
+from abc import abstractmethod
+
+from bscampp import get_logger, log_exception
+from bscampp.configs import Configs
+
+_LOG = get_logger(__name__)
+
+'''
+Template class Job for running external software/jobs
+'''
+class Job(object):
+    def __init__(self):
+        self.job_type = ""
+        self.errors = []
+        self.b_ignore_error = False
+        self.pid = -1
+        self.returncode = 0
+
+    def __call__(self):
+        return self.run()
+
+    def get_pid(self):
+        return self.pid
+
+    # run the job with given invocation and raise errors when encountered
+    def run(self, stdin="", lock=None, logging=False, shell=False):
+        try:
+            cmd, outpath = self.get_invocation()
+            _LOG.debug(f'Running job_type: {self.job_type}, output: {outpath}')
+
+            # failsafe for NotImplemented jobs
+            if len(cmd) == 0:
+                raise ValueError(
+                        f'{self.job_type} does not have a valid run command. '
+                        'It might be due to (invalid input type, etc.).')
+
+            # identify binaries as the first field
+            binpath = cmd[0]
+            # deal with special cases, e.g., python, java
+            if binpath == 'java':
+                binpath = cmd[2]
+            elif binpath == 'python' or binpath == 'python3':
+                binpath = cmd[1]
+            assert os.path.exists(binpath) or binpath == 'gzip', \
+                    ('executable for %s does not exist: %s' %
+                        (self.job_type, binpath))
+            assert \
+                (binpath.count('/')== 0 or os.path.exists(binpath)), \
+                ('path for %s does not exist (%s)' %
+                    (self.job_type, binpath))
+
+            _LOG.debug('Arguments: %s', ' '.join(
+                (str(x) if x is not None else '?NoneType?' for x in cmd)))
+
+            # logging to local or to PIPE
+            stderr, stdout = '', ''
+            scmd = ' '.join(cmd)
+            if logging:
+                logpath = os.path.join(
+                        os.path.dirname(outpath), 'f{self.job_type}.txt')
+                outlogging = open(logpath, 'w', 1)
+
+                # TODO: may need to deal with piping in the future, for now
+                # it is not needed
+                p = Popen(cmd, text=True, bufsize=1,
+                        stdin=subprocess.PIPE,
+                        stdout=outlogging, stderr=subprocess.PIPE)
+                self.pid = p.pid
+                stdout, stderr = p.communicate(input=stdin)
+                outlogging.close()
+            else:
+                p = Popen(cmd, text=True, bufsize=1,
+                        stdin=subprocess.PIPE,
+                        stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                self.pid = p.pid
+                stdout, stderr = p.communicate(input=stdin)
+            self.returncode = p.returncode
+
+            # successful run and write to log
+            if self.returncode == 0:
+                if lock:
+                    try:
+                        lock.acquire()
+                        _LOG.debug(f'{self.job_type} completed, output: {outpath}')
+                    finally:
+                        lock.release()
+                else:
+                    _LOG.debug(f'{self.job_type} completed, output: {outpath}')
+                return outpath
+            else:
+                error_msg = ' '.join([f'Error occurred running {self.job_type}.',
+                    f'returncode: {self.returncode}'])
+                if lock:
+                    try:
+                        lock.acquire()
+                        _LOG.error(error_msg + '\nSTDOUT: ' + stdout +
+                                '\nSTDERR: ' + stderr)
+                    finally:
+                        lock.release()
+                else:
+                    _LOG.error(error_msg + '\nSTDOUT: ' + stdout +
+                            '\nSTDERR: ' + stderr)
+                exit(1)
+        except Exception:
+            log_exception(_LOG)
+
+    # implemented in subclass
+    # return: (cmd, outpath)
+    @abstractmethod
+    def get_invocation(self):
+        raise NotImplementedError(
+            'get_invocation() should be implemented by subclasses.')
+
+'''
+A EPA-ng job that runs EPA-ng with given parameters
+'''
+class EPAngJob(Job):
+    def __init__(self, **kwargs):
+        Job.__init__(self)
+        self.job_type = 'epa-ng'
+
+        self.path = ''
+        self.info_path = ''
+        self.tree_path = ''
+        self.aln_path = ''
+        self.qaln_path = ''
+        self.outdir = ''
+        self.num_cpus = 1
+
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def get_invocation(self):
+        self.outpath = os.path.join(self.outdir, 'epa_result.jplace') 
+        cmd = [self.path,
+                '-m', self.info_path,
+                '-t', self.tree_path, '-s', self.aln_path,
+                '-q', self.qaln_path, '-w', self.outdir,
+                '-T', str(self.num_cpus), '--redo']
+        return cmd, self.outpath
+
+'''
+A taxtastic job that create a refpkg based on given parameters
+'''
+class TaxtasticJob(Job):
+    def __init__(self, **kwargs):
+        Job.__init__(self)
+        self.job_type = 'taxit'
+
+        self.path = ''
+        self.outdir = ''
+        self.name = ''
+        self.tree_path = ''
+        self.aln_path = ''
+        self.info_path = ''
+
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def get_invocation(self):
+        self.outpath = os.path.join(self.outdir)
+        cmd = [self.path, 'create', '-P', self.outdir,
+                '-l', self.name, '--aln-fasta', self.aln_path,
+                '--tree-file', self.tree_path]
+        # check which model file is provided
+        if 'bestModel' in self.info_path:
+            cmd.extend(['--model-file', self.info_path])
+        else:
+            cmd.extend(['--tree-stats', self.info_path])
+        return cmd, self.outpath
+
+'''
+A pplacer job that uses taxtastic refpkg to place sequences
+'''
+class PplacerTaxtasticJob(Job):
+    def __init__(self, **kwargs):
+        Job.__init__(self)
+        self.job_type = 'pplacer-taxtastic'
+
+        self.path = ''
+        self.refpkg_dir = ''
+        self.qaln_path = ''
+        self.outdir = ''
+        self.outpath = ''
+        self.model = 'GTR'
+        self.num_cpus = 1
+
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+
+    def get_invocation(self):
+        # outpath defined 
+        cmd = [self.path, '-m', self.model,
+                '-c', self.refpkg_dir, '-o', self.outpath,
+                '-j', str(self.num_cpus), self.qaln_path]
+        return cmd, self.outpath

bscampp/pipeline.py

@@ -0,0 +1,224 @@
+import json, time, sys, os, shutil
+from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
+import argparse
+
+from bscampp import get_logger, log_exception, __version__
+from bscampp.configs import *
+from bscampp.functions import *
+import bscampp.utils as utils
+
+from multiprocessing import Manager
+from concurrent.futures import ProcessPoolExecutor
+
+_LOG = get_logger(__name__)
+
+# process pool initializer
+def initial_pool(parser, cmdline_args):
+    # avoid redundant logging for child process
+    buildConfigs(parser, cmdline_args, child_process=True)
+
+# main pipeline for BSCAMPP
+def bscampp_pipeline(*args, **kwargs):
+    t0 = time.perf_counter()
+    m = Manager(); lock = m.Lock()
+
+    # parse command line arguments and build configurations
+    parser, cmdline_args = parseArguments()
+
+    # initialize multiprocessing (if needed)
+    _LOG.warning('Initializing ProcessPoolExecutor...')
+    pool = ProcessPoolExecutor(Configs.num_cpus, initializer=initial_pool,
+            initargs=(parser, cmdline_args,))
+
+    # (0) temporary files wrote to here
+    workdir = os.path.join(Configs.outdir, f'tmp{Configs.tmpfilenbr}')
+    try:
+        if not os.path.isdir(workdir):
+            os.makedirs(workdir)
+    except OSError:
+        log_exception(_LOG)
+
+    # (1) read in tree, alignment, and separate reference sequences from
+    # query sequences
+    tree, leaf_dict, aln_path, aln, qaln_path, qaln = readData(workdir)
+
+    # (2) compute closest leaves for all query sequences
+    query_votes_dict, query_top_vote_dict = getClosestLeaves(
+            aln_path, qaln_path, aln, qaln, workdir)
+
+    # (3) first assign all queries to their closest-leaf subtrees,
+    # then do reassignment to minimize distance between each's top vote
+    # and the subtree's seed leaf
+    new_subtree_dict, placed_query_list = assignQueriesToSubtrees(
+            query_votes_dict, query_top_vote_dict, tree, leaf_dict)
+
+    # (4) perform placement for each subtree
+    output_jplace = placeQueriesToSubtrees(tree, leaf_dict, new_subtree_dict,
+            placed_query_list, aln, qaln, cmdline_args, workdir, pool, lock)
+
+    # (5) write the output jplace to local
+    writeOutputJplace(output_jplace)
+
+    # shutdown pool
+    _LOG.warning('Shutting down ProcessPoolExecutor...')
+    pool.shutdown()
+    _LOG.warning('ProcessPoolExecutor shut down.')
+    
+    # clean up temp files if not keeping
+    if not Configs.keeptemp:
+        _LOG.info('Removing temporary files...')
+        clean_temp_files()
+
+    # stop BSCAMPP 
+    send = time.perf_counter()
+    _LOG.info('BSCAMPP completed in {} seconds...'.format(send - t0))
+
+def clean_temp_files():
+    # all temporary files/directories to remove
+    temp_items = [f'tmp{Configs.tmpfilenbr}']
+    for temp in temp_items:
+        temp_path = os.path.join(Configs.outdir, temp)
+        if os.path.isfile(temp_path):
+            os.remove(temp_path)
+        elif os.path.isdir(temp_path):
+            shutil.rmtree(temp_path)
+        else:
+            continue
+        _LOG.info(f'- Removed {temp}')
+
+def parseArguments():
+    global _root_dir, main_config_path
+    parser = _init_parser()
+    cmdline_args = sys.argv[1:]
+    
+    # build config
+    buildConfigs(parser, cmdline_args)
+    _LOG.info('BSCAMPP is running with: {}'.format(
+        ' '.join(cmdline_args)))
+    getConfigs()
+
+    return parser, cmdline_args
+
+def _init_parser():
+    # example usage
+    example_usages = '''Example usages:
+> default
+    %(prog)s -i raxml.info 
+'''
+
+    parser = ArgumentParser(
+            description=(
+                "This program runs BSCAMPP, a scalable phylogenetic "
+                "placement framework that scales EPA-ng/pplacer "
+                "to very large tree placement."
+                ),
+            conflict_handler='resolve',
+            epilog=example_usages,
+            formatter_class=utils.SmartHelpFormatter,
+            )
+    parser.add_argument('-v', '--version', action='version',
+            version="%(prog)s " + __version__)
+    parser.groups = dict()
+    required = True
+
+    ## add a subcommand for updating configuration file without running
+    ## the BSCAMPP pipeline
+    #subparsers = parser.add_subparsers(dest='command',
+    #        help='Subcommands for BSCAMPP')
+    #update_parser = subparsers.add_parser('update-configs',
+    #        help='Update the configuration file without running BSCAMPP.')
+
+    ## try update args requirement if subcommand(s) are used
+    #if 'update-configs' in sys.argv:
+    #    required = False
+
+    # basic group
+    basic_group = parser.add_argument_group(
+            "Basic parameters".upper(),
+            "These are the basic parameters for BSCAMPP.")
+    parser.groups['basic_group'] = basic_group
+
+    basic_group.add_argument('--placement-method', type=str,
+                  help='The base placement method to use. Default: epa-ng',
+                  choices=['epa-ng', 'pplacer'], default='epa-ng',
+                  required=False)
+    basic_group.add_argument("-i", "--info", "--info-path", type=str,
+                  dest="info_path",
+                  help=("Path to model parameters. E.g., .bestModel "
+                  "from RAxML/RAxML-ng"),
+                  required=required, default=None)
+    basic_group.add_argument("-t", "--tree", "--tree-path", type=str,
+                  dest="tree_path",
+                  help="Path to reference tree with estimated branch lengths",
+                  required=required, default=None)
+    basic_group.add_argument("-a", "--alignment", "--aln-path", type=str,
+                  dest="aln_path",
+                  help=("Path for reference sequence alignment in "
+                  "FASTA format. Optionally with query sequences. "
+                  "Query alignment can be specified with --qaln-path"), 
+                  required=required, default=None)
+    basic_group.add_argument("-q", "--qalignment", "--qaln-path", type=str,
+                  dest="qaln_path",
+                  help=("Optionally provide path to query sequence alignment "
+                  "in FASTA format. Default: None"),
+                  required=False, default=None)
+    basic_group.add_argument("-d", "--outdir", type=str,
+                  help="Directory path for output. Default: bscampp_output/",
+                  required=False, default="bscampp_output")
+    basic_group.add_argument("-o", "--output", type=str, dest="outname",
+                  help="Output file name. Default: bscampp_result.jplace",
+                  required=False, default="bscampp_result.jplace")
+    basic_group.add_argument("--threads", "--num-cpus", type=int,
+                  dest="num_cpus",
+                  help="Number of cores for parallelization, default: -1 (all)",
+                  required=False, default=-1)
+
+    # advanced parameter settings
+    advance_group = parser.add_argument_group(
+            "Advance parameters".upper(),
+            ("These parameters control how BSCAMPP is run. "
+             "The default values are set based on experiments."
+             ))
+    parser.groups['advance_group'] = advance_group
+
+    advance_group.add_argument("-m", "--model", type=str,
+                  help="Model used for edge distances. Default: GTR",
+                  required=False, default="GTR")
+    advance_group.add_argument("-b", "--subtreesize", type=int,
+                  help="Integer size of the subtree. Default: 2000",
+                  required=False, default=2000)
+    advance_group.add_argument("-V", "--votes", type=int,
+                  help="Number of votes per query sequence. Default: 5",
+                  required=False, default=5)
+    advance_group.add_argument("--similarityflag", type=str2bool,
+                  help="Boolean, True if maximizing sequence similarity "
+                  "instead of simple Hamming distance (ignoring gap "
+                  "sites in the query). Default: True",
+                  required=False, default=True)
+    
+    # miscellaneous group
+    misc_group = parser.add_argument_group(
+            "Miscellaneous parameters".upper(),)
+    parser.groups['misc_group'] = misc_group
+
+    misc_group.add_argument("-n","--tmpfilenbr", type=int,
+                  help="Temporary file indexing. Default: 0",
+                  required=False, default=0)
+    misc_group.add_argument("--fragmentflag", type=str2bool,
+                  help="If queries contains fragments. Default: True",
+                  required=False, default=True)
+    misc_group.add_argument("--keeptemp", type=str2bool,
+                  help="Boolean, True to keep all temporary files. "
+                  "Default: False",
+                  required=False, default=False)
+    return parser
+
+def str2bool(b):
+    if isinstance(b, bool):
+       return b
+    if b.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    elif b.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    else:
+        raise argparse.ArgumentTypeError('Boolean value expected.')        

bscampp/tools/hamming_distance/CMakeLists.txt

@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.6)
+
+project(hamming CXX)
+
+find_package(OpenMP)
+
+add_executable (hamming src/new_hamming.cpp)
+add_executable (fragment_hamming src/fragment_hamming.cpp)
+add_executable (homology src/homology.cpp)
+
+target_link_libraries(fragment_hamming OpenMP::OpenMP_CXX)
+target_link_libraries(hamming OpenMP::OpenMP_CXX)
+target_link_libraries(homology OpenMP::OpenMP_CXX)