drep 3.4.5__tar.gz → 3.6.1__tar.gz

{drep-3.4.5 → drep-3.6.1}/PKG-INFO

@@ -1,8 +1,16 @@
 Metadata-Version: 2.1
 Name: drep
-Version: 3.4.5
+Version: 3.6.1
 Summary: De-replication of microbial genomes assembled from multiple samples
 Home-page: https://github.com/MrOlm/drep
 Author: Matt Olm
 Author-email: mattolm@berkeley.edu
 License: MIT
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: seaborn
+Requires-Dist: matplotlib
+Requires-Dist: biopython
+Requires-Dist: scikit-learn
+Requires-Dist: tqdm
+Requires-Dist: pytest

{drep-3.4.5 → drep-3.6.1}/README.md

@@ -9,7 +9,7 @@ Manual, installation instructions, and API are at available at
 [ReadTheDocs](https://drep.readthedocs.io/en/latest/)
 
 Publication is available at
-[ISMEJ](http://www.nature.com/ismej/journal/vaop/ncurrent/full/ismej2017126a.html)
+[ISMEJ](https://doi.org/10.1038/ismej.2017.126)
 
 Open source pre-print publication is available at
 [bioRxiv](https://doi.org/10.1101/108142)

drep-3.6.1/drep/VERSION

@@ -0,0 +1 @@
+3.6.1

{drep-3.4.5 → drep-3.6.1}/drep/__init__.py

@@ -2,7 +2,7 @@
 
 from subprocess import call
 import os
-from Bio import SeqIO
+# from Bio import SeqIO
 import shutil
 import multiprocessing
 import multiprocessing.dummy

{drep-3.4.5 → drep-3.6.1}/drep/argumentParser.py

@@ -44,7 +44,7 @@ def printHelp():
     print('                ...::: dRep v' + VERSION + ' :::...''')
     print('''\
 
-  Matt Olm. MIT License. Banfield Lab, UC Berkeley. 2017 (last updated 2023)
+  Matt Olm. MIT License. Banfield Lab, UC Berkeley. 2017 (last updated 2025)
 
   See https://drep.readthedocs.io/en/latest/index.html for documentation
   Choose one of the operations below for more detailed help. 
@@ -115,16 +115,20 @@ def parse_args(args):
     Clustflags = cluster_parent.add_argument_group('GENOME COMPARISON OPTIONS')
     Clustflags.add_argument("--S_algorithm", help="R|Algorithm for secondary clustering comaprisons:\n" \
                                                   + "fastANI = Kmer-based approach; very fast\n" \
+                                                  + "skani = Even faster Kmer-based approacht\n" \
                                                   + "ANImf   = (DEFAULT) Align whole genomes with nucmer; filter alignment; compare aligned regions\n" \
                                                   + "ANIn    = Align whole genomes with nucmer; compare aligned regions\n" \
                                                   + "gANI    = Identify and align ORFs; compare aligned ORFS\n" \
                                                   + "goANI   = Open source version of gANI; requires nsmimscan\n",
-                            default='fastANI', choices={'ANIn', 'gANI', 'ANImf', 'goANI', 'fastANI'})
+                            default='fastANI', choices={'ANIn', 'gANI', 'ANImf', 'goANI', 'fastANI', 'skani'})
     Clustflags.add_argument("-ms", "--MASH_sketch", help="MASH sketch size", default=1000)
     Clustflags.add_argument("--SkipMash", help="Skip MASH clustering,\
                             just do secondary clustering on all genomes", action='store_true')
     Clustflags.add_argument("--SkipSecondary", help="Skip secondary clustering, just perform MASH\
                             clustering", action='store_true')
+    Clustflags.add_argument("--skani_extra",
+                         help="Extra arguments to pass to skani triangle",
+                         default="", type=str)
     Clustflags.add_argument("--n_PRESET", help="R|Presets to pass to nucmer\n" \
                                                + "tight   = only align highly conserved regions\n" \
                                                + "normal  = default ANIn parameters", choices=['normal', 'tight'],
@@ -145,6 +149,8 @@ def parse_args(args):
     Compflags.add_argument("--clusterAlg", help="Algorithm used to cluster genomes (passed\
                         to scipy.cluster.hierarchy.linkage", default='average',
                            choices={'single', 'complete', 'average', 'weighted', 'centroid', 'median', 'ward'})
+    Compflags.add_argument("--low_ram_primary_clustering", help="Use a memory-efficient algorithm for primary clustering. This only affects primary clustering and not secondary clustering.", 
+                          action='store_true', default=False)
 
     GRflags = cluster_parent.add_argument_group('GREEDY CLUSTERING OPTIONS\n'
                                                 'These decrease RAM use and runtime at the expense of a minor loss in '

{drep-3.4.5 → drep-3.6.1}/drep/d_analyze.py

@@ -728,11 +728,11 @@ def plot_winner_scoring_complex(Wdb, Sdb, Cdb, Gdb, plot_dir= False, **kwargs):
         for variable in x['variable'].unique():
             vals += [v for v in x['value'][x['variable'] == variable].tolist()]
 
-        # Add un-normalized values to barplots
-        i = 0
-        for p in g.patches:
-            g.annotate("{0:.1f}".format(vals[i]), (p.get_width(), p.get_y()+(p.get_height()/1.1) ), fontsize=8)
-            i += 1
+        # # Add un-normalized values to barplots
+        # i = 0
+        # for p in g.patches:
+        #     g.annotate("{0:.1f}".format(vals[i]), (p.get_width(), p.get_y()+(p.get_height()/1.1) ), fontsize=8)
+        #     i += 1
 
         plt.title('Scoring of cluster {0}'.format(cluster))
         plt.xlabel('Normalized Score')
@@ -1378,11 +1378,11 @@ def _make_scoring_plot(db, bars,**kwargs):
     for variable in x['variable'].unique():
         vals += [v for v in x['value'][x['variable'] == variable].tolist()]
 
-    # Add un-normalized values to barplots
-    i = 0
-    for p in g.patches:
-        g.annotate("{0:.1f}".format(vals[i]), (p.get_width(), p.get_y()+(p.get_height()/1.1) ), fontsize=8)
-        i += 1
+    # # Add un-normalized values to barplots
+    # i = 0
+    # for p in g.patches:
+    #     g.annotate("{0:.1f}".format(vals[i]), (p.get_width(), p.get_y()+(p.get_height()/1.1) ), fontsize=8)
+    #     i += 1
 
     # Add taxonomy if available
     axes = plt.gca()

{drep-3.4.5 → drep-3.6.1}/drep/d_bonus.py

@@ -98,7 +98,7 @@ def check_dependencies(print_out=False):
     For all possible dependencies, see if you can find them
     '''
     for dep in ['mash', 'nucmer', 'checkm', 'ANIcalculator', 'prodigal', 'centrifuge',
-                'nsimscan', 'fastANI']:
+                'nsimscan', 'fastANI', 'skani']:
         loc, works = find_program(dep)
         works_message = {True:'all good', False:'!!! ERROR !!!'}[works]
         message = '{0:.<40} {1:15} (location = {2})'.format(dep, works_message, loc)

{drep-3.4.5 → drep-3.6.1}/drep/d_cluster/cluster_utils.py

@@ -6,6 +6,7 @@ import numpy as np
 import pandas as pd
 import scipy.cluster
 from scipy.spatial import distance as ssd
+import networkx as nx
 
 import drep.d_cluster.utils
 
@@ -87,7 +88,33 @@ def iteratre_clusters(Bdb, Cdb, id='primary_cluster'):
         yield d, cluster
 
 
-def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10):
+def cluster_threshold_graph_optimized(db, linkage_cutoff=0.10, linkage_method='single'):
+    # Filter distances below threshold first
+    filtered_edges = db[db['dist'] <= linkage_cutoff]
+    
+    # Create graph directly from filtered edges
+    G = nx.Graph()
+    G.add_edges_from(filtered_edges[['genome1', 'genome2']].values)
+    
+    # Log that we're using the optimized method
+    logging.debug("Using low-RAM optimized clustering method with {0} edges".format(len(filtered_edges)))
+    
+    # Find connected components
+    clusters = {}
+    for cluster_id, component in enumerate(nx.connected_components(G)):
+        for genome in component:
+            clusters[genome] = cluster_id + 1
+            
+    # Add isolated nodes (if needed)
+    all_genomes = set(db['genome1']).union(set(db['genome2']))
+    for genome in all_genomes:
+        if genome not in clusters:
+            clusters[genome] = len(clusters)
+            
+    # Return clusters and a special value indicating we used the optimized method
+    return clusters, "optimized_method_used"
+
+def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10, low_ram=False):
     '''
     Perform hierarchical clustering on a symmetrical distiance matrix
 
@@ -95,6 +122,7 @@ def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10):
         db: result of db.pivot usually
         linkage_method: passed to scipy.cluster.hierarchy.fcluster
         linkage_cutoff: distance to draw the clustering line (default = .1)
+        low_ram: whether to use the memory-efficient algorithm
 
     Returns:
         list: [Cdb, linkage]
@@ -102,21 +130,32 @@ def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10):
     # Save names
     names = list(db.columns)
 
-    # Generate linkage dataframe
-    arr =  np.asarray(db)
-    try:
-        arr = ssd.squareform(arr)
-    except:
-        logging.error("The database passed in is not symmetrical!")
-        logging.error(arr)
-        logging.error(names)
-        sys.exit()
-    linkage = scipy.cluster.hierarchy.linkage(arr, method= linkage_method)
-
-    # Form clusters
-    fclust = scipy.cluster.hierarchy.fcluster(linkage,linkage_cutoff, \
-                    criterion='distance')
-    # Make Cdb
-    Cdb = drep.d_cluster.utils._gen_cdb_from_fclust(fclust,names)
-
-    return Cdb, linkage
+    if low_ram:
+        # Convert to long format for optimized clustering
+        db_long = db.stack().reset_index()
+        db_long.columns = ['genome1', 'genome2', 'dist']
+        clusters, _ = cluster_threshold_graph_optimized(db_long, linkage_cutoff, linkage_method)
+        
+        # Convert clusters to Cdb format
+        Cdb = pd.DataFrame({'genome': list(clusters.keys()), 
+                          'cluster': list(clusters.values())})
+        return Cdb, _
+    else:
+        # Generate linkage dataframe
+        arr =  np.asarray(db)
+        try:
+            arr = ssd.squareform(arr)
+        except:
+            logging.error("The database passed in is not symmetrical!")
+            logging.error(arr)
+            logging.error(names)
+            sys.exit()
+        linkage = scipy.cluster.hierarchy.linkage(arr, method= linkage_method)
+
+        # Form clusters
+        fclust = scipy.cluster.hierarchy.fcluster(linkage,linkage_cutoff, \
+                        criterion='distance')
+        # Make Cdb
+        Cdb = drep.d_cluster.utils._gen_cdb_from_fclust(fclust,names)
+
+        return Cdb, linkage

{drep-3.4.5 → drep-3.6.1}/drep/d_cluster/compare_utils.py

@@ -264,6 +264,7 @@ def cluster_mash_database(db, **kwargs):
     Keyword arguments:
         clusterAlg: how to cluster database (default = single)
         P_ani: threshold to cluster at (default = 0.9)
+        low_ram_primary_clustering: whether to use memory-efficient algorithm
 
     Returns:
         list: [Cdb, [linkage, linkage_db, arguments]]
@@ -273,12 +274,13 @@ def cluster_mash_database(db, **kwargs):
     # Load key words
     P_Lmethod = kwargs.get('clusterAlg','single')
     P_Lcutoff = 1 - kwargs.get('P_ani',.9)
+    low_ram = kwargs.get('low_ram_primary_clustering', False)
 
     # Do the actual clustering
     db['dist'] = 1 - db['similarity']
     linkage_db = db.pivot(index="genome1", columns="genome2", values="dist")
     Cdb, linkage = drep.d_cluster.cluster_utils.cluster_hierarchical(linkage_db, linkage_method= P_Lmethod, \
-                                                                     linkage_cutoff= P_Lcutoff)
+                                                                     linkage_cutoff= P_Lcutoff, low_ram=low_ram)
     Cdb = Cdb.rename(columns={'cluster':'primary_cluster'})
     Cdb['primary_cluster'] = Cdb['primary_cluster'].astype(int)
 
@@ -290,6 +292,7 @@ def cluster_mash_database(db, **kwargs):
     return Cdb, cluster_ret
 
 def secondary_clustering(Bdb, Cdb, algorithm, data_folder, **kwargs):
+
     if kwargs.get('greedy_secondary_clustering', False) != True:
         Ndb = pd.DataFrame()
         for bdb, name in iteratre_clusters(Bdb, Cdb, id='primary_cluster'):
@@ -368,6 +371,12 @@ def compare_genomes(bdb, algorithm, data_folder, **kwargs):
             df = drep.d_cluster.external.run_pairwise_fastANI(genome_list, working_data_folder, **kwargs)
             return df
 
+        elif algorithm == 'skani':
+            genome_list = bdb['location'].tolist()
+            working_data_folder = os.path.join(data_folder, 'skani_files/')
+            df = drep.d_cluster.external.run_pairwise_skani(genome_list, working_data_folder, **kwargs)
+            return df
+
         elif algorithm == 'gANI':
             # Figure out prodigal folder
             wd = kwargs.get('wd', False)

{drep-3.4.5 → drep-3.6.1}/drep/d_cluster/external.py

@@ -115,6 +115,88 @@ def run_pairwise_fastANI(genome_list, outdir, **kwargs):
         logging.error("CRITICAL ERROR WITH SECONDARY CLUSTERING CODE {0}; SKIPPING".format(code))
         return pd.DataFrame()
 
+def run_pairwise_skani(genome_list, outdir, **kwargs):
+    code = drep.d_cluster.utils._randomString(stringLength=10)
+    extra_cmd = kwargs.get('skani_extra', "")
+
+    # Make folders
+    if not os.path.exists(outdir):
+        os.makedirs(outdir)
+    tmp_dir = os.path.join(outdir, 'tmp/')
+    if not os.path.exists(tmp_dir):
+        os.makedirs(tmp_dir)
+
+    # Make genome list
+    glist = os.path.join(tmp_dir, 'genomeList')
+    glist = _make_glist(genome_list, glist)
+
+    # Gen command
+    exe_loc = drep.get_exe('skani')
+    out_base = os.path.join(outdir, 'skani_out_{0}'.format(code))
+    #cmd = [exe_loc, '--ql', glist, '--rl', glist, '-o', out_base, '--matrix', '-t', str(p), "--minFraction", str(0)]
+    cmd = [exe_loc, "triangle", "-t", "1", '-o', out_base, '--full-matrix', '-l', glist, '--detailed', '-s', "1", '--min-af', "0"]
+    if extra_cmd != "":
+        cmd += extra_cmd.split(' ')
+
+    logging.debug(' '.join(cmd) + ' ' + code)
+
+    # Run command
+    if ('wd' in kwargs) and (kwargs.get('debug', False)):
+        logdir = kwargs.get('wd').get_dir('cmd_logs')
+    else:
+        logdir = False
+    drep.thread_cmds([cmd], shell=False, logdir=logdir, t=1)
+
+    # Load results
+    fdb = load_skani(out_base)
+
+    return fdb
+
+def load_triangle_matrix(file_path):
+    with open(file_path, 'r') as f:
+        lines = f.readlines()
+
+    # Get the number of genomes
+    num_genomes = int(lines[0].strip())
+
+    # Initialize lists to store data
+    reference = []
+    query = []
+    ani = []
+
+    # Parse the matrix
+    for i in range(1, num_genomes + 1):
+        row = lines[i].strip().split('\t')
+        ref_genome = row[0]
+        for j in range(1, i + 1):
+            query_genome = lines[j].strip().split('\t')[0]
+            ani_value = float(row[j])
+            reference.append(ref_genome)
+            query.append(query_genome)
+            ani.append(ani_value)
+
+    # Create a DataFrame
+    df = pd.DataFrame({'reference': reference, 'querry': query, 'ani': ani})
+    return df
+
+def load_matrix_to_dataframe(file_path):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    num_genomes = int(lines[0].strip())
+    genomes = [line.split('\t')[0] for line in lines[1:num_genomes + 1]]
+
+    data = []
+    for i in range(num_genomes):
+        for j in range(num_genomes):
+            genome1 = genomes[i]
+            genome2 = genomes[j]
+            ani = float(lines[i + 1].split('\t')[j + 1])
+            data.append([genome1, genome2, ani])
+
+    df = pd.DataFrame(data, columns=['reference', 'querry', 'ani'])
+    return df
+
 def fastani_one_vs_many(one, many, genome_rep_file, outdir, **kwargs):
     p = kwargs.get('processors', 6)
     code = drep.d_cluster.utils._randomString(stringLength=10)
@@ -147,6 +229,28 @@ def load_fastani(file):
 
     return fdb
 
+def load_skani(file):
+    # Load the ani triangle
+    adb = load_triangle_matrix(file)
+    adb['ani'] = adb['ani'] / 100
+
+    # Make symnetrical
+    db = adb.rename(columns={'reference':'querry', 'querry':'reference'})
+    db = db[db['reference'] != db['querry']]
+    adb = pd.concat([adb, db], ignore_index=True).reset_index(drop=True)
+
+    # Load the af triangle
+    tdb = load_matrix_to_dataframe(file + '.af').rename(columns={'ani':'alignment_coverage'})
+
+    # Merge
+    assert len(adb) == len(tdb)
+    fdb = pd.merge(adb, tdb, how='inner')
+
+    # parse
+    for c in ['reference', 'querry']:
+        fdb[c] = [drep.d_cluster.utils._get_genome_name_from_fasta(x) for x in fdb[c]]
+
+    return fdb
 
 def _fix_fastani(odb):
     # Add back missing genomes

{drep-3.4.5 → drep-3.6.1}/drep/d_cluster/utils.py

@@ -103,6 +103,8 @@ def estimate_time(comps, alg):
         time = comps * .5
     elif alg == 'fastANI':
         time = comps * 0.00667
+    elif alg == 'skani':
+        time = comps * 0.00667
     return time
 
 
@@ -352,7 +354,7 @@ def gen_goANI_cmd(file, g1, g2, exe):
         return []
 
     else:
-        cmd = [exe,'--om','TABX', g1, g2, file]
+        cmd = [exe,'--om','TABX', '--rpq', '1', g1, g2, file]
     return cmd
 
 def process_deltafiles(deltafiles, org_lengths, logger=None, **kwargs):

{drep-3.4.5 → drep-3.6.1}/drep/d_filter.py

@@ -11,9 +11,6 @@ import gzip
 import pandas as pd
 import os
 import sys
-import shutil
-import multiprocessing
-from Bio import SeqIO
 
 import drep
 import drep.WorkDirectory
@@ -449,32 +446,43 @@ def calc_n50(loc):
     Calculate the N50 of a .fasta file
 
     Args:
-        fasta_loc: location of .fasta file.
+        loc: location of .fasta file.
 
     Returns:
         int: N50 of .fasta file.
     '''
     lengths = []
+    is_header = True
 
-
-    if loc[-3:] == '.gz':
-        with gzip.open(loc, "rt") as handle:
-            for seq_record in SeqIO.parse(handle, "fasta"):
-                lengths.append(len(seq_record))
+    if loc.endswith('.gz'):
+        open_func = gzip.open
+        mode = 'rt'
     else:
-        for seq_record in SeqIO.parse(loc, "fasta"):
-            lengths.append(len(seq_record))
-
-    # for seq_record in SeqIO.parse(loc, "fasta"):
-    #     lengths.append(len(seq_record))
-
-    half = sum(lengths)/2
+        open_func = open
+        mode = 'r'
+
+    with open_func(loc, mode) as handle:
+        seq_length = 0
+        for line in handle:
+            if line.startswith('>'):
+                if not is_header:
+                    lengths.append(seq_length)
+                    seq_length = 0
+                is_header = True
+            else:
+                seq_length += len(line.strip())
+                is_header = False
+        if not is_header:
+            lengths.append(seq_length)
+
+    half = sum(lengths) / 2
     tally = 0
     for l in sorted(lengths, reverse=True):
         tally += l
         if tally > half:
             return l
 
+
 def run_prodigal(genome_list, out_dir, **kwargs):
     '''
     Run prodigal on a set of genomes, store the output in the out_dir
@@ -647,23 +655,29 @@ def _iterate_checkm_groups(genome_folder_whole, checkm_outf_whole, checkm_group_
 
 def calc_fasta_length(fasta_loc):
     '''
-    Calculate the length of the .fasta file and retun length
+    Calculate the length of the .fasta file and return length
 
     Args:
         fasta_loc: location of .fasta file
 
     Returns:
-        int: total length of all .fasta files
+        int: total length of all sequences in the .fasta file
     '''
     total = 0
 
-    if fasta_loc[-3:] == '.gz':
-        with gzip.open(fasta_loc, "rt") as handle:
-            for seq_record in SeqIO.parse(handle, "fasta"):
-                total += len(seq_record)
+    if fasta_loc.endswith('.gz'):
+        open_func = gzip.open
+        mode = 'rt'
     else:
-        for seq_record in SeqIO.parse(fasta_loc, "fasta"):
-            total += len(seq_record)
+        open_func = open
+        mode = 'r'
+
+    with open_func(fasta_loc, mode) as handle:
+        for line in handle:
+            if line.startswith('>'):
+                pass
+            else:
+                total += len(line.strip())
 
     return total
 

{drep-3.4.5 → drep-3.6.1}/drep.egg-info/PKG-INFO

@@ -1,8 +1,16 @@
 Metadata-Version: 2.1
 Name: drep
-Version: 3.4.5
+Version: 3.6.1
 Summary: De-replication of microbial genomes assembled from multiple samples
 Home-page: https://github.com/MrOlm/drep
 Author: Matt Olm
 Author-email: mattolm@berkeley.edu
 License: MIT
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: seaborn
+Requires-Dist: matplotlib
+Requires-Dist: biopython
+Requires-Dist: scikit-learn
+Requires-Dist: tqdm
+Requires-Dist: pytest

{drep-3.4.5 → drep-3.6.1}/drep.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 README.md
+pyproject.toml
 setup.py
 bin/dRep
 drep/VERSION
@@ -28,4 +29,5 @@ drep/d_cluster/greedy_clustering.py
 drep/d_cluster/parsers.py
 drep/d_cluster/utils.py
 helper_scripts/ScaffoldLevel_dRep.py
-helper_scripts/parse_stb.py
+helper_scripts/parse_stb.py
+tests/test_suite.py

{drep-3.4.5 → drep-3.6.1}/helper_scripts/ScaffoldLevel_dRep.py

@@ -8,7 +8,6 @@
 
 
 import os
-import sys
 import gzip
 import glob
 import textwrap
@@ -18,15 +17,13 @@ import argparse
 import pandas as pd
 
 from shutil import copyfile
-import subprocess
 from subprocess import call
 from collections import defaultdict
 
 from tqdm import tqdm
 
-from Bio import SeqIO
-
 import drep
+import drep.d_filter
 
 def version():
     versionFile = open(os.path.join(drep.__path__[0], 'VERSION'))
@@ -128,8 +125,8 @@ class ANImCommand(abstractCommand):
 
         if method == 'simple':
             # Get the organism lengths
-            qlen = fasta_length(self.config['querry'])
-            rlen = fasta_length(self.config['reference'])
+            qlen = drep.d_filter.calc_fasta_length(self.config['querry'])
+            rlen = drep.d_filter.calc_fasta_length(self.config['reference'])
 
             # Parse the reults
             delta = get_delta_loc(self.config['prefix'])
@@ -156,7 +153,6 @@ class ANImCommand(abstractCommand):
 
     def __str__(self):
         ''' Show the command parameters '''
-        return pprint.pformat(self.config)
 
 def gen_mummer_cmd(**kwargs):
     '''
@@ -660,16 +656,27 @@ def load_scaffold2length(fasta):
     Return a database of scaffold to length
     '''
     table = defaultdict(list)
+    is_header = True
 
-    if fasta[-3:] == '.gz':
-        with gzip.open(fasta, "rt") as handle:
-            for seq_record in SeqIO.parse(handle, "fasta"):
-                table['scaffold'].append(seq_record.id)
-                table['length'].append(len(seq_record))
+    if fasta.endswith('.gz'):
+        open_func = gzip.open
+        mode = 'rt'
     else:
-        for seq_record in SeqIO.parse(str(fasta), "fasta"):
-            table['scaffold'].append(seq_record.id)
-            table['length'].append(len(seq_record))
+        open_func = open
+        mode = 'r'
+
+    with open_func(fasta, mode) as handle:
+        for line in handle:
+            if line.startswith('>'):
+                is_header = True
+                scaffold_id = line[1:].strip().split(' ')[0]
+            elif is_header:
+                table['scaffold'].append(scaffold_id)
+                table['length'].append(0)
+                is_header = False
+            else:
+                table['length'][-1] += len(line.strip())
+
     return pd.DataFrame(table)
 
 def add_cov_ani(Ddb, s2l):
@@ -712,28 +719,46 @@ def split_into_chunks(file, max_len, temp_folder):
     chunks = []
 
     chunk_number = 0
-    records = []
     chunk_len = 0
-    for r in SeqIO.parse(file, "fasta"):
-        records.append(r)
-        chunk_len += len(r)
+    chunk_records = []
+    record_id = ''
+    record_seq = ''
+
+    with open(file, 'r') as handle:
+        for line in handle:
+            if line.startswith('>'):
+                if chunk_len + len(record_seq) > max_len and len(chunk_records) > 0:
+                    # Save this chunk
+                    new_loc = newfile_base + '_CHUNK_{0}.fasta'.format(chunk_number)
+                    with open(new_loc, 'w') as chunk_file:
+                        chunk_file.write(''.join(chunk_records))
+                    chunks.append(new_loc)
+
+                    chunk_len = 0
+                    chunk_number += 1
+                    chunk_records = []
+
+                if record_id:
+                    chunk_records.append('>' + record_id + '\n' + record_seq + '\n')
+                    chunk_len += len(record_seq)
+
+                record_id = line[1:].strip().split(' ')[0]
+                record_seq = ''
+            else:
+                record_seq += line.strip()
+
+        # Add the last record
+        if record_id:
+            chunk_records.append('>' + record_id + '\n' + record_seq + '\n')
+            chunk_len += len(record_seq)
 
-        if chunk_len > max_len:
-            # Save this chunk
+        # Save the final chunk
+        if len(chunk_records) > 0:
             new_loc = newfile_base + '_CHUNK_{0}.fasta'.format(chunk_number)
-            SeqIO.write(records, new_loc, "fasta")
+            with open(new_loc, 'w') as chunk_file:
+                chunk_file.write(''.join(chunk_records))
             chunks.append(new_loc)
 
-            chunk_len = 0
-            chunk_number += 1
-            records = []
-
-    # Save the final chunk
-    if len(records) > 0:
-        new_loc = newfile_base + '_CHUNK_{0}.fasta'.format(chunk_number)
-        SeqIO.write(records, new_loc, "fasta")
-        chunks.append(new_loc)
-
     return chunks
 
 def compare_wrapper(current, new, scaffDb, temp_folder, **kwargs):
@@ -875,23 +900,38 @@ def reconsile(Sdb, current, new, newFile, temp_folder, **kwargs):
 
     return Rdb
 
-
 def write_new(ori_loc, new_loc, to_rm, verbose=False):
     '''
     Write a new .fasta file removing certain scaffolds
     '''
-    records = []
-    if ori_loc[-3:] == '.gz':
-        with gzip.open(ori_loc, "rt") as handle:
-            for r in SeqIO.parse(handle, "fasta"):
-                records.append(r)
+    if ori_loc.endswith('.gz'):
+        open_func = gzip.open
+        mode = 'rt'
     else:
-        for r in SeqIO.parse(ori_loc, "fasta"):
-            records.append(r)
+        open_func = open
+        mode = 'r'
+
+    ori_len = 0
+    count = 0
+    with open_func(ori_loc, mode) as handle, open(new_loc, 'w') as out_handle:
+        record_id = ''
+        record_seq = ''
+        for line in handle:
+            if line.startswith('>'):
+                ori_len += 1
+                if record_id and record_id not in to_rm:
+                    out_handle.write('>' + record_id + '\n' + record_seq + '\n')
+                    count += 1
+                record_id = line[1:].strip().split(' ')[0]
+                record_seq = ''
+            else:
+                record_seq += line.strip()
+
+        # Write the last record
+        if record_id and record_id not in to_rm:
+            out_handle.write('>' + record_id + '\n' + record_seq + '\n')
+            count += 1
 
-    ori_len = len(records)
-    records = (r for r in records if r.id not in to_rm)
-    count = SeqIO.write(records, new_loc, "fasta")
     if verbose:
         print("Removed {0} scaffolds from {1}".format(ori_len - count, ori_loc))
 
@@ -957,9 +997,30 @@ class test_dRepScaffolds():
         to_rm = ['N5_271_010G1_scaffold_119']
         new_loc = self.test_dir + 'new_file.fasta'
 
-        ori_len = len([r for r in SeqIO.parse(self.fastas[0], "fasta")])
+        def count_fasta_records(fasta_loc):
+            count = 0
+            record_flag = False
+            if fasta_loc.endswith('.gz'):
+                open_func = gzip.open
+                mode = 'rt'
+            else:
+                open_func = open
+                mode = 'r'
+
+            with open_func(fasta_loc, mode) as handle:
+                for line in handle:
+                    if line.startswith('>'):
+                        count += 1
+                        record_flag = True
+                    elif not record_flag:
+                        # In case the file starts with sequences without headers
+                        count += 1
+                        record_flag = True
+            return count
+
+        ori_len = count_fasta_records(self.fastas[0])
         write_new(self.fastas[0], new_loc, to_rm, verbose=False)
-        new_len = len([r for r in SeqIO.parse(new_loc, "fasta")])
+        new_len = count_fasta_records(new_loc)
 
         assert new_len + 1 == ori_len
 

{drep-3.4.5 → drep-3.6.1}/helper_scripts/parse_stb.py

@@ -3,50 +3,55 @@
 import os
 import sys
 import gzip
-import logging
 import textwrap
 import argparse
 
-from Bio import SeqIO
-
 def extract_bins(fasta, stb_file, out_base):
     if (out_base != '') & (out_base[-1] != '/'):
         out_base += '_'
 
     # Make scaffold to bin dictionary
     stb = {}
-    stb_reader = open(stb_file)
-    for line in stb_reader:
-        line = line.strip()
-
-        if line.startswith('#') or line.startswith('scaffold_name'):
-            continue
+    with open(stb_file) as stb_reader:
+        for line in stb_reader:
+            line = line.strip()
 
-        scaffold = line.split('\t')[0].strip()
-        bin = line.split('\t')[1].strip()
-        stb[scaffold] = bin
+            if line.startswith('#') or line.startswith('scaffold_name'):
+                continue
 
-    # Do it with append
-    if fasta[-3:] == '.gz':
-        handle = gzip.open(fasta, "rt")
-        seqs = SeqIO.parse(handle, "fasta")
+            scaffold, bin = line.split('\t')[:2]
+            stb[scaffold.strip()] = bin.strip()
 
+    # Parse the FASTA file and write to bins
+    if fasta.endswith('.gz'):
+        open_func = gzip.open
+        mode = 'rt'
     else:
-        seqs = SeqIO.parse(fasta, "fasta")
-
-    for seq_record in seqs:
-        id = str(seq_record.id).strip()
-        if id not in stb:
-            # print("{0} not in stb".format(id))
-            continue
-        fasta = stb[id]
+        open_func = open
+        mode = 'r'
+
+    with open_func(fasta, mode) as handle:
+        record_id = ''
+        record_seq = ''
+        for line in handle:
+            if line.startswith('>'):
+                if record_id and record_id in stb:
+                    bin = stb[record_id]
+                    with open(f"{out_base}{bin}.fa", 'a') as nw:
+                        nw.write(f">{record_id}\n{record_seq}\n")
+                record_id = line[1:].strip().split(' ')[0]
+                record_seq = ''
+            else:
+                record_seq += line.strip()
+
+        # Write the last record
+        if record_id and record_id in stb:
+            bin = stb[record_id]
+            with open(f"{out_base}{bin}.fa", 'a') as nw:
+                nw.write(f">{record_id}\n{record_seq}\n")
 
-        nw = open("{0}{1}.fa".format(out_base, fasta), 'a')
-        nw.write('\n'.join([">{0}".format(id), str(seq_record.seq), '']))
-        nw.close()
-
-    if fasta[-3:] == '.gz':
-        handle.close()
+import os
+import gzip
 
 def gen_stb(fastas):
     if ((len(fastas) == 1) & (not fastas[0].endswith('.gz'))):
@@ -67,18 +72,22 @@ def gen_stb(fastas):
     stb = {}
     for fasta in fastas:
         bin = os.path.basename(fasta)
-        if bin[-3:] == '.gz':
-            with gzip.open(fasta, "rt") as handle:
-                for seq_record in SeqIO.parse(handle, "fasta"):
-                    id = str(seq_record.id).strip()
-                    stb[id] = bin
+        if fasta.endswith('.gz'):
+            open_func = gzip.open
+            mode = 'rt'
         else:
-            iter = SeqIO.parse(fasta, "fasta")
-            for seq_record in iter:
-                id = str(seq_record.id).strip()
-                stb[id] = bin
+            open_func = open
+            mode = 'r'
+
+        with open_func(fasta, mode) as handle:
+            for line in handle:
+                if line.startswith('>'):
+                    id = line[1:].strip().split(' ')[0]
+                    stb[id] = bin
+
     return stb
 
+
 def print_stb(stb, output):
     file = open(output,'w')
     for key in sorted(stb, key=stb.get):

drep-3.6.1/pyproject.toml

@@ -0,0 +1,3 @@
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v"

drep-3.6.1/tests/test_suite.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+###############################################################################
+#
+# test_suite.py - process several E. coli genomes to verify operation of dRep
+#
+###############################################################################
+
+import tests.test_rerun
+import tests.test_unit
+import tests.test_dereplicate
+import tests.test_filter
+import tests.test_cluster
+import tests.test_choose
+import tests.test_analyze
+import tests.test_bonus
+import tests.test_taxonomy
+
+if __name__ == '__main__':
+    # tests.test_rerun.test_rerun().run()
+    # tests.test_unit.test_unit().run()
+    tests.test_dereplicate.test_dereplicate().run()
+    # tests.test_filter.test_filter().run()
+    # tests.test_cluster.test_cluster().run()
+    # tests.test_choose.test_choose().run()
+    # tests.test_analyze.test_analyze().run()
+    # tests.test_bonus.test_bonus().run()
+    # tests.test_taxonomy.test_taxonomy().run()
+
+    print("Everything seems to be working swimmingly!")

drep-3.4.5/drep/VERSION

@@ -1 +0,0 @@
-3.4.5