drep 3.7.1__tar.gz → 4.0.0__tar.gz

{drep-3.7.1 → drep-4.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: drep
-Version: 3.7.1
+Version: 4.0.0
 Summary: De-replication of microbial genomes assembled from multiple samples
 Home-page: https://github.com/MrOlm/drep
 Author: Matt Olm
@@ -13,12 +13,5 @@ Requires-Dist: matplotlib
 Requires-Dist: biopython
 Requires-Dist: scikit-learn
 Requires-Dist: tqdm
-Requires-Dist: networkx
 Requires-Dist: setuptools
 Requires-Dist: pytest
-Dynamic: author
-Dynamic: author-email
-Dynamic: home-page
-Dynamic: license
-Dynamic: requires-dist
-Dynamic: summary

drep-4.0.0/README.md

@@ -0,0 +1,64 @@
+# dRep
+
+[![Downloads](https://pepy.tech/badge/drep)](https://pepy.tech/project/drep)
+[![Downloads](https://pepy.tech/badge/drep/week)](https://pepy.tech/project/drep)
+
+dRep is a python program for rapidly comparing large numbers of genomes. dRep can also "de-replicate" a genome set by identifying groups of highly similar genomes and choosing the best representative genome for each genome set.
+
+Manual, installation instructions, and API are at available at
+[ReadTheDocs](https://drep.readthedocs.io/en/latest/)
+
+Publication is available at
+[ISMEJ](https://doi.org/10.1038/ismej.2017.126)
+
+Open source pre-print publication is available at
+[bioRxiv](https://doi.org/10.1101/108142)
+
+## ⚡ New in v4
+
+dRep v4 uses [skani](https://github.com/bluenote-1577/skani) for **both** primary and secondary genome comparisons by default, replacing v3's default of MASH (primary) + fastANI (secondary). skani is much faster than that pair, and it *streams* its comparisons instead of building an all-vs-all matrix in memory — so `dereplicate` runs far quicker and its memory footprint grows roughly linearly with genome count rather than quadratically.
+
+**Whole-pipeline `dRep dereplicate` on 10,000 genomes** — identical inputs and settings:
+
+| | v3 defaults (MASH → fastANI) | v4 defaults (skani) |
+|---|---|---|
+| Wall-clock time | 6 h 15 min | **14.5 min** (~26× faster) |
+| Peak memory (RSS) | ~13 GB | ~7 GB |
+
+*Benchmarked on an Apple M1 Pro with `-p 10`; the memory advantage widens further at larger genome counts.*
+
+## Installation with pip
+```
+$ pip install drep
+```
+
+## Quick start
+
+### Genome comparison:
+```
+$ dRep compare output_directory -g path/to/genomes/*.fasta
+```
+
+### Genome de-replication:
+```
+$ dRep dereplicate output_directory -g path/to/genomes/*.fasta
+```
+
+### Make sure dependencies are properly installed:
+```
+$ dRep check_dependencies
+```
+
+## Dependencies
+### Near Essential
+* [skani](https://github.com/bluenote-1577/skani) - Makes primary clusters and performs the default secondary comparison (v0.2+ confirmed works)
+* [CheckM](http://ecogenomics.github.io/CheckM/) - Determines contamination and completeness of genomes (v1.0.7 confirmed works). Only needed for `dereplicate`; skip it with `--genomeInfo` or `--ignoreGenomeQuality`
+
+### Optional
+
+* [Mash](https://genomebiology.biomedcentral.com/articles/10.1186/s13059-016-0997-x>) - Only needed for `--primary_algorithm MASH` (v1.1.1 confirmed works)
+* [MUMmer](http://mummer.sourceforge.net/) - Only needed for the ANIm comparison methods (v3.23 confirmed works)
+* [fastANI](https://github.com/ParBLiSS/FastANI) - An alternative fast secondary clustering algorithm
+* [gANI (aka ANIcalculator)](https://ani.jgi-psf.org/html/download.php?) - Performs gANI comparison method (v1.0 confirmed works)
+* [Prodigal](http://prodigal.ornl.gov/) - Used be both checkM and gANI (v2.6.3 confirmed works)
+* [NSimScan](https://pubmed.ncbi.nlm.nih.gov/27153714/) - Only needed for goANI algorithm (open source version of gANI)

drep-4.0.0/drep/VERSION

@@ -0,0 +1 @@
+4.0.0

{drep-3.7.1 → drep-4.0.0}/drep/argumentParser.py

@@ -114,16 +114,43 @@ def parse_args(args):
 
     Clustflags = cluster_parent.add_argument_group('GENOME COMPARISON OPTIONS')
     Clustflags.add_argument("--S_algorithm", help="R|Algorithm for secondary clustering comaprisons:\n" \
+                                                  + "skani   = (DEFAULT) Kmer-based approach; fastest and most accurate.\n" \
+                                                  + "          When paired with --primary_algorithm skani, secondary reuses\n" \
+                                                  + "          the comparisons already done during primary clustering.\n" \
                                                   + "fastANI = Kmer-based approach; very fast\n" \
-                                                  + "skani = Even faster Kmer-based approacht\n" \
-                                                  + "ANImf   = (DEFAULT) Align whole genomes with nucmer; filter alignment; compare aligned regions\n" \
+                                                  + "ANImf   = Align whole genomes with nucmer; filter alignment; compare aligned regions\n" \
                                                   + "ANIn    = Align whole genomes with nucmer; compare aligned regions\n" \
                                                   + "gANI    = Identify and align ORFs; compare aligned ORFS\n" \
                                                   + "goANI   = Open source version of gANI; requires nsmimscan\n",
-                            default='fastANI', choices={'ANIn', 'gANI', 'ANImf', 'goANI', 'fastANI', 'skani'})
+                            default='skani', choices={'ANIn', 'gANI', 'ANImf', 'goANI', 'fastANI', 'skani'})
+    Clustflags.add_argument("--primary_algorithm", help="R|Program to use for primary clustering.\n" \
+                            + "skani  = (DEFAULT) skani triangle --sparse. Only above-threshold pairs\n" \
+                            + "         are produced, so there is no N^2 matrix in RAM or on disk, and\n" \
+                            + "         a skani --S_algorithm can reuse these comparisons instead of\n" \
+                            + "         recomputing them.\n" \
+                            + "MASH   = all-vs-all Mash. Pre-v4 behavior; builds the full N^2 table.",
+                            default='skani', choices={'MASH', 'skani'})
+    Clustflags.add_argument("--no_reuse_primary_comparisons", dest='reuse_primary_comparisons',
+                            help="Re-run skani during secondary clustering instead of reusing the "
+                                 "comparisons already computed during primary clustering. Only "
+                                 "relevant with --primary_algorithm skani and a skani --S_algorithm, "
+                                 "where the two stages otherwise compute the same ANI values twice. "
+                                 "Reuse is exact, so this is mostly a debugging escape hatch.",
+                            action='store_false', default=True)
+    Clustflags.add_argument("--primary_skani_min_af",
+                            help="Minimum percent of a genome that must align for a pair to form a "
+                                 "primary-clustering edge (--primary_algorithm skani only). skani's ANI "
+                                 "is measured within aligned regions only, so without this filter genomes "
+                                 "sharing just a small conserved region become edges and single linkage "
+                                 "chains them into one huge cluster. The default reproduces the MASH "
+                                 "partition closely; lower it only if you have very fragmented genomes "
+                                 "and understand the chaining risk.",
+                            default=15, type=float)
     Clustflags.add_argument("-ms", "--MASH_sketch", help="MASH sketch size", default=1000)
-    Clustflags.add_argument("--SkipMash", help="Skip MASH clustering,\
-                            just do secondary clustering on all genomes", action='store_true')
+    Clustflags.add_argument("--SkipMash", help="Skip primary clustering entirely and run secondary\
+                            clustering on all genomes at once. (Named for when primary clustering was\
+                            always MASH; it applies to whichever --primary_algorithm is in use.)",
+                            action='store_true')
     Clustflags.add_argument("--SkipSecondary", help="Skip secondary clustering, just perform MASH\
                             clustering", action='store_true')
     Clustflags.add_argument("--skani_extra",
@@ -146,10 +173,19 @@ def parse_args(args):
                                                             + "total   = 2*(aligned length) / (sum of total genome lengths)\n" \
                                                             + "larger  = max((aligned length / genome 1), (aligned_length / genome2))\n",
                            choices=['total', 'larger'], default='larger')
-    Compflags.add_argument("--clusterAlg", help="Algorithm used to cluster genomes (passed\
-                        to scipy.cluster.hierarchy.linkage", default='average',
+    Compflags.add_argument("--clusterAlg", help="Algorithm used to cluster genomes during SECONDARY\
+                        clustering (passed to scipy.cluster.hierarchy.linkage)", default='average',
+                           choices={'single', 'complete', 'average', 'weighted', 'centroid', 'median', 'ward'})
+    Compflags.add_argument("--primary_clusterAlg", help="R|Algorithm used to cluster genomes during PRIMARY\n" \
+                        "(MASH/skani) clustering. The default 'single' is equivalent to connected\n" \
+                        "components and is computed with a fast, low-memory streaming algorithm that\n" \
+                        "scales to very large genome sets. Any other choice falls back to the classic\n" \
+                        "dense scipy path (see --classic_primary_clustering).", default='single',
                            choices={'single', 'complete', 'average', 'weighted', 'centroid', 'median', 'ward'})
-    Compflags.add_argument("--low_ram_primary_clustering", help="Use a memory-efficient algorithm for primary clustering. This only affects primary clustering and not secondary clustering.", 
+    Compflags.add_argument("--classic_primary_clustering", help="Force the classic dense (scipy) primary\
+                        clustering path instead of the streaming single-linkage algorithm. Uses much more\
+                        RAM at scale but reproduces pre-v4 behavior and allows non-single linkage methods\
+                        and the primary dendrogram plot.",
                           action='store_true', default=False)
 
     GRflags = cluster_parent.add_argument_group('GREEDY CLUSTERING OPTIONS\n'

{drep-3.7.1 → drep-4.0.0}/drep/d_analyze.py

@@ -138,6 +138,7 @@ def mash_dendrogram_from_wd(wd, plot_dir=False):
         Cdb = wd.get_db('Cdb', return_none=False)
         Pcluster = wd.get_primary_linkage()
         Plinkage = Pcluster['linkage']
+        Plinkage_db = Pcluster.get('db')
         clust_args = wd.arguments['cluster']
         PL_thresh = clust_args.get('P_ani', False)
         if PL_thresh != False:
@@ -150,10 +151,19 @@ def mash_dendrogram_from_wd(wd, plot_dir=False):
         logging.error("Skipping plot 1 - cannot generate with multiround_primary_clustering enabled")
         return
 
+    if Plinkage is None or isinstance(Plinkage, str):
+        logging.error("Skipping plot 1 - no primary linkage matrix was computed (too many genomes, or a streaming primary algorithm was used)")
+        return
+
+    # Leaf labels have to come from whatever the linkage was built on. The sparse
+    # skani Mdb only holds above-threshold pairs, so a genome with no relatives is
+    # absent from it and labels derived from Mdb would not match the linkage.
+    names = list(Plinkage_db.columns) if Plinkage_db is not None else None
+
     # Make the plot
     logging.info("Plotting primary dendrogram")
     plot_MASH_dendrogram(Mdb, Cdb, Plinkage, threshold = PL_thresh,\
-                    plot_dir = plot_dir)
+                    plot_dir = plot_dir, names = names)
 
 def plot_secondary_dendrograms_from_wd(wd, plot_dir, **kwargs):
     '''
@@ -614,7 +624,7 @@ def plot_ANIn_vs_len(Mdb,Ndb,exclude_zero_MASH=True):
 CLUSETER PLOTS
 """
 
-def plot_MASH_dendrogram(Mdb, Cdb, linkage, threshold=False, plot_dir=False):
+def plot_MASH_dendrogram(Mdb, Cdb, linkage, threshold=False, plot_dir=False, names=None):
     '''
     Make a dendrogram of the primary clustering
 
@@ -624,6 +634,11 @@ def plot_MASH_dendrogram(Mdb, Cdb, linkage, threshold=False, plot_dir=False):
         linkage: Result of scipy.cluster.hierarchy.linkage
         threshold (optional): Line to plot on x-axis
         plot_dir (optional): Location to store plot
+        names (optional): Leaf labels, in the order the linkage was built from.
+            Required when Mdb does not contain every genome -- the sparse skani
+            Mdb only holds above-threshold pairs, so a genome with no relatives
+            never appears in it and deriving labels from Mdb would silently
+            mismatch the linkage.
 
     Returns:
         Makes and shows plot
@@ -633,8 +648,9 @@ def plot_MASH_dendrogram(Mdb, Cdb, linkage, threshold=False, plot_dir=False):
     if Mdb['genome1'].dtype.name == 'category':
         logging.error("WARNING: Primary dendrogram labels may be shuffled! Load as csv to prevent this")
 
-    db = Mdb.pivot(index="genome1", columns="genome2", values="similarity")
-    names = list(db.columns)
+    if names is None:
+        db = Mdb.pivot(index="genome1", columns="genome2", values="similarity")
+        names = list(db.columns)
     name2cluster = Cdb.set_index('genome')['primary_cluster'].to_dict()
     name2color = gen_color_dictionary(names, name2cluster)
 
@@ -1073,6 +1089,32 @@ def gen_color_list(names,name2cluster):
 
     return colors
 
+# UC Berkeley palette. The point of coloring clusters is to tell neighbouring
+# ones apart, not to identify a cluster by its color, so a handful of distinct
+# colors cycled is strictly more readable than giving every cluster its own
+# barely-distinguishable shade.
+CLUSTER_COLORS = [
+    '#003262',  # Berkeley Blue
+    '#FDB515',  # California Gold
+    '#3B7EA1',  # Founders Rock
+]
+
+
+def _cluster_sort_key(cluster):
+    '''
+    Order clusters naturally so that cycling colors lands adjacent clusters on
+    different colors. Handles primary clusters ('2') and secondary clusters
+    ('2_10'), sorting numerically where possible: 2_2 before 2_10, not after.
+    '''
+    key = []
+    for part in str(cluster).split('_'):
+        try:
+            key.append((0, float(part), ''))
+        except ValueError:
+            key.append((1, 0.0, part))
+    return key
+
+
 def gen_color_dictionary(names, name2cluster):
     '''
     Make the dictionary name2color
@@ -1084,27 +1126,14 @@ def gen_color_dictionary(names, name2cluster):
     Returns:
         dict: name -> color
     '''
-    #cm = _rand_cmap(len(set(name2cluster.values()))+1,type='bright')
-    vals = np.linspace(0,1,len(set(name2cluster.values()))+1)
-    np.random.shuffle(vals)
-    cm = plt.cm.colors.ListedColormap(plt.cm.jet(vals))
-
-    # 1. generate cluster to color
-    cluster2color = {}
-    clusters = set(name2cluster.values())
-    NUM_COLORS = len(clusters)
-    for cluster in clusters:
-        try:
-            cluster2color[cluster] = cm(1.*int(cluster)/NUM_COLORS)
-        except:
-            cluster2color[cluster] = cm(1.*float(str(cluster).split('_')[1])/NUM_COLORS)
-
-    #2. name to color
-    name2color = {}
-    for name in names:
-        name2color[name] = cluster2color[name2cluster[name]]
+    # Cycle a small palette in cluster order. This is deterministic: the previous
+    # implementation shuffled an unseeded colormap, so the same analysis produced
+    # different colors on every run.
+    clusters = sorted(set(name2cluster.values()), key=_cluster_sort_key)
+    cluster2color = {c: CLUSTER_COLORS[i % len(CLUSTER_COLORS)]
+                     for i, c in enumerate(clusters)}
 
-    return name2color
+    return {name: cluster2color[name2cluster[name]] for name in names}
 
 def _comp_cluster(c):
     '''

{drep-3.7.1 → drep-4.0.0}/drep/d_cluster/cluster_utils.py

@@ -6,7 +6,6 @@ import numpy as np
 import pandas as pd
 import scipy.cluster
 from scipy.spatial import distance as ssd
-import networkx as nx
 
 import drep.d_cluster.utils
 
@@ -88,41 +87,18 @@ def iteratre_clusters(Bdb, Cdb, id='primary_cluster'):
         yield d, cluster
 
 
-def cluster_threshold_graph_optimized(db, linkage_cutoff=0.10, linkage_method='single'):
-    # Filter distances below threshold first
-    filtered_edges = db[db['dist'] <= linkage_cutoff]
-    
-    # Create graph directly from filtered edges
-    G = nx.Graph()
-    G.add_edges_from(filtered_edges[['genome1', 'genome2']].values)
-    
-    # Log that we're using the optimized method
-    logging.debug("Using low-RAM optimized clustering method with {0} edges".format(len(filtered_edges)))
-    
-    # Find connected components
-    clusters = {}
-    for cluster_id, component in enumerate(nx.connected_components(G)):
-        for genome in component:
-            clusters[genome] = cluster_id + 1
-            
-    # Add isolated nodes (if needed)
-    all_genomes = set(db['genome1']).union(set(db['genome2']))
-    for genome in all_genomes:
-        if genome not in clusters:
-            clusters[genome] = len(clusters)
-            
-    # Return clusters and a special value indicating we used the optimized method
-    return clusters, "optimized_method_used"
-
-def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10, low_ram=False):
+def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10):
     '''
     Perform hierarchical clustering on a symmetrical distiance matrix
 
+    Note this builds a dense matrix and is O(N^2) in memory. Single-linkage
+    primary clustering goes through drep.d_cluster.union_find instead, which is
+    equivalent but does not need the matrix.
+
     Args:
         db: result of db.pivot usually
         linkage_method: passed to scipy.cluster.hierarchy.fcluster
         linkage_cutoff: distance to draw the clustering line (default = .1)
-        low_ram: whether to use the memory-efficient algorithm
 
     Returns:
         list: [Cdb, linkage]
@@ -130,32 +106,21 @@ def cluster_hierarchical(db, linkage_method= 'single', linkage_cutoff= 0.10, low
     # Save names
     names = list(db.columns)
 
-    if low_ram:
-        # Convert to long format for optimized clustering
-        db_long = db.stack().reset_index()
-        db_long.columns = ['genome1', 'genome2', 'dist']
-        clusters, _ = cluster_threshold_graph_optimized(db_long, linkage_cutoff, linkage_method)
-        
-        # Convert clusters to Cdb format
-        Cdb = pd.DataFrame({'genome': list(clusters.keys()), 
-                          'cluster': list(clusters.values())})
-        return Cdb, _
-    else:
-        # Generate linkage dataframe
-        arr =  np.asarray(db)
-        try:
-            arr = ssd.squareform(arr)
-        except:
-            logging.error("The database passed in is not symmetrical!")
-            logging.error(arr)
-            logging.error(names)
-            sys.exit()
-        linkage = scipy.cluster.hierarchy.linkage(arr, method= linkage_method)
-
-        # Form clusters
-        fclust = scipy.cluster.hierarchy.fcluster(linkage,linkage_cutoff, \
-                        criterion='distance')
-        # Make Cdb
-        Cdb = drep.d_cluster.utils._gen_cdb_from_fclust(fclust,names)
-
-        return Cdb, linkage
+    # Generate linkage dataframe
+    arr =  np.asarray(db)
+    try:
+        arr = ssd.squareform(arr)
+    except:
+        logging.error("The database passed in is not symmetrical!")
+        logging.error(arr)
+        logging.error(names)
+        sys.exit()
+    linkage = scipy.cluster.hierarchy.linkage(arr, method= linkage_method)
+
+    # Form clusters
+    fclust = scipy.cluster.hierarchy.fcluster(linkage,linkage_cutoff, \
+                    criterion='distance')
+    # Make Cdb
+    Cdb = drep.d_cluster.utils._gen_cdb_from_fclust(fclust,names)
+
+    return Cdb, linkage