drep 4.0.0__tar.gz → 4.0.2__tar.gz

{drep-4.0.0 → drep-4.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: drep
-Version: 4.0.0
+Version: 4.0.2
 Summary: De-replication of microbial genomes assembled from multiple samples
 Home-page: https://github.com/MrOlm/drep
 Author: Matt Olm
@@ -15,3 +15,9 @@ Requires-Dist: scikit-learn
 Requires-Dist: tqdm
 Requires-Dist: setuptools
 Requires-Dist: pytest
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary

{drep-4.0.0 → drep-4.0.2}/README.md

@@ -52,7 +52,7 @@ $ dRep check_dependencies
 ## Dependencies
 ### Near Essential
 * [skani](https://github.com/bluenote-1577/skani) - Makes primary clusters and performs the default secondary comparison (v0.2+ confirmed works)
-* [CheckM](http://ecogenomics.github.io/CheckM/) - Determines contamination and completeness of genomes (v1.0.7 confirmed works). Only needed for `dereplicate`; skip it with `--genomeInfo` or `--ignoreGenomeQuality`
+* [CheckM](http://ecogenomics.github.io/CheckM/) - Determines contamination and completeness of genomes (v1.0.7 confirmed works). Only needed for `dereplicate`; skip it with `--genomeInfo` (raw CheckM2 or CheckM1 output can be passed directly) or `--ignoreGenomeQuality`
 
 ### Optional
 

drep-4.0.2/drep/VERSION

@@ -0,0 +1 @@
+4.0.2

{drep-4.0.0 → drep-4.0.2}/drep/argumentParser.py

@@ -95,10 +95,11 @@ def parse_args(args):
                 quality filtering. NOT RECOMMENDED! This is useful for use with bacteriophages\
                 or eukaryotes or things where checkM scoring does not work. Will only \
                 choose genomes based on length and N50", action='store_true')
-    Iflags.add_argument('--genomeInfo', help='location of .csv file containing quality \
-                    information on the genomes. Must contain: ["genome"(filename of .fasta file \
+    Iflags.add_argument('--genomeInfo', help='location of .csv or .tsv file containing quality \
+                    information on the genomes (the delimiter is detected automatically). Must contain: ["genome"(filename of .fasta file \
                     of that genome, including extension e.g. genome.fasta), "completeness"(0-100 value for completeness of the genome), \
-                    "contamination"(0-100 value of the contamination of the genome)]')
+                    "contamination"(0-100 value of the contamination of the genome)]. Raw CheckM2 and CheckM1 \
+                    output can also be provided directly')
     Iflags.add_argument("--checkM_method", help="Either lineage_wf (more accurate) \
                         or taxonomy_wf (faster)", choices={'taxonomy_wf', 'lineage_wf'}, \
                         default='lineage_wf')

{drep-4.0.0 → drep-4.0.2}/drep/d_choose.py

@@ -37,7 +37,7 @@ def d_choose_wrapper(wd, **kwargs):
         **kwargs: Command line arguments
 
     Keyword args:
-        genomeInfo: .csv genomeInfo file
+        genomeInfo: .csv or .tsv genomeInfo file
         ignoreGenomeQuality: Don't run checkM or do any quality-based filtering (not recommended)
         checkM_method: Either lineage_wf (more accurate) or taxonomy_wf (faster)
 

{drep-4.0.0 → drep-4.0.2}/drep/d_cluster/external.py

@@ -717,17 +717,31 @@ def add_avani(db):
         db: dataframe
     '''
 
-    logging.debug('making dictionary for average_ani')
-    combo2value = {}
-    for i, row in db.iterrows():
-        combo2value["{0}-vs-{1}".format(row['querry'], row['reference'])] \
-            = row['ani']
-
-    logging.debug('list comprehension for average_ani')
-    db['av_ani'] = [np.mean([combo2value["{0}-vs-{1}".format(q, r)],
-                        combo2value["{0}-vs-{1}".format(r, q)]]) if r != q else 1\
-                        for q, r in zip(db['querry'].tolist(),
-                        db['reference'].tolist())]
+    # Vectorized: this used to iterrows() into a string-keyed dict, which took
+    # hours and hundreds of GB once a single primary cluster held tens of
+    # thousands of genomes (issue #308). Each (reference, querry) pair is encoded
+    # as one integer so its reverse can be looked up in bulk.
+    logging.debug('averaging reciprocal ANI values')
+    ref = db['reference'].values
+    qry = db['querry'].values
+    names = pd.Index(pd.unique(np.concatenate([ref, qry])))
+    n = len(names)
+    r = names.get_indexer(ref).astype(np.int64)
+    q = names.get_indexer(qry).astype(np.int64)
+
+    # When a pair appears more than once the last value wins, as before
+    ani = pd.Series(db['ani'].values, index=r * n + q)
+    ani = ani[~ani.index.duplicated(keep='last')]
+    forward = ani.reindex(r * n + q).values
+    reverse = ani.reindex(q * n + r).values
+
+    same = r == q
+    missing = ~np.isin(q * n + r, ani.index.values) & ~same
+    if missing.any():
+        i = np.argmax(missing)
+        raise KeyError("{0}-vs-{1}".format(ref[i], qry[i]))
+
+    db['av_ani'] = np.where(same, 1, (forward + reverse) / 2)
 
     logging.debug('averageing done')
 

{drep-4.0.0 → drep-4.0.2}/drep/d_filter.py

@@ -28,7 +28,7 @@ def d_filter_wrapper(wd, **kwargs):
 
     Keyword Args:
         genomes: genomes to filter in .fasta format
-        genomeInfo: location of .csv file with the columns: ["genome"(basename of .fasta file of that genome), "completeness"(0-100 value for completeness of the genome), "contamination"(0-100 value of the contamination of the genome)]
+        genomeInfo: location of .csv or .tsv file with the columns: ["genome"(basename of .fasta file of that genome), "completeness"(0-100 value for completeness of the genome), "contamination"(0-100 value of the contamination of the genome)]
 
         processors: Threads to use with checkM / prodigal
         overwrite: Overwrite existing data in the work folder
@@ -127,6 +127,118 @@ def sanity_check(bdb, **kwargs):
         logging.info(f"Hey! You're running multiround_primary_clustering but not run_tertiary_clustering! You should add --run_tertiary_clustering when running with multiround_primary_clustering to avoid weird placement; see https://drep.readthedocs.io/en/latest/choosing_parameters.html#using-greedy-algorithms for more info")
 
 
+GENOMEINFO_REQUIRED_COLUMNS = ['genome', 'completeness', 'contamination']
+GENOMEINFO_DELIMITERS = [(',', 'comma'), ('\t', 'tab'), (';', 'semicolon'), ('|', 'pipe')]
+
+# Columns that dRep knows what to do with in a genomeInfo table
+GENOMEINFO_KNOWN_COLUMNS = ['genome', 'completeness', 'contamination',
+                            'strain_heterogeneity', 'length', 'N50',
+                            'location', 'centrality']
+
+# Raw CheckM2 (quality_report.tsv) and CheckM1 (--tab_table) column names
+GENOMEINFO_COLUMN_TRANSLATION = {
+    'Name': 'genome',                                # CheckM2
+    'Bin Id': 'genome',                              # CheckM1
+    'Completeness': 'completeness',                  # both
+    'Contamination': 'contamination',                # both
+    'Strain heterogeneity': 'strain_heterogeneity',  # CheckM1
+}
+
+# CheckM2 run with --general / --specific / --allmodels reports these instead
+# of a plain "Completeness" column; the first one present is used
+GENOMEINFO_COMPLETENESS_FALLBACKS = ['Completeness_General', 'Completeness_Specific']
+
+def _genomeInfo_translation(columns):
+    '''
+    Figure out how to rename raw CheckM1 / CheckM2 columns into genomeInfo columns
+
+    Raw CheckM2 output is a perfectly good genomeInfo file except that its
+    columns are named "Name", "Completeness", and "Contamination", so translate
+    it rather than making the user do it by hand (issue #305). Columns that are
+    already there under the dRep name are never overwritten
+
+    Args:
+        columns: the columns of the table that was loaded
+
+    Returns:
+        dict: {current column name: dRep column name}
+    '''
+    columns = list(columns)
+    translation = {c: n for c, n in GENOMEINFO_COLUMN_TRANSLATION.items()
+                   if (c in columns) and (n not in columns)}
+
+    if ('completeness' not in columns) and ('Completeness' not in columns):
+        for c in GENOMEINFO_COMPLETENESS_FALLBACKS:
+            if c in columns:
+                translation[c] = 'completeness'
+                break
+
+    # Never rename two columns into the same one; first in the file wins
+    deduped = {}
+    for c in columns:
+        n = translation.get(c, None)
+        if (n is not None) and (n not in deduped.values()):
+            deduped[c] = n
+
+    return deduped
+
+def load_genomeInfo(location):
+    '''
+    Load a user-provided genomeInfo file, figuring out the format as you go
+
+    pandas happily parses a .tsv with sep=',' (you just get one mangled column),
+    so the delimiter can't be picked based on which read_csv call fails to raise
+    an exception. Instead try each delimiter and keep the one that actually
+    yields the columns dRep needs (issue #305). Raw CheckM1 / CheckM2 output is
+    then translated into genomeInfo columns
+
+    Args:
+        location: location of the genomeInfo file
+
+    Returns:
+        DataFrame: genomeInfo
+    '''
+    best = None
+    for sep, sep_name in GENOMEINFO_DELIMITERS:
+        try:
+            db = pd.read_csv(location, sep=sep)
+        except Exception:
+            continue
+        translation = _genomeInfo_translation(db.columns)
+        columns = [translation.get(c, c) for c in db.columns]
+        found = len([c for c in GENOMEINFO_REQUIRED_COLUMNS if c in columns])
+        score = (found, len(db.columns))
+        if (best is None) or (score > best[0]):
+            best = (score, sep_name, db, translation)
+
+    if best is None:
+        raise Exception("Cannot parse the genomeInfo file {0}; it must be a "
+                        "comma- or tab-delimited table".format(location))
+
+    (found, _), sep_name, Idb, translation = best
+    logging.debug("Parsed genomeInfo file {0} as {1}-delimited; the columns are "
+                  "{2}".format(location, sep_name, list(Idb.columns)))
+
+    # Translate raw CheckM1 / CheckM2 output
+    if len(translation) > 0:
+        logging.info("Translating the columns of the provided genomeInfo: {0}".format(
+            ', '.join(['{0} -> {1}'.format(c, n) for c, n in translation.items()])))
+        for c in GENOMEINFO_COMPLETENESS_FALLBACKS:
+            if translation.get(c, None) == 'completeness':
+                logging.warning('This CheckM2 output has no "Completeness" column, '
+                                'so "{0}" is being used as completeness'.format(c))
+        Idb = Idb.rename(columns=translation)
+        Idb = Idb[[c for c in GENOMEINFO_KNOWN_COLUMNS if c in Idb.columns]]
+
+    if found < len(GENOMEINFO_REQUIRED_COLUMNS):
+        logging.warning("Could not find the columns {0} in the genomeInfo file "
+                        "{1}; the best guess of its format is {2}-delimited "
+                        "with the columns {3}".format(
+                            GENOMEINFO_REQUIRED_COLUMNS, location, sep_name,
+                            list(Idb.columns)))
+
+    return Idb
+
 def _get_run_genomeInfo(workDirectory, bdb, **kwargs):
     '''
     Through kwargs and the wd, get genomeInfo
@@ -145,10 +257,7 @@ def _get_run_genomeInfo(workDirectory, bdb, **kwargs):
 
     if kwargs.get('genomeInfo', None) != None:
         logging.debug("Loading provided genome quality information")
-        try:
-            Idb = pd.read_csv(kwargs.get('genomeInfo'))
-        except:
-            Idb = pd.read_csv(kwargs.get('genomeInfo'), sep='\t')
+        Idb = load_genomeInfo(kwargs.get('genomeInfo'))
         try:
             Tdb = _validate_genomeInfo(Idb, bdb)
         except IndexError:
@@ -212,7 +321,8 @@ def _validate_genomeInfo(Idb, bdb):
     # Make sure it has required columns
     for r in ['completeness', 'contamination', 'genome']:
         if r not in Idb.columns:
-            raise KeyError("{0} missing from GenomeInfo".format(r))
+            raise KeyError("{0} missing from GenomeInfo; the columns that are "
+                           "there are {1}".format(r, list(Idb.columns)))
 
     # Make sure correct datatypes
     for r in ['completeness', 'contamination', 'strain_heterogeneity']:
@@ -228,6 +338,28 @@ def _validate_genomeInfo(Idb, bdb):
                 Idb['genome'] = [os.path.basename(x) for x in Idb['location']]
                 break
 
+    # See if the file extension was stripped (this is what CheckM2 does)
+    have = set(Idb['genome'].astype(str))
+    missing = [g for g in bdb['genome'].unique() if g not in have]
+    if len(missing) > 0:
+        matches = {}
+        for genome in missing:
+            base = os.path.splitext(genome)[0]
+            if base in have:
+                matches.setdefault(base, []).append(genome)
+
+        # Only correct names that map back to a single genome
+        ambiguous = {b: g for b, g in matches.items() if len(g) > 1}
+        if len(ambiguous) > 0:
+            logging.warning("Provided genome info is missing the file extension on "
+                            "genomes that differ only by extension, so it can't be "
+                            "corrected: {0}".format(sorted(ambiguous.keys())))
+        translation = {b: g[0] for b, g in matches.items() if len(g) == 1}
+        if len(translation) > 0:
+            logging.warning("Provided genome info is missing the file extension on "
+                            "{0} genomes- correcting".format(len(translation)))
+            Idb['genome'] = [translation.get(g, g) for g in Idb['genome'].astype(str)]
+
     # Make sure it matchs up with bdb
     for genome in list(bdb['genome'].unique()):
         if genome not in Idb['genome'].tolist():

{drep-4.0.0 → drep-4.0.2}/drep.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: drep
-Version: 4.0.0
+Version: 4.0.2
 Summary: De-replication of microbial genomes assembled from multiple samples
 Home-page: https://github.com/MrOlm/drep
 Author: Matt Olm
@@ -15,3 +15,9 @@ Requires-Dist: scikit-learn
 Requires-Dist: tqdm
 Requires-Dist: setuptools
 Requires-Dist: pytest
+Dynamic: author
+Dynamic: author-email
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary

drep-4.0.0/drep/VERSION

@@ -1 +0,0 @@
-4.0.0