cagecleaner 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
File without changes
@@ -0,0 +1,2 @@
1
+ from .cagecleaner import main
2
+ main()
@@ -0,0 +1,590 @@
1
+ """
2
+ Usage: `cagecleaner <options>`
3
+
4
+ This python script cleans up redundant hits from the cblaster tool. It has primarly been developed and tested for usage with the NCBI nr database.
5
+
6
+ In its simplest use case, it takes the binary and summary output files as arguments,
7
+ along with a percent identity cutoff to dereplicate the genomes using the skDER tool.
8
+ In addition, it recovers some of the gene cluster diversity lost by the dereplication by assessing gene cluster content and hit score outliers.
9
+
10
+ This tool will produce four final output files
11
+ - cleaned_binary.csv: a file structured in the same way as the cblaster binary output, containing only the retained hits.
12
+ - clusters.txt: the corresponding cluster IDs from the cblaster summary file for each cleaned hit.
13
+ - genome_cluster_sizes.txt: the number of genomes in a dereplication genome cluster, referred to by the dereplication representative genome.
14
+ - genome_cluster_status.txt: a table with scaffoldz IDs, their representative genome assembly and their dereplication status.
15
+
16
+ There are four possible dereplication statuses:
17
+ - 'dereplication_representative': this scaffold is part of the genome assembly that has been selected as the representative of a genome cluster.
18
+ - 'readded_by_content': this scaffold has been kept as it contains a hit that is different in content from the one of the dereplication representative.
19
+ - 'readded_by_score': this scaffold has been kept as it contains a hit that has an outlier cblaster score.
20
+ - 'redundant': this scaffold has not been retained and is therefore removed from the final output.
21
+ """
22
+
23
+ import pandas as pd
24
+ import sys
25
+ import subprocess
26
+ import re
27
+ import os
28
+ from more_itertools import batched
29
+ import gzip
30
+ import shutil
31
+ import argparse
32
+ from Bio import SeqIO
33
+ from scipy.stats import zscore
34
+ from random import choice
35
+
36
+ ACCESSIONS_SCRIPT = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'get_accessions.sh')
37
+ DOWNLOAD_SCRIPT = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'download_assemblies.sh')
38
+ DEREPLICATE_SCRIPT = os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), 'dereplicate_assemblies.sh')
39
+
40
+ GENOMES = "data/genomes"
41
+ SKDER_OUT = "data/skder_out"
42
+
43
+ def validate_input_files(path_to_binary: str, path_to_summary:str) -> bool:
44
+ """
45
+ This function takes the path to a binary and summary file as input and validates if the files are useable for downstream analysis.
46
+
47
+ :param:path_to_binary: Path to the binary file.
48
+ :param:path_to_summary: Path to summary file.
49
+ :rtype bool: True if all checks pass.
50
+ """
51
+
52
+ try:
53
+
54
+ print("--- STEP 0: Validating input files. ---")
55
+ print("Validating binary file.")
56
+
57
+ # First we check if the file exists:
58
+ if not os.path.isfile(path_to_binary):
59
+ raise FileNotFoundError(f"'{path_to_binary}' does not exist. Please check if the binary file path is correct and try again.")
60
+
61
+ # Now we check if the file has at least 6 columns:
62
+ data = pd.read_table(path_to_binary, sep = "\\s{2,}", engine = 'python')
63
+ assert len(data.columns) > 6, "The amount of columns does not correspond to a cblaster binary output file. Please check the file structure. At least 6 columns should be present."
64
+
65
+ # Check if the required column names are present (Organism, Scaffold...):
66
+ assert {"Organism", "Scaffold", "Start", "End", "Score"} < set(data.columns), ...
67
+ "The column names do not correspond to a cblaster binary output file. Please check the file structure.\n The columns 'Organism', 'Scaffold', 'Start', 'End' and 'Score' should be present."
68
+
69
+ # Check if there are at least two hits (3 because of header):
70
+ assert len(data.index) > 3, "We are expecting at least two hits in the cblaster binary output file."
71
+
72
+ print("CHECK")
73
+
74
+ print("Validating summary file.")
75
+
76
+ # Again, check if the summary file exists:
77
+ if not os.path.isfile(path_to_summary):
78
+ raise FileNotFoundError(f"'{path_to_summary}' does not exist. Please check if the summary file path is correct and try again.")
79
+
80
+ # The amount of appearances of the word "Cluster" in the summary file should correspond to the amount of lines
81
+ # in the binary file:
82
+ with open (path_to_summary, 'r') as file:
83
+ assert len(re.findall("Cluster", file.read())) == len(data.index), "The numbers of identifiers in the binary and the summary file do no match."
84
+
85
+ print("CHECK")
86
+
87
+ return True
88
+
89
+ # In case an unknown error occurs, return that the validation did not succeed.
90
+ except:
91
+ return False
92
+
93
+
94
+ def get_scaffolds(path_to_binary: str) -> list:
95
+ """
96
+ This function extracts the scaffold IDs from the cblaster binary output file.
97
+
98
+ :param str path_to_binary: Path to the cblaster binary output file.
99
+ :rtype list: A list containing Genbank and RefSeq Nucleotide IDs.
100
+ """
101
+ print("\n--- STEP 1: Extracting scaffold IDs. ---")
102
+
103
+ # Read the file using a variable series of spaces as separator and extract the second column (containing the scaffold IDs)
104
+ scaffolds = pd.read_table(path_to_binary, sep = "\\s{2,}", engine = 'python', usecols = [1])['Scaffold'].to_list()
105
+
106
+ print(f"Extracted {len(scaffolds)} scaffold IDs")
107
+
108
+ return scaffolds
109
+
110
+
111
+ def get_assemblies(scaffolds: list) -> list:
112
+ """
113
+ This function obtains the genome assembly ID for each scaffold ID obtained by get_scaffolds().
114
+ It uses the NCBI Entrez-Direct utilities via a bash subprocess.
115
+
116
+ :param list scaffolds: A list containing Genbank and RefSeq Nucleotide IDs.
117
+ :rtype list: A list of Genbank and RefSeq Assembly IDs.
118
+ """
119
+
120
+ print("\n--- STEP 2: Retrieving genome assembly IDs. ---")
121
+
122
+ # save the scaffold list in a temporary file
123
+ with open('scaffolds.txt', 'w') as handle:
124
+ handle.writelines('\n'.join(scaffolds))
125
+
126
+ # map to assembly IDs using E-utilities
127
+ subprocess.run(['bash', ACCESSIONS_SCRIPT, 'scaffolds.txt'], check = True)
128
+
129
+ # read the result file
130
+ with open('assembly_accessions', 'r') as handle:
131
+ assemblies = [l.rstrip() for l in handle.readlines()]
132
+
133
+ # remove the temporary files
134
+ os.remove('scaffolds.txt')
135
+ os.remove('assembly_accessions')
136
+
137
+ return assemblies
138
+
139
+
140
+ def download_genomes(assemblies: list, batch_size: int = 300) -> None:
141
+ """
142
+ This function downloads the full nucleotide fasta files of all found assemblies using the NCBI Datasets CLI via a bash subprocess.
143
+ The assemblies are downloaded in batches of 300 by default and saved in the temporary folder data/genomes in the working directory.
144
+
145
+ :param list assemblies: A list containing Genbank and RefSeq assembly IDs.
146
+ :param int batch_size: The number of assemblies to download per batch. [default: 300]
147
+ """
148
+
149
+ print("\n--- STEP 3: Downloading genomes. ---")
150
+
151
+ # Prepare the batches and save them in a temporary file
152
+ with open('download_batches.txt', 'w') as file:
153
+ download_batches = list(batched(assemblies, batch_size))
154
+ for batch in download_batches:
155
+ file.write(' '.join(batch) + '\n')
156
+
157
+ # Run the bash script to download and cluster genomes:
158
+ subprocess.run(["bash", DOWNLOAD_SCRIPT], check=True)
159
+
160
+ # Remove temporary files
161
+ os.remove('download_batches.txt')
162
+
163
+ return None
164
+
165
+
166
+ def map_scaffolds_to_assemblies(scaffolds: list, assemblies: list) -> dict:
167
+ """
168
+ This function maps the scaffolds in the list of cblaster hits to the genome assembly they are part of. To get the exact scaffold IDs of each assembly,
169
+ scaffolds are retrieved from the headers of the assembly fasta files. Prefixes are split off from both scaffold ID sets (the one from cblaster,
170
+ and the one from the NCBI download) during mapping as scaffold IDs mapped by the E-utilities sometimes do not correspond with the ones
171
+ in the downloaded fasta files.
172
+
173
+ :param list scaffolds: A list containing Genbank and RefSeq Nucleotide IDs.
174
+ :param assemblies: A list containing Genbank and RefSeq Assembly IDs.
175
+ :rtype dict: A dictionary with scaffold Nucleotide IDs as keys and a containing assembly ID as value
176
+ """
177
+
178
+ print("\n--- STEP 4: Mapping local scaffold IDs to assembly IDs. ---")
179
+
180
+ # Auxiliary function to split off prefixes from Nucleotide accession IDs
181
+ def split_off_prefix(txt: str) -> str:
182
+ if '_' not in txt:
183
+ return txt
184
+ else:
185
+ return '_'.join(txt.split('_')[1:])
186
+
187
+ # Auxiliary function to link the Nucleotide accession ID without prefix back to the original ID
188
+ def map_back(no_prefix: str, scaffolds: list) -> str:
189
+ return [s for s in scaffolds if no_prefix in s][0]
190
+
191
+ scaffolds_set = set([split_off_prefix(s) for s in scaffolds]) # scaffolds from cblaster, deprefixed
192
+ mappings = {}
193
+
194
+ for assmbl in assemblies:
195
+
196
+ # Find the path to the downloaded fasta file for this assembly ID
197
+ assmbl_file = [a for a in os.listdir(GENOMES) if assmbl in a][0]
198
+
199
+ # All genomes were gzip-compressed
200
+ with gzip.open(os.path.join(GENOMES, assmbl_file), "rt") as genome:
201
+ try:
202
+ # get all scaffold Nucleotide accession IDs in this assembly
203
+ scaffolds_in_this_assembly = [record.id for record in SeqIO.parse(genome, 'fasta')]
204
+
205
+ # split off any prefix
206
+ scaffolds_in_this_assembly = [split_off_prefix(i) for i in scaffolds_in_this_assembly]
207
+
208
+ # find the ones we have a cblaster hit for
209
+ scaffolds_in_this_assembly = set(scaffolds_in_this_assembly)
210
+ found_scaffolds_no_prefix = list(scaffolds_set.intersection(scaffolds_in_this_assembly))
211
+
212
+ # map the deprefixed IDs back to the original ones
213
+ found_scaffolds = [map_back(s, scaffolds) for s in found_scaffolds_no_prefix]
214
+
215
+ # add a mapping item for all scaffolds with a cblaster hit
216
+ for scaff in found_scaffolds:
217
+ mappings[scaff] = assmbl
218
+
219
+ except IndexError:
220
+ print(f'No corresponding scaffold accession ID could be found for {assmbl}!')
221
+
222
+ print(f"Found {len(mappings)} scaffold-assembly links")
223
+
224
+ return mappings
225
+
226
+
227
+ def dereplicate_genomes(ani_threshold: float = 99.0, nb_cores: int = 1) -> None:
228
+ """
229
+ This function takes a list of assembly IDs and calls a helper bash script that dereplicates the genomes using skDER.
230
+ The default ANI cutoff for dereplication is 99 %.
231
+
232
+ A file called "dereplicated_assemblies.txt" is then generated by the helper script, in addition to skDER's output which can be found
233
+ in the temporary folder data/skder_out in the working directory.'
234
+
235
+ :param float ani_threshold: The percent identity cutoff for dereplicating genomes (see skDER docs) [default: 99.0]
236
+ :param int nb_cores: The number of cores to be available for dereplication [default: 1]
237
+ """
238
+ print("\n--- STEP 5: Dereplicating genomes. ---")
239
+
240
+ subprocess.run(['bash', DEREPLICATE_SCRIPT, str(ani_threshold), str(nb_cores)], check = True)
241
+
242
+ return None
243
+
244
+
245
+ def parse_dereplication_clusters(scaffold_assembly_pairs: dict) -> pd.DataFrame:
246
+ """
247
+ This function parses the secondary clustering result file from skDER to characterise the size and the members of the genome clusters.
248
+ It lists which genomes were selected as representative and to which genome cluster all assemblies belong.
249
+ In addition, it produces a report file with the size of the genome clusters.
250
+
251
+ :param dict scaffold_assembly_pairs: A dictionary mapping scaffold Nucleotide IDs to their overarching Assembly ID.
252
+ :rtype Pandas dataframe: A dataframe with Nucleotide IDs in the index and columns 'representative' and 'dereplication_status',
253
+ which, resp., refer to the dereplication representative of that assembly,
254
+ and the dereplication status of that assembly ('dereplication_representative' or 'redundant').
255
+ """
256
+ print("\n--- STEP 6: Parsing dereplication genome clusters. ---")
257
+
258
+ # Auxiliary function to extract assembly accession IDs from the file paths listed in the skDER output table
259
+ def extract_assembly(path: str) -> str:
260
+ pattern = re.compile("GC[A|F]_[0-9]{9}\\.[1-9]+")
261
+ assembly = pattern.findall(path)[0]
262
+ return assembly
263
+
264
+ # Auxiliary function to rename the column names parsed from the skDER output
265
+ def remap_type_label(label: str) -> str:
266
+ mapping = {'representative_to_self': 'dereplication_representative',
267
+ 'within_cutoffs_requested': 'redundant'}
268
+ return mapping[label]
269
+
270
+ # Auxiliary function to retrieve all scaffold Nucleotide IDs with a cblaster hit for a certain assembly ID
271
+ def map_scaffolds(assembly: str, scaffold_assembly_pairs: dict) -> list:
272
+ return [s for s,a in scaffold_assembly_pairs.items() if a == assembly]
273
+
274
+ # Parse the skDER output table. Rename the columns and extract Assembly IDs on-the-fly.
275
+ genome_clusters_df = pd.read_table(os.path.join(SKDER_OUT, "skDER_Clustering.txt"),
276
+ converters={'assembly': extract_assembly,
277
+ 'representative': extract_assembly,
278
+ 'dereplication_status': remap_type_label},
279
+ names = ['assembly', 'representative', 'dereplication_status'],
280
+ usecols = [0,1,4], header = 0, index_col = 'assembly'
281
+ ).sort_values(by = ['representative','dereplication_status'])
282
+
283
+ ## Replace Assembly accession IDs by Nucleotide IDs.
284
+ ## If there are multiple Nucleotide IDs with a cblaster hit, the records will be replicated.
285
+
286
+ # Deconstruct the initial dataframe from the parsing into a dictionary with index values as keys and data as values
287
+ genome_clusters_records = genome_clusters_df.to_dict(orient = 'index')
288
+
289
+ # Build a similar dictionary using Nucleotide accessions as keys, replicating data if needed
290
+ genome_clusters = {}
291
+ for assembly, data in genome_clusters_records.items():
292
+ mapped_scaffolds = map_scaffolds(assembly, scaffold_assembly_pairs) # map assembly to a list of scaffold(s) with a cblaster hit
293
+ for scaffold in mapped_scaffolds:
294
+ genome_clusters[scaffold] = data
295
+
296
+ # Construct a new dataframe from this Nucleotide-keyed dictionary
297
+ genome_clusters = pd.DataFrame.from_dict(genome_clusters, orient = 'index', columns = ['representative', 'dereplication_status'])
298
+ genome_clusters = genome_clusters.sort_values(by = ['representative','dereplication_status'])
299
+
300
+ # Determine the size of each genome cluster and write report file
301
+ clust_size = pd.DataFrame(genome_clusters.groupby(by = "representative")['representative'].count()).rename(columns = {'representative': 'size'})
302
+ clust_size.to_csv('genome_cluster_sizes.txt', sep = "\t")
303
+ print('Genome cluster sizes written in genome_cluster_sizes.txt')
304
+
305
+ return genome_clusters
306
+
307
+
308
+ def recover_hits(path_to_binary: str, genome_clusters_mapping: pd.DataFrame, not_by_content: bool = False, not_by_score: bool = False, outlier_z: float = 2, min_score_diff: float = 0.1) -> pd.DataFrame:
309
+ """
310
+ This function recovers some variation in gene clusters that was lost due to dereplicating the hosting genomes. It offers two approaches to
311
+ flag interesting gene clusters that will be kept in the output.
312
+
313
+ 1) Different gene cluster content
314
+ A scaffold that is part of a dereplication cluster may have a different gene cluster content than its representative, i.e. a different number of
315
+ identified homologs. The cblaster binary table also lists the number of homologs were found in a gene cluster for each query gene.
316
+ If these numbers are different from the ones of the representative assembly, flag to keep this scaffold and its hit.
317
+
318
+ 2) Outlier cblaster score
319
+ cblaster adds a 'Score' column in the binary output table that captures the numbers of homologs as well as aggregates the level of homology
320
+ of the entire cluster. If this score is significantly different from the other scores, this may indicate multiple gene cluster lineages within
321
+ this genome cluster, or a remarkably fast evolving gene cluster. In any case, it is interesting to retain this hit.
322
+ Signficance is currently determined using z-scores for each cluster content group.
323
+
324
+ :param str path_to_binary: path to the cblaster binary result table
325
+ :param pandas Dataframe genome_clusters_mapping: dataframe returned by the parse_dereplication_clusters() function, containing the dereplication
326
+ status and representative of each assembly
327
+ :param bool not_by_content: flag to disable recovering gene clusters by gene cluster content, also disables recovery by outlier score [default: False]
328
+ :param bool not_by_score: flag to disable recovering gene clusters by outlier cblaster score [default: False]
329
+ :param float outlier_z: minimum absolute value of the z-score to consider a hit as an outlier
330
+ :param float min_score_diff: minimum difference in cblaster score between a hit and the mode score when determining outlier scores.
331
+ Outliers with a score difference below this value are discarded. [default: 0.1]
332
+ :rtype Pandas dataframe: updated dereplication status table, now also containing flags to keep scaffolds and their gene clusters in the final output
333
+ """
334
+ print("\n--- STEP 7: Recovering gene cluster diversity. ---")
335
+
336
+ # Make a copy of the original genome cluster table
337
+ updated_mapping = genome_clusters_mapping.copy()
338
+
339
+ # Skip this if not recovering by gene cluster content
340
+ if not(not_by_content):
341
+
342
+ # Read the relevant columns from the cblaster binary table
343
+ with open(path_to_binary, 'r'):
344
+ hits = pd.read_table(path_to_binary, sep = "\\s{2,}", engine = 'python',
345
+ usecols = lambda x: x not in ['Organism', 'Start', 'End'],
346
+ index_col = "Scaffold")
347
+
348
+ ## Hits are recovered within dereplication clusters so we will check the hits of each dereplication cluster
349
+ # Get the IDs of all assemblies grouped by their dereplication representative
350
+ grouped_mapping = genome_clusters_mapping.reset_index(names = 'scaffold').groupby('representative').agg(list)
351
+ genome_groups = dict(zip(grouped_mapping.index, grouped_mapping['scaffold']))
352
+
353
+ for representative, group in genome_groups.items():
354
+
355
+ # Get the cblaster hit data for scaffolds of this dereplication cluster
356
+ hits_this_group = hits.loc[group]
357
+
358
+ # Split the dereplication grouping further into gene cluster subgroups as reflected by the number of homologs of each query gene
359
+ hits_this_group_by_content_group = [l.to_list() for l in hits_this_group.groupby(
360
+ list(set(hits_this_group.columns).difference({'Score'})) # to get all query gene columns
361
+ ).groups.values()]
362
+
363
+ # Loop over all structural subgroups
364
+ for content_group in hits_this_group_by_content_group:
365
+ scores_this_content_group = hits_this_group.loc[content_group, 'Score'] # cblaster scores
366
+ mode_score_this_content_group = float(scores_this_content_group.mode().iloc[0]) # modal cblaster score
367
+ zscores_this_content_group = scores_this_content_group.transform(zscore) # zscores
368
+
369
+ # If the result of the z-score calculation yields all NaNs, all cblaster scores were identical, implying there is no score outlier.
370
+ # In that case, we can continue flagging different gene cluster contents, if any
371
+ # If the user is not interested in flagging score outlier hits, we end up in this case anyway.
372
+ if not_by_score or zscores_this_content_group.isna().all():
373
+
374
+ # If the dereplication representative is one of the assemblies that has a scaffold in this structural subgroup,
375
+ # then we already have a hit from this subgroup, so we can skip this one.
376
+ dereplication_status_this_content_group = [genome_clusters_mapping.loc[m, 'dereplication_status'] for m in content_group]
377
+ if "dereplication_representative" in dereplication_status_this_content_group:
378
+ continue
379
+
380
+ # If the dereplication representative is not in this structural subgroup, randomly pick a member as a representative hit to keep
381
+ else:
382
+ content_group_representative = choice(content_group)
383
+ updated_mapping.at[content_group_representative, 'dereplication_status'] = 'readded_by_cluster_content'
384
+
385
+ # The case there were different cblaster scores and the user is interested in score outliers
386
+ else:
387
+
388
+ # Select hits that have an absolute z-score above the threshold,
389
+ # and of which the cblaster score is sufficiently different from the modal score,
390
+ # to avoid 'false' outliers that are just a little bit more different than most hits
391
+ outliers_this_content_group = zscores_this_content_group.loc[
392
+ (zscores_this_content_group.abs() >= outlier_z) &
393
+ ((scores_this_content_group - mode_score_this_content_group).abs() >= min_score_diff)].index
394
+
395
+ # All score outliers will be added
396
+ for outlier in outliers_this_content_group.to_list():
397
+ # If the overarching assembly of this outlier is the dereplication representative, skip flagging it
398
+ dereplication_status_outlier = genome_clusters_mapping.loc[outlier, 'dereplication_status']
399
+ if dereplication_status_outlier == "dereplication_representative":
400
+ continue
401
+ else:
402
+ updated_mapping.at[outlier, 'dereplication_status'] = 'readded_by_outlier_score'
403
+
404
+ # We still have to flag a non-outlier representative of this structural subgroup
405
+ non_outliers_this_content_group = zscores_this_content_group.drop(index = outliers_this_content_group).index.to_list()
406
+
407
+ # If this subgroup contains a scaffold of the dereplication representative, skip this subgroup
408
+ dereplication_status_non_outliers_this_content_group = [genome_clusters_mapping.loc[m, 'dereplication_status']
409
+ for m in non_outliers_this_content_group]
410
+ if "dereplication_representative" in dereplication_status_non_outliers_this_content_group:
411
+ continue
412
+
413
+ # If not, pick a random representative from the non-outlier hits
414
+ else:
415
+ content_group_representative = choice(non_outliers_this_content_group)
416
+ updated_mapping.at[content_group_representative, 'dereplication_status'] = 'readded_by_cluster_content'
417
+
418
+ # There is nothing to recover in this case
419
+ else:
420
+ print('All revisiting options have been disabled. Not updating the genome grouping labels.')
421
+
422
+ # Tidy up the updated mapping and write report file
423
+ updated_mapping = updated_mapping.sort_values(by = ['representative', 'dereplication_status'])
424
+ updated_mapping.reset_index(names = 'scaffold').to_csv('genome_cluster_status.txt', sep = "\t", index = False)
425
+
426
+ return updated_mapping
427
+
428
+
429
+ def get_dereplicated_scaffolds(genome_clusters: pd.DataFrame) -> list:
430
+ """
431
+ This function retrieves the final retained scaffold IDs, after dereplication and hit recovery.
432
+
433
+ :param Pandas dataframe; A dataframe containing the final status of each scaffold and its dereplication representative
434
+ :rtype list: A list containing the retained scaffolds
435
+ """
436
+ print("\n--- STEP 8: Gathering retained scaffold IDs. ---")
437
+
438
+ dereplicated_scaffolds = genome_clusters[genome_clusters['dereplication_status'] != 'redundant'].index.to_list()
439
+
440
+ print(f"Got {len(dereplicated_scaffolds)} representative scaffold IDs")
441
+
442
+ return dereplicated_scaffolds
443
+
444
+
445
+ def write_output(dereplicated_scaffolds:list, path_to_summary:str, path_to_binary:str) -> None:
446
+ """
447
+ This function takes a list of retained scaffold IDs and the cblaster summary and binary output files.
448
+ It writes the corresponding Cluster IDs and cblaster hit entries to a file.
449
+
450
+ :param list dereplicated_scaffolds: A list containing the retained scaffold IDs.
451
+ :param str path_to_summary: Path to summary file.
452
+ :param str path_to_binary: Path to binary file.
453
+ """
454
+ print("\n--- STEP 9: Generating output files. ---")
455
+
456
+ ## First we do the binary file:
457
+
458
+ # Create intermediary list to store the cleaned hits:
459
+ cleaned_hits = []
460
+
461
+ with open(path_to_binary, 'r',) as file:
462
+ file_content = file.read()
463
+ # We also capture the header to write to our cleaned file later:
464
+ header = file_content.split("\n")[0] + "\n"
465
+
466
+ # Loop over the cleaned scaffold IDs and match them in the binary file using regex.
467
+ for scaffold in dereplicated_scaffolds:
468
+ pattern = f".*{scaffold}.*"
469
+ # Append to the list of cleaned hits:
470
+ cleaned_hits.append(re.search(pattern, file_content).group(0))
471
+
472
+ with open('cleaned_binary.txt', 'w') as file:
473
+ file.write(header)
474
+ for cleaned_hit in cleaned_hits:
475
+ file.write(f"{cleaned_hit}\n")
476
+
477
+ ## Secondly, the cluster IDs. Same principle but then with the summary file:
478
+
479
+ # Create an intermediary list to store the cluster IDs:
480
+ clusters = []
481
+
482
+ with open(path_to_summary, 'r') as file:
483
+ file_content = file.read()
484
+
485
+ # Loop over each scaffold ID
486
+ for scaffold in dereplicated_scaffolds:
487
+ pattern = f"({scaffold}\n[-]*\n)(Cluster \\d*)"
488
+ clusters.append(re.search(pattern, file_content).group(2))
489
+
490
+ with open('clusters.txt', 'w') as file:
491
+ for cluster in clusters:
492
+ file.write(f"{cluster}\n")
493
+
494
+ return None
495
+
496
+
497
+ def parse_arguments():
498
+ """
499
+ Argument parsing function
500
+ """
501
+
502
+ # Auxiliary function to check the value range of the ANI threshold
503
+ def check_percentage(value):
504
+ if 0 <= float(value) <= 100:
505
+ return value
506
+ else:
507
+ raise argparse.ArgumentTypeError("%s should be a percentage value between 0 and 100" % value)
508
+
509
+ parser = argparse.ArgumentParser(prog = 'ccleaner', description = "Tool to reduce redundancy in cblaster hits")
510
+ parser.add_argument('-o', '--output', dest = "work_dir", default = '.')
511
+ parser.add_argument('-b', '--binary', dest = "binary")
512
+ parser.add_argument('-s', '--summary', dest = "summary")
513
+ parser.add_argument('-c', '--cores', dest = 'cores', default = 1)
514
+ parser.add_argument('-a', dest = 'ani', default = 99.0, type = check_percentage)
515
+ parser.add_argument('--no-content-revisit', dest = 'no_revisit_by_content', default = False, action = "store_true")
516
+ parser.add_argument('--no-score-revisit', dest = 'no_revisit_by_score', default = False, action = "store_true")
517
+ parser.add_argument('--min-z-score', dest = 'outlier_z', default = 2.0)
518
+ parser.add_argument('--min-score-diff', dest = 'min_score_diff', default = 0.1)
519
+ parser.add_argument('--download-batch', dest = 'download_batch', default = 300)
520
+ parser.add_argument('--validate-files', dest = "validate_inputs", default = False, action = "store_true")
521
+ parser.add_argument('--keep-downloads', dest = "keep_downloads", default = False, action = "store_true")
522
+ parser.add_argument('--keep-dereplication', dest = "keep_dereplication", default = False, action = "store_true")
523
+ parser.add_argument('--keep-intermediate', dest = "keep_intermediate", default = False, action = "store_true")
524
+ args = parser.parse_args()
525
+
526
+ return args
527
+
528
+ def main():
529
+ """
530
+ Main function
531
+ """
532
+ args = parse_arguments()
533
+
534
+ work_dir = args.work_dir.rstrip('/')
535
+ path_to_binary = os.path.join(os.getcwd(), args.binary)
536
+ path_to_summary = os.path.join(os.getcwd(), args.summary)
537
+ nb_cores = int(args.cores)
538
+ ani_threshold = float(args.ani)
539
+ no_revisit_by_content = args.no_revisit_by_content
540
+ no_revisit_by_score = args.no_revisit_by_score
541
+ outlier_z = float(args.outlier_z)
542
+ min_score_diff = float(args.min_score_diff)
543
+ download_batch_size = int(args.download_batch)
544
+ validate_inputs = args.validate_inputs
545
+ keep_downloads = args.keep_downloads
546
+ keep_intermediate = args.keep_intermediate
547
+ keep_dereplication = args.keep_dereplication
548
+
549
+ os.makedirs(work_dir, exist_ok = True)
550
+ os.chdir(work_dir)
551
+
552
+ # Validate the input files. If a covered assertation fails, it will fail inside the function. An unknown error is captured by this
553
+ # main assert statement
554
+ if validate_inputs:
555
+ assert validate_input_files(path_to_binary, path_to_summary), ...
556
+ "Something was wrong with the input files! Please check them and try again."
557
+
558
+ # extract scaffold IDs from the cblaster output
559
+ scaffolds = get_scaffolds(path_to_binary)
560
+ # link to assembly IDs via the NCBI E-utilities
561
+ assemblies = get_assemblies(scaffolds)
562
+ # download assemblies using the NCBI Datasets CLI
563
+ download_genomes(assemblies, download_batch_size)
564
+ # map the downloaded scaffold IDs to assembly IDs
565
+ scaffold_assembly_pairs = map_scaffolds_to_assemblies(scaffolds, assemblies)
566
+ # dereplicate the genomes using skDER
567
+ dereplicate_genomes(ani_threshold, nb_cores)
568
+ # parse the secondary clustering from the skDER output and construct a dereplication status table
569
+ genome_cluster_composition = parse_dereplication_clusters(scaffold_assembly_pairs)
570
+ # recover gene cluster hits and update
571
+ updated_status = recover_hits(path_to_binary, genome_cluster_composition, no_revisit_by_content, no_revisit_by_score, outlier_z, min_score_diff)
572
+ # retrieve the finally retained scaffold IDs from the updated status table
573
+ dereplicated_scaffolds = get_dereplicated_scaffolds(updated_status)
574
+ # generate final output files
575
+ write_output(dereplicated_scaffolds, path_to_summary, path_to_binary)
576
+
577
+ # Finish by removing intermediate results if requested
578
+ if not(keep_intermediate):
579
+ if not(keep_downloads) and not(keep_dereplication):
580
+ shutil.rmtree('data')
581
+ else:
582
+ if not(keep_downloads):
583
+ shutil.rmtree(GENOMES)
584
+ if not(keep_dereplication):
585
+ shutil.rmtree(SKDER_OUT)
586
+
587
+ print(f"All done! Results are written to {work_dir}")
588
+
589
+ if __name__ == "__main__":
590
+ main()
@@ -0,0 +1,20 @@
1
+ #!/bin/bash
2
+
3
+ # This helper script dereplicates previously downloaded genomes using skDER.
4
+
5
+ echo Preparing to dereplicate genomes...
6
+
7
+ # If the download folder still exists, it will be deleted:
8
+ rm -rf data/downloads
9
+
10
+ pi_cutoff=$1
11
+ nb_cores=$2
12
+
13
+ # Run skDER on the downloaded genomes and enable secondary clustering (-n flag):
14
+ echo Dereplicating genomes with percent identity cutoff of $pi_cutoff.
15
+ echo -e "Starting skDER\n"
16
+ skder -g data/genomes/* -o data/skder_out -i $pi_cutoff -c $nb_cores -n
17
+
18
+ # skDER stores the dereplicated genomes in its own output folder. Compare the amount of files in skder_out folder with initial folder where
19
+ # all genomes reside.
20
+ echo -e "\nDereplication done! $(ls data/genomes | wc -w) genomes were reduced to $(ls data/skder_out/Dereplicated_Representative_Genomes | wc -w) genomes"
@@ -0,0 +1,49 @@
1
+ #!/bin/bash
2
+
3
+ # This helper script reads a formatted text file with batches of assembly IDs and downloads the corresponding genomes.
4
+
5
+ echo Preparing to download assemblies...
6
+
7
+ # If intermediary/output files and folders already exist, they will be deleted:
8
+ rm -rf data
9
+
10
+ # Set some useful variables:
11
+ assembly_count=$(cat download_batches.txt | wc -w) # Amount of words corresponds to amount of assembly IDs.
12
+ batch_count=$(cat download_batches.txt | wc -l) # Each line is one batch. Enforced by the parent python script.
13
+
14
+ echo Got $assembly_count assembly IDs. Downloading genomes in $batch_count batches...
15
+
16
+ # Create the directory to store all genomes
17
+ mkdir -p data/genomes data/downloads
18
+
19
+ # Here we loop over each line in the 'download_batches.txt' file:
20
+ batch_counter=1
21
+ while read line; do
22
+ echo -e "\nBatch $batch_counter/$batch_count"
23
+
24
+ # Download the dehydrated genome package. xargs removes the trailing whitespace:
25
+ datasets download genome accession $(echo "$line" | xargs) --dehydrated
26
+
27
+ # Unzip the file in a folder called "genomes". This folder is located two steps up in the data folder
28
+ unzip -d data/downloads ncbi_dataset.zip && rm ncbi_dataset.zip
29
+
30
+ # Rehydrate:
31
+ datasets rehydrate --directory data/downloads --gzip
32
+
33
+ # Put all genomes from this batch into one directory
34
+ mv data/downloads/ncbi_dataset/data/GC*/* data/genomes/
35
+
36
+ # Clean up
37
+ rm -rf data/downloads
38
+
39
+ batch_counter=$((batch_counter+1))
40
+ done < download_batches.txt
41
+
42
+ echo -e "\nDownloading finished!"
43
+
44
+ # Print amount of files in the /genomes/all/ folder. This should equal the initial amount of assembly IDs
45
+ echo $(ls data/genomes | wc -w) genomes in $(pwd)/data/genomes.
46
+ echo Directory size: $(du --human-readable -s data/genomes). # Size of the genomes folder
47
+
48
+ # Cleaning up
49
+ rm -rf data/downloads
@@ -0,0 +1,40 @@
1
+ #!/bin/bash
2
+
3
+ # This helper script retrieves the NCBI Assembly IDs linked to a list of Nucleotide IDs using the NCBI E-utilities.
4
+ # It discerns between non-WGS entries and WGS entries. While the former contain sequence data and are directly linked to an assembly ID,
5
+ # the latter are only indirectly linked to an assembly ID via its master record.
6
+
7
+ scaffolds_list=$1
8
+
9
+ ## non-WGS entries
10
+ echo "--> Checking for non-WGS entries"
11
+ echo "Found $(cat $scaffolds_list | grep -E '_|^[A-Z|0-9]{8}\.[1-9]' | wc -l) non-WGS Nucleotide accession codes."
12
+ echo "Querying NCBI..."
13
+ # non-WGS entries can be directly fetched via a Nucleotide-Assembly link.
14
+ elink -db nucleotide -target assembly -id $(cat $scaffolds_list | grep -E '_|^[A-Z|0-9]{8}\.[1-9]') | efetch -format docsum | xtract -pattern DocumentSummary -element AssemblyAccession > non_wgs_assembly_accessions
15
+ echo "Got $(cat non_wgs_assembly_accessions | wc -l) non-WGS Assembly accession codes"
16
+
17
+ ## WGS entries
18
+ echo "--> Checking for WGS entries"
19
+ echo "Found $(cat $scaffolds_list | grep -vE '_|^[A-Z|0-9]{8}\.[1-9]' | wc -l) WGS Nucleotide accession codes."
20
+ # WGS entries need to be redirected to the master record holding all the separate scaffold records in Nucleotide
21
+ # before linking to Assembly and fetching the assembly
22
+ echo "Redirecting to WGS master record accession codes..."
23
+ cat $scaffolds_list | grep -vE '_|^[A-Z|0-9]{8}\.[1-9]' |
24
+ sed 's/[1-9][0-9]{5}\.[1-9]/000000/g' |
25
+ sed 's/[1-9][0-9]{4}\.[1-9]/00000/g' |
26
+ sed 's/[1-9][0-9]{3}\.[1-9]/0000/g' |
27
+ sed 's/[1-9][0-9]{2}\.[1-9]/000/g' |
28
+ sed 's/[1-9][0-9]\.[1-9]/00/g' |
29
+ sed 's/[1-9]\.[1-9]/0/g' > wgs_masters
30
+ echo "Querying NCBI..."
31
+ elink -db nucleotide -target assembly -id $(cat wgs_masters) | efetch -format docsum | xtract -pattern DocumentSummary -element AssemblyAccession > wgs_assembly_accessions
32
+ echo "Got $(cat wgs_assembly_accessions | wc -l) WGS Assembly accession codes"
33
+
34
+ ## concatenating the retrieved accession codes
35
+ echo "Merging results..."
36
+ cat non_wgs_assembly_accessions wgs_assembly_accessions | sort -u > assembly_accessions
37
+ echo "Found $(cat assembly_accessions | wc -l) accession codes after removing duplicates"
38
+
39
+ ## Cleaning up
40
+ rm *wgs_*
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Lucas De Vrieze
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,14 @@
1
+ Metadata-Version: 2.1
2
+ Name: cagecleaner
3
+ Version: 0.0.1
4
+ Summary: Redundancy removal tool for cblaster hits
5
+ Author: Lucas De Vrieze
6
+ Author-email: lucas.devrieze@kuleuven.be
7
+ License: MIT
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
13
+ Requires-Python: >=3.10
14
+ License-File: LICENSE
@@ -0,0 +1,12 @@
1
+ cagecleaner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ cagecleaner/__main__.py,sha256=YgDdH7LpwjR2oYkZlzs537vso6IQVGcKMi2T01TK7O0,37
3
+ cagecleaner/cagecleaner.py,sha256=5tcnXTO7tW5-KOZvz6QJhEKzUsLMucP76UxR-lUqAfA,31833
4
+ cagecleaner-0.0.1.data/scripts/dereplicate_assemblies.sh,sha256=eMMl_CrZz5plt1iA_UzMSq8HRKOgHo--xxNL7PBkEsg,811
5
+ cagecleaner-0.0.1.data/scripts/download_assemblies.sh,sha256=vbiSKnOBjVAcGxoE9gbb_f0fWZyjgzYXFyq5aQCXFYc,1824
6
+ cagecleaner-0.0.1.data/scripts/get_accessions.sh,sha256=IBjeSnr3dy7RKr2D2tIix8ZYMo41apKbxIMC4-xfzl0,2172
7
+ cagecleaner-0.0.1.dist-info/LICENSE,sha256=VQC-1gyJrLi5_dEbrhcEjcGdhnVi8G4HTzzdd7xJRCk,1072
8
+ cagecleaner-0.0.1.dist-info/METADATA,sha256=8FyXcWUEJERxKsGYsm-0SXNDb6d25inKgO62d9ybA-I,489
9
+ cagecleaner-0.0.1.dist-info/WHEEL,sha256=A3WOREP4zgxI0fKrHUG8DC8013e3dK3n7a6HDbcEIwE,91
10
+ cagecleaner-0.0.1.dist-info/entry_points.txt,sha256=ICL0MhQTcTuKbTCMz9ixG7vm-48GO0TLtAqhZINR-iM,61
11
+ cagecleaner-0.0.1.dist-info/top_level.txt,sha256=Sb5MYGEyvYIEbvd7xCDTR4dceipIFUoZKO-aw75C33E,12
12
+ cagecleaner-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (75.7.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ cagecleaner = cagecleaner.cagecleaner:main
@@ -0,0 +1 @@
1
+ cagecleaner