cfoldseeker 0.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cfoldseeker-0.0.0/LICENSE +21 -0
- cfoldseeker-0.0.0/PKG-INFO +30 -0
- cfoldseeker-0.0.0/README.md +2 -0
- cfoldseeker-0.0.0/cfoldseeker/__init__.py +0 -0
- cfoldseeker-0.0.0/cfoldseeker/build_cds_db.py +503 -0
- cfoldseeker-0.0.0/cfoldseeker/classes.py +796 -0
- cfoldseeker-0.0.0/cfoldseeker/communication.py +213 -0
- cfoldseeker-0.0.0/cfoldseeker/local.py +308 -0
- cfoldseeker-0.0.0/cfoldseeker/local_clustered.py +193 -0
- cfoldseeker-0.0.0/cfoldseeker/main.py +360 -0
- cfoldseeker-0.0.0/cfoldseeker/remote.py +724 -0
- cfoldseeker-0.0.0/cfoldseeker/remote_parsers.py +128 -0
- cfoldseeker-0.0.0/cfoldseeker.egg-info/PKG-INFO +30 -0
- cfoldseeker-0.0.0/cfoldseeker.egg-info/SOURCES.txt +18 -0
- cfoldseeker-0.0.0/cfoldseeker.egg-info/dependency_links.txt +1 -0
- cfoldseeker-0.0.0/cfoldseeker.egg-info/entry_points.txt +3 -0
- cfoldseeker-0.0.0/cfoldseeker.egg-info/requires.txt +12 -0
- cfoldseeker-0.0.0/cfoldseeker.egg-info/top_level.txt +1 -0
- cfoldseeker-0.0.0/pyproject.toml +54 -0
- cfoldseeker-0.0.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lucas De Vrieze
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cfoldseeker
|
|
3
|
+
Version: 0.0.0
|
|
4
|
+
Summary: Find gene clusters via protein structure similarity
|
|
5
|
+
Author-email: Lucas De Vrieze <lucas.devrieze@kuleuven.be>, Miguel Biltjes <miguel.biltjes@kuleuven.be>
|
|
6
|
+
Maintainer-email: Lucas De Vrieze <lucas.devrieze@kuleuven.be>, Miguel Biltjes <miguel.biltjes@kuleuven.be>
|
|
7
|
+
License: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
13
|
+
Requires-Python: >=3.12.0
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: biopython
|
|
17
|
+
Requires-Dist: tqdm
|
|
18
|
+
Requires-Dist: cblaster>=1.3.20
|
|
19
|
+
Requires-Dist: polars>=1.0.0
|
|
20
|
+
Requires-Dist: networkx
|
|
21
|
+
Requires-Dist: requests
|
|
22
|
+
Requires-Dist: fastexcel
|
|
23
|
+
Requires-Dist: kegg-pull
|
|
24
|
+
Provides-Extra: doc
|
|
25
|
+
Requires-Dist: sphinx; extra == "doc"
|
|
26
|
+
Requires-Dist: sphinx_rtd_theme; extra == "doc"
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
|
|
29
|
+
# cfoldseeker
|
|
30
|
+
A structural twist of cblaster [placeholder repo]
|
|
File without changes
|
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import sys
|
|
6
|
+
import logging
|
|
7
|
+
import warnings
|
|
8
|
+
warnings.filterwarnings('ignore')
|
|
9
|
+
from Bio import Entrez
|
|
10
|
+
import polars as pl
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from tqdm.contrib.concurrent import thread_map
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
LOG = logging.getLogger(__name__)
|
|
16
|
+
logging.basicConfig(
|
|
17
|
+
level = logging.INFO,
|
|
18
|
+
format = "[%(asctime)s] %(levelname)s [%(filename)s: %(funcName)s] - %(message)s",
|
|
19
|
+
datefmt="%H:%M:%S",
|
|
20
|
+
handlers = [logging.StreamHandler(sys.stdout)]
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def parse_arguments() -> argparse.Namespace:
|
|
25
|
+
"""
|
|
26
|
+
This function parses the arguments given through the command line.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
None
|
|
30
|
+
|
|
31
|
+
Returns:
|
|
32
|
+
A Namespace object holding the parsed arguments
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
parser = argparse.ArgumentParser(
|
|
36
|
+
prog = 'cfoldseeker-cds',
|
|
37
|
+
epilog =
|
|
38
|
+
"""
|
|
39
|
+
Lucas De Vrieze, Miguel Biltjes
|
|
40
|
+
(c) 2026 Masschelein lab, VIB
|
|
41
|
+
""",
|
|
42
|
+
formatter_class = argparse.RawDescriptionHelpFormatter,
|
|
43
|
+
description =
|
|
44
|
+
"""
|
|
45
|
+
Helper tool to construct a CDS coordinates DB for cfoldseeker
|
|
46
|
+
""",
|
|
47
|
+
add_help = False
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
parser.add_argument('-i', '--input', dest = 'input', type = Path, default = Path('.'), help = "Path to folder holding the input files or NCBI package (default: current directory)")
|
|
51
|
+
parser.add_argument('-m', '--mode', dest = 'mode', type = str, required = True, choices = ['ncbi-gff', 'ncbi-package', 'bakta-gff', 'tsv', 'excel'],
|
|
52
|
+
help = 'File parsing mode (choices: ncbi-gff, bakta-gff, tsv, excel).')
|
|
53
|
+
parser.add_argument('-o', '--output', dest = 'output', type = Path, default = Path('local_db'), help = "Filepath to save CDS coordinate DB (default: local_db).")
|
|
54
|
+
parser.add_argument('-gz', '--gzip', dest = 'gzip', default = False, action = 'store_true', help = "Gzip output (default: False).")
|
|
55
|
+
parser.add_argument('-tn', '--use-taxon-names', dest = 'use_taxa', default = False, action = 'store_true', help = "Use taxon names as labels for all files instead of filenames (default: False).")
|
|
56
|
+
parser.add_argument('-c', '--cores', dest = 'cores', type = int, default = 1, help = "Number of cores available to use (default: 1).")
|
|
57
|
+
parser.add_argument('-f', '--force', dest = 'force', default = False, action = 'store_true', help = "Force overwriting output (default: false).")
|
|
58
|
+
parser.add_argument('-np', '--no-progress', dest = 'no_progress', default = False, action = "store_true", help = "Don't show progress bar (default: False).")
|
|
59
|
+
parser.add_argument('-h', '--help', action = 'help', help = "Show this help message and exit")
|
|
60
|
+
|
|
61
|
+
args = parser.parse_args()
|
|
62
|
+
|
|
63
|
+
if not args.input.is_dir():
|
|
64
|
+
msg = 'Input folder does not exist.'
|
|
65
|
+
LOG.critical(msg)
|
|
66
|
+
raise argparse.ArgumentError(msg)
|
|
67
|
+
|
|
68
|
+
match args.mode:
|
|
69
|
+
case 'ncbi-gff' | 'bakta-gff':
|
|
70
|
+
any(args.input.glob('*.gff')), "Input folder does not contain GFF files (mind the .gff extension)."
|
|
71
|
+
case 'ncbi-package':
|
|
72
|
+
any(args.input.glob('ncbi_dataset/data/*/genomic.gff')), "NCBI package does not contain GFF files."
|
|
73
|
+
case 'tsv':
|
|
74
|
+
any(args.input.glob('*.tsv')), "Input folder does not contain TSV files (mind the .tsv extension)."
|
|
75
|
+
case 'excel':
|
|
76
|
+
any(args.input.glob('*.xlsx')), "Input folder does not contain Excel files (mind the .xlsx extension)."
|
|
77
|
+
|
|
78
|
+
if args.output.exists():
|
|
79
|
+
if args.force:
|
|
80
|
+
LOG.warning("Output already exists, but it will be overwritten.")
|
|
81
|
+
else:
|
|
82
|
+
msg = "Output already exists! Rerun with -f to overwrite it."
|
|
83
|
+
LOG.error(msg)
|
|
84
|
+
raise IOError(msg)
|
|
85
|
+
else:
|
|
86
|
+
args.output.parent.mkdir(parents = True, exist_ok = True)
|
|
87
|
+
|
|
88
|
+
return args
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _parse_one_ncbi_gff(numbered_filepath: tuple, in_package: bool = False) -> pl.DataFrame:
|
|
92
|
+
"""
|
|
93
|
+
Parses a single NCBI GFF file into a Polars DataFrame.
|
|
94
|
+
|
|
95
|
+
Extracts CDS features and their associated metadata from an NCBI-formatted GFF file,
|
|
96
|
+
including taxonomic ID, gene tags, product names, genomic coordinates, and strand
|
|
97
|
+
information. Aggregates exon coordinates for multi-exon CDS records.
|
|
98
|
+
|
|
99
|
+
Args:
|
|
100
|
+
numbered_filepath (tuple): A tuple containing (index, Path) where index is the file
|
|
101
|
+
number and Path is the file path to the GFF file.
|
|
102
|
+
in_package (bool): If True, extracts filename from parent directory (for NCBI package
|
|
103
|
+
structure). If False, uses file stem as filename. Defaults to False.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
A Polars DataFrame with columns: gene_tag, name, contig, coords, strand,
|
|
107
|
+
taxon_id, and filename. Exons are aggregated into comma-separated coordinates.
|
|
108
|
+
|
|
109
|
+
Note:
|
|
110
|
+
The column filename is a temporary column that is removed by a later method in the workflow.
|
|
111
|
+
It is necessary to construct a taxon name column with NCBI Assembly accession IDs.
|
|
112
|
+
"""
|
|
113
|
+
df = pl.scan_csv(numbered_filepath[1], separator = "\t", has_header = False, comment_prefix = '#',
|
|
114
|
+
new_columns = ['seqid', 'source', 'type', 'start', 'end', 'score', 'strand', 'phase', 'attributes']
|
|
115
|
+
).select(['seqid', 'type', 'start', 'end', 'strand', 'attributes'])
|
|
116
|
+
|
|
117
|
+
df_regions = df.filter(pl.col('type') == 'region')
|
|
118
|
+
df_cds = df.filter(pl.col('type') == 'CDS')
|
|
119
|
+
|
|
120
|
+
# Extract taxon ID
|
|
121
|
+
taxon_id = df_regions.select(pl.col('attributes').str.extract(r'taxon:([0-9]+)')).head(1)
|
|
122
|
+
taxon_id = taxon_id.collect().rows()[0][0]
|
|
123
|
+
|
|
124
|
+
# Get filename
|
|
125
|
+
if in_package:
|
|
126
|
+
filename = numbered_filepath[1].parent.name
|
|
127
|
+
else:
|
|
128
|
+
filename = numbered_filepath[1].stem
|
|
129
|
+
|
|
130
|
+
# Populate the CDS coords DB
|
|
131
|
+
df_cds = df_cds.drop('type')
|
|
132
|
+
cds_record = df_cds.select([pl.col('attributes').str.extract(r'protein_id=([^;]+)').alias('gene_tag'),
|
|
133
|
+
pl.col('attributes').str.extract(r'product=([^;]+)').alias('name'),
|
|
134
|
+
pl.col('seqid').alias('contig'),
|
|
135
|
+
pl.concat_str(['start', 'end'], separator = '..').alias('coords'),
|
|
136
|
+
pl.col('strand'),
|
|
137
|
+
pl.lit(taxon_id).alias('taxon_id'),
|
|
138
|
+
pl.lit(filename).alias('filename') # temporary filename column to help construct a taxon name column later
|
|
139
|
+
])
|
|
140
|
+
cds_record = cds_record.drop_nulls(subset = 'gene_tag')
|
|
141
|
+
|
|
142
|
+
# Aggregate multiple CDSes (i.e. exons) in one record
|
|
143
|
+
cds_record = cds_record.group_by(pl.all().exclude('coords')).agg(pl.col('coords').str.join(','))
|
|
144
|
+
|
|
145
|
+
return cds_record.collect()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _parse_one_bakta_gff(numbered_filepath: tuple) -> pl.DataFrame:
|
|
149
|
+
"""
|
|
150
|
+
Parses a single Bakta GFF file into a Polars DataFrame.
|
|
151
|
+
|
|
152
|
+
Extracts CDS features and metadata from a Bakta-formatted GFF file, including
|
|
153
|
+
locus tags, product names, genomic coordinates, and strand information. Aggregates
|
|
154
|
+
exon coordinates for multi-exon CDS records.
|
|
155
|
+
|
|
156
|
+
Args:
|
|
157
|
+
numbered_filepath (tuple): A tuple containing (index, Path) where index is used as
|
|
158
|
+
the generic taxon ID and Path is the file path to the GFF file.
|
|
159
|
+
|
|
160
|
+
Returns:
|
|
161
|
+
A Polars DataFrame with columns: gene_tag, name, contig, coords, strand,
|
|
162
|
+
taxon_id, and filename. Exons are aggregated into comma-separated coordinates.
|
|
163
|
+
"""
|
|
164
|
+
df = pl.scan_csv(numbered_filepath[1], separator = "\t", has_header = False, comment_prefix = '#',
|
|
165
|
+
new_columns = ['seqid', 'source', 'type', 'start', 'end', 'score', 'strand', 'phase', 'attributes']
|
|
166
|
+
).select(['seqid', 'type', 'start', 'end', 'strand', 'attributes'])
|
|
167
|
+
|
|
168
|
+
df_cds = df.filter(pl.col('type') == 'CDS')
|
|
169
|
+
|
|
170
|
+
# Assign generic taxon ID
|
|
171
|
+
taxon_id = numbered_filepath[0]
|
|
172
|
+
|
|
173
|
+
# Get filename
|
|
174
|
+
filename = numbered_filepath[1].stem
|
|
175
|
+
|
|
176
|
+
# Populate the CDS coords DB
|
|
177
|
+
df_cds = df_cds.drop('type')
|
|
178
|
+
cds_record = df_cds.select([pl.col('attributes').str.extract(r'locus_tag=([^;]+)').alias('gene_tag'),
|
|
179
|
+
pl.col('attributes').str.extract(r'product=([^;]+)').alias('name'),
|
|
180
|
+
pl.col('seqid').alias('contig'),
|
|
181
|
+
pl.concat_str(['start', 'end'], separator = '..').alias('coords'),
|
|
182
|
+
pl.col('strand'),
|
|
183
|
+
pl.lit(taxon_id).alias('taxon_id'),
|
|
184
|
+
pl.lit(filename).alias('filename') # temporary filename column to help construct a taxon name column later
|
|
185
|
+
])
|
|
186
|
+
cds_record = cds_record.drop_nulls(subset = 'gene_tag')
|
|
187
|
+
|
|
188
|
+
# Aggregate multiple CDSes (i.e. exons) in one record
|
|
189
|
+
cds_record = cds_record.group_by(pl.all().exclude('coords')).agg(pl.col('coords').str.join(','))
|
|
190
|
+
|
|
191
|
+
return cds_record.collect()
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _parse_one_tsv(numbered_filepath: tuple) -> pl.DataFrame:
|
|
195
|
+
"""
|
|
196
|
+
Parses a single TSV file into a Polars DataFrame.
|
|
197
|
+
|
|
198
|
+
Reads a tab-separated values file with CDS coordinate data. Expected header:
|
|
199
|
+
gene_tag, name, contig, start, end, strand, taxon_id, taxon_name. Aggregates
|
|
200
|
+
exon coordinates for multi-exon CDS records.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
numbered_filepath (tuple): A tuple containing (index, Path) where index is unused
|
|
204
|
+
and Path is the file path to the TSV file.
|
|
205
|
+
|
|
206
|
+
Returns:
|
|
207
|
+
A Polars DataFrame with columns: gene_tag, name, contig, coords, strand,
|
|
208
|
+
taxon_id, taxon_name. Exons are aggregated into comma-separated coordinates.
|
|
209
|
+
"""
|
|
210
|
+
# Envisioned header: ['gene_tag', 'name', 'contig', 'start', 'end', 'strand', 'taxon_id', 'taxon_name']
|
|
211
|
+
# Exons at multiple lines
|
|
212
|
+
df = pl.scan_csv(numbered_filepath[1], separator = "\t", has_header = True, comment_prefix = '#')
|
|
213
|
+
|
|
214
|
+
# Populate the CDS coords DB
|
|
215
|
+
cds_record = df.with_columns(pl.concat_str(['start', 'end'], separator = '..').alias('coords'))
|
|
216
|
+
cds_record = cds_record.drop(['start', 'end'])
|
|
217
|
+
|
|
218
|
+
# Aggregate multiple CDSes (i.e. exons) in one record
|
|
219
|
+
cds_record = cds_record.group_by(pl.all().exclude('coords')).agg(pl.col('coords').str.join(','))
|
|
220
|
+
|
|
221
|
+
return cds_record.collect()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _parse_one_excel(numbered_filepath: tuple) -> pl.DataFrame:
|
|
225
|
+
"""
|
|
226
|
+
Parses a single Excel file into a Polars DataFrame.
|
|
227
|
+
|
|
228
|
+
Reads an Excel spreadsheet with CDS coordinate data. Expected header:
|
|
229
|
+
gene_tag, name, contig, start, end, strand, taxon_id, taxon_name. Aggregates
|
|
230
|
+
exon coordinates for multi-exon CDS records.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
numbered_filepath (tuple): A tuple containing (index, Path) where index is unused
|
|
234
|
+
and Path is the file path to the Excel file.
|
|
235
|
+
|
|
236
|
+
Returns:
|
|
237
|
+
A Polars DataFrame with columns: gene_tag, name, contig, coords, strand,
|
|
238
|
+
taxon_id, taxon_name. Exons are aggregated into comma-separated coordinates.
|
|
239
|
+
"""
|
|
240
|
+
# Envisioned header: ['gene_tag', 'name', 'contig', 'start', 'end', 'strand', 'taxon_id', 'taxon_name']
|
|
241
|
+
# Exons at multiple lines
|
|
242
|
+
df = pl.read_excel(numbered_filepath[1], has_header = True).lazy()
|
|
243
|
+
|
|
244
|
+
# Populate the CDS coords DB
|
|
245
|
+
cds_record = df.with_columns(pl.concat_str(['start', 'end'], separator = '..').alias('coords'))
|
|
246
|
+
cds_record = cds_record.drop(['start', 'end'])
|
|
247
|
+
|
|
248
|
+
# Aggregate multiple CDSes (i.e. exons) in one record
|
|
249
|
+
cds_record = cds_record.group_by(pl.all().exclude('coords')).agg(pl.col('coords').str.join(','))
|
|
250
|
+
|
|
251
|
+
return cds_record.collect()
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def parse_inputs(input_path: Path, parsing_mode: str, n_workers: int = 1, no_progress: bool = False) -> pl.DataFrame:
|
|
255
|
+
"""
|
|
256
|
+
Parses all input files and constructs a draft CDS coordinates database.
|
|
257
|
+
|
|
258
|
+
Dispatches file parsing based on the specified parsing mode, using
|
|
259
|
+
parallel processing to handle multiple files efficiently. Concatenates all parsed
|
|
260
|
+
results into a single DataFrame.
|
|
261
|
+
|
|
262
|
+
Args:
|
|
263
|
+
input_path (Path): Path to the folder containing input files.
|
|
264
|
+
parsing_mode (str): File format mode - one of: 'ncbi-gff', 'ncbi-package',
|
|
265
|
+
'bakta-gff', 'tsv', or 'excel'.
|
|
266
|
+
n_workers (int): Number of worker threads for parallel file parsing.
|
|
267
|
+
Defaults to 1.
|
|
268
|
+
no_progress (bool): If True, suppresses the progress bar during parsing.
|
|
269
|
+
Defaults to False.
|
|
270
|
+
|
|
271
|
+
Returns:
|
|
272
|
+
A Polars DataFrame containing concatenated CDS records from all input files
|
|
273
|
+
with columns: gene_tag, name, contig, coords, strand, taxon_id, and filename
|
|
274
|
+
(or taxon_name for TSV/Excel formats).
|
|
275
|
+
"""
|
|
276
|
+
# Parse all GFFs
|
|
277
|
+
match parsing_mode:
|
|
278
|
+
case 'ncbi-gff':
|
|
279
|
+
LOG.info('Parsing all input files as NCBI GFFs')
|
|
280
|
+
parsed_gffs_to_concat = thread_map(_parse_one_ncbi_gff, list(enumerate(input_path.glob('*.gff'))),
|
|
281
|
+
max_workers = n_workers,
|
|
282
|
+
leave = False,
|
|
283
|
+
disable = no_progress)
|
|
284
|
+
case 'ncbi-package':
|
|
285
|
+
LOG.info('Parsing all input files as an NCBI GFF package')
|
|
286
|
+
parsed_gffs_to_concat = thread_map(lambda x: _parse_one_ncbi_gff(x, in_package = True),
|
|
287
|
+
list(enumerate(input_path.glob('ncbi_dataset/data/*/genomic.gff'))),
|
|
288
|
+
max_workers = n_workers,
|
|
289
|
+
leave = False,
|
|
290
|
+
disable = no_progress)
|
|
291
|
+
case 'bakta-gff':
|
|
292
|
+
LOG.info('Parsing all input files as Bakta GFFs')
|
|
293
|
+
parsed_gffs_to_concat = thread_map(_parse_one_bakta_gff, list(enumerate(input_path.glob('*.gff'))),
|
|
294
|
+
max_workers = n_workers,
|
|
295
|
+
leave = False,
|
|
296
|
+
disable = no_progress)
|
|
297
|
+
case 'tsv':
|
|
298
|
+
LOG.info('Parsing all input files as TSVs')
|
|
299
|
+
parsed_gffs_to_concat = thread_map(_parse_one_tsv, list(enumerate(input_path.glob('*.tsv'))),
|
|
300
|
+
max_workers = n_workers,
|
|
301
|
+
leave = False,
|
|
302
|
+
disable = no_progress)
|
|
303
|
+
case 'excel':
|
|
304
|
+
LOG.info('Parsing all input files as Excel files')
|
|
305
|
+
parsed_gffs_to_concat = thread_map(_parse_one_excel, list(enumerate(input_path.glob('*.xlsx'))),
|
|
306
|
+
max_workers = n_workers,
|
|
307
|
+
leave = False,
|
|
308
|
+
disable = no_progress)
|
|
309
|
+
|
|
310
|
+
# Concatenate all parsed GFF tables
|
|
311
|
+
LOG.info('Constructing CDS coordinates DB')
|
|
312
|
+
cds_db = pl.concat(parsed_gffs_to_concat)
|
|
313
|
+
|
|
314
|
+
return cds_db
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def check_duplicate_contigs(cds_db: pl.DataFrame, parsing_mode: str) -> pl.DataFrame:
|
|
318
|
+
"""
|
|
319
|
+
Check for and attempt to fix duplicate contig labels per taxon.
|
|
320
|
+
|
|
321
|
+
Detects cases where the same contig label appears in multiple taxa. For
|
|
322
|
+
Bakta GFF files, attempts to prepend the existing locus tag prefix to make contigs
|
|
323
|
+
unique. For other formats, exits with an error.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
cds_db (polars.DataFrame): A dataframe containing CDS records with 'contig',
|
|
327
|
+
'taxon_id', and 'gene_tag' columns.
|
|
328
|
+
parsing_mode (str)): The format mode used for parsing ('bakta-gff', 'ncbi-gff',
|
|
329
|
+
'ncbi-package', 'tsv', or 'excel').
|
|
330
|
+
|
|
331
|
+
Returns:
|
|
332
|
+
cds_db (polars.DataFrame): The input DataFrame with modified contig labels if a fix
|
|
333
|
+
was applied (Bakta mode only).
|
|
334
|
+
|
|
335
|
+
Mutates:
|
|
336
|
+
cds_db (polars.DataFrame): The input DataFrame with modified contig labels if a fix
|
|
337
|
+
was applied (Bakta mode only).
|
|
338
|
+
|
|
339
|
+
Raises:
|
|
340
|
+
RuntimeError: If duplicate contigs are detected and cannot be fixed, or if
|
|
341
|
+
fix attempt fails.
|
|
342
|
+
"""
|
|
343
|
+
# Check that no contig label occurs in combination with multiple taxon IDs
|
|
344
|
+
contig_taxa_combs = cds_db.group_by(['contig']).agg(pl.col('taxon_id').unique())
|
|
345
|
+
nb_contig_taxa_combs = contig_taxa_combs.with_columns(nb_combs = pl.col('taxon_id').list.len())
|
|
346
|
+
multi_taxa_contig = nb_contig_taxa_combs['nb_combs'] > 1
|
|
347
|
+
if multi_taxa_contig.any():
|
|
348
|
+
LOG.error("Detected duplicate contig labels for a taxon!")
|
|
349
|
+
|
|
350
|
+
# In case of Bakta GFFs, we may still fix this if Bakta autogenerated a unique locus tax prefix,
|
|
351
|
+
# by prepending that locus tag prefix
|
|
352
|
+
if parsing_mode == 'bakta-gff':
|
|
353
|
+
LOG.error("Trying to fix this by prepending the locus tag prefix...")
|
|
354
|
+
# Extract the gene tag prefix
|
|
355
|
+
cds_db = cds_db.with_columns(locus_tag_prefix = pl.col('gene_tag').str.split('_').list.reverse().list.slice(1).list.reverse().list.join('_'))
|
|
356
|
+
# Preprend it
|
|
357
|
+
cds_db = cds_db.with_columns(contig = pl.concat_str(['contig', 'locus_tag_prefix'], separator = '_'))
|
|
358
|
+
cds_db = cds_db.drop('locus_tag_prefix')
|
|
359
|
+
|
|
360
|
+
# Check if fix succeeded
|
|
361
|
+
# => No contig label occurring in combination with multiple taxon IDs
|
|
362
|
+
contig_taxa_combs_fixed = cds_db.group_by(['contig']).agg(pl.col('taxon_id').unique())
|
|
363
|
+
nb_contig_taxa_combs_fixed = contig_taxa_combs_fixed.with_columns(nb_combs = pl.col('taxon_id').list.len())
|
|
364
|
+
|
|
365
|
+
multi_taxa_contig_fixed = nb_contig_taxa_combs_fixed['nb_combs'] > 1
|
|
366
|
+
contigs_fixed_counts = not(cds_db.select('contig').is_duplicated().any())
|
|
367
|
+
|
|
368
|
+
# If Not all contig labels are unique despite the fix
|
|
369
|
+
if multi_taxa_contig_fixed.any() and contigs_fixed_counts:
|
|
370
|
+
msg = 'Duplicate gene tag fix failed! Exiting.'
|
|
371
|
+
LOG.critical(msg)
|
|
372
|
+
raise RuntimeError(msg)
|
|
373
|
+
else:
|
|
374
|
+
LOG.warning('Fix succeeded, but I had to change your contig labels!')
|
|
375
|
+
|
|
376
|
+
# For other input types, we don't provide a potential fix
|
|
377
|
+
else:
|
|
378
|
+
msg = "Duplicate contig labels detected for a taxon! No fix provided for this parsing mode."
|
|
379
|
+
LOG.critical(msg)
|
|
380
|
+
raise RuntimeError(msg)
|
|
381
|
+
|
|
382
|
+
return cds_db
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def set_taxon_labels(cds_db: pl.DataFrame, use_taxa: bool, parsing_mode: str, max_attempts: int = 3) -> pl.DataFrame:
|
|
386
|
+
"""
|
|
387
|
+
Set taxon labels as either scientific names or filenames.
|
|
388
|
+
|
|
389
|
+
For NCBI files, optionally fetches the scientific names from NCBI Taxonomy
|
|
390
|
+
via BioPython's NCBI Entrez API with retry logic.
|
|
391
|
+
For Bakta GFF files, generates generic labels or uses filenames.
|
|
392
|
+
For TSV and Excel files, preserves user-provided annotations.
|
|
393
|
+
|
|
394
|
+
Args:
|
|
395
|
+
cds_db (polars DataFrame): Dataframe containing CDS records with 'taxon_id',
|
|
396
|
+
'filename', and optionally 'gene_tag' columns.
|
|
397
|
+
use_taxa (bool): If True, uses scientific names (NCBI) or generates generic
|
|
398
|
+
names (Bakta). If False, uses filenames as taxon labels.
|
|
399
|
+
parsing_mode (str)): The format mode used for parsing ('ncbi-gff', 'ncbi-package',
|
|
400
|
+
'bakta-gff', 'tsv', or 'excel').
|
|
401
|
+
max_attempts (int): Maximum numbers of times to attempt fetching the taxon names
|
|
402
|
+
using Entrez. Defaults to 3.
|
|
403
|
+
|
|
404
|
+
Returns:
|
|
405
|
+
cds_db (polars.DataFrame): The input DataFrame with a new 'taxon_name' column and
|
|
406
|
+
the 'filename' column removed.
|
|
407
|
+
|
|
408
|
+
Mutates:
|
|
409
|
+
cds_db (polars.DataFrame): The input DataFrame with a new 'taxon_name' column and
|
|
410
|
+
the 'filename' column removed.
|
|
411
|
+
|
|
412
|
+
Note:
|
|
413
|
+
This function removes the temporary column 'filename' if present, as it may
|
|
414
|
+
have been introduced when parsing NCBI GFF files.
|
|
415
|
+
"""
|
|
416
|
+
# In case of NCBI files
|
|
417
|
+
if 'ncbi' in parsing_mode:
|
|
418
|
+
# Fetch all taxon names if requested
|
|
419
|
+
if use_taxa:
|
|
420
|
+
LOG.info('Fetching taxon names using NCBI Entrez')
|
|
421
|
+
all_taxon_ids = cds_db.select('taxon_id').unique().to_series().to_list()
|
|
422
|
+
for attempt in range(max_attempts):
|
|
423
|
+
try:
|
|
424
|
+
with Entrez.esummary(db = 'taxonomy', id = all_taxon_ids) as handle:
|
|
425
|
+
records = list(Entrez.read(handle))
|
|
426
|
+
all_taxon_names = [str(i['ScientificName']) for i in records]
|
|
427
|
+
break
|
|
428
|
+
except:
|
|
429
|
+
if attempt+1<max_attempts:
|
|
430
|
+
LOG.warning(f'Failed fetching taxon names in attempt {attempt+1}. Retrying...')
|
|
431
|
+
continue
|
|
432
|
+
else:
|
|
433
|
+
msg = f'Failed fetching taxon names in {max_attempts} attempts. Giving up...'
|
|
434
|
+
LOG.error(msg)
|
|
435
|
+
raise RuntimeError(msg)
|
|
436
|
+
|
|
437
|
+
# Join with the CDS DB
|
|
438
|
+
LOG.info('Adding taxon name column')
|
|
439
|
+
id_name_map = pl.DataFrame({'taxon_id': all_taxon_ids, 'taxon_name': all_taxon_names})
|
|
440
|
+
cds_db = cds_db.join(id_name_map, on = 'taxon_id', how = 'left', maintain_order = "left")
|
|
441
|
+
|
|
442
|
+
# If not requested, use filenames
|
|
443
|
+
else:
|
|
444
|
+
cds_db = cds_db.with_columns(pl.col('filename').alias('taxon_name'))
|
|
445
|
+
|
|
446
|
+
# In case of Bakta GFF files
|
|
447
|
+
elif parsing_mode == 'bakta-gff':
|
|
448
|
+
# Generate a generic taxon name if requested
|
|
449
|
+
if use_taxa:
|
|
450
|
+
LOG.info('Generating generic taxon names')
|
|
451
|
+
cds_db = cds_db.with_columns("Taxon " + pl.col('taxon_id').alias('taxon_name'))
|
|
452
|
+
# Otherwise, use the filename
|
|
453
|
+
else:
|
|
454
|
+
cds_db = cds_db.with_columns(pl.col('filename').alias('taxon_name'))
|
|
455
|
+
|
|
456
|
+
# For other parsing modes, keep the user's annotations
|
|
457
|
+
|
|
458
|
+
# Drop the temporary filename column
|
|
459
|
+
cds_db = cds_db.drop('filename', strict = False)
|
|
460
|
+
|
|
461
|
+
return cds_db
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def main():
|
|
465
|
+
"""
|
|
466
|
+
Main entry point for the CDS database construction tool.
|
|
467
|
+
|
|
468
|
+
Orchestrates the complete workflow: parses command-line arguments, loads
|
|
469
|
+
and parses input files, validates contig uniqueness, assigns taxon labels,
|
|
470
|
+
and writes the final CDS coordinates database to disk as a tab-separated file.
|
|
471
|
+
Supports optional gzip compression.
|
|
472
|
+
"""
|
|
473
|
+
# Process arguments
|
|
474
|
+
args = parse_arguments()
|
|
475
|
+
cores = args.cores
|
|
476
|
+
input_path = args.input
|
|
477
|
+
output_path = args.output
|
|
478
|
+
parsing_mode = args.mode
|
|
479
|
+
use_taxa = args.use_taxa
|
|
480
|
+
no_progress = args.no_progress
|
|
481
|
+
if args.gzip:
|
|
482
|
+
gzip = "gzip"
|
|
483
|
+
else:
|
|
484
|
+
gzip = "uncompressed"
|
|
485
|
+
|
|
486
|
+
# Parse the input files
|
|
487
|
+
cds_db = parse_inputs(input_path, parsing_mode, n_workers = cores, no_progress = no_progress)
|
|
488
|
+
|
|
489
|
+
# Check for duplicate contig labels
|
|
490
|
+
cds_db = check_duplicate_contigs(cds_db, parsing_mode)
|
|
491
|
+
|
|
492
|
+
# Keep taxon names as assembly labels or use the filenames
|
|
493
|
+
cds_db = set_taxon_labels(cds_db, use_taxa, parsing_mode)
|
|
494
|
+
|
|
495
|
+
# Write results
|
|
496
|
+
LOG.info('Writing DB to disk')
|
|
497
|
+
cds_db = cds_db.select(['gene_tag', 'name', 'contig', 'strand', 'coords', 'taxon_id', 'taxon_name'])
|
|
498
|
+
cds_db.write_csv(output_path, separator = '\t', include_header = False, compression = gzip)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
if __name__ == "__main__":
|
|
502
|
+
main()
|
|
503
|
+
|