gspec 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gapit/__init__.py +3 -0
- gapit/blast.py +291 -0
- gapit/bundled.py +240 -0
- gapit/bundled_cluster.py +130 -0
- gapit/cli.py +136 -0
- gapit/cluster.py +367 -0
- gapit/cluster_alignment.py +322 -0
- gapit/cluster_math.py +163 -0
- gapit/cluster_profile.py +97 -0
- gapit/clusterbuild.py +424 -0
- gapit/cmd_db.py +203 -0
- gapit/cmd_db_build.py +142 -0
- gapit/cmd_db_info.py +63 -0
- gapit/cmd_db_install.py +126 -0
- gapit/cmd_db_outdated.py +50 -0
- gapit/cmd_db_rename.py +35 -0
- gapit/cmd_db_search.py +82 -0
- gapit/cmd_db_setup.py +87 -0
- gapit/cmd_screen.py +474 -0
- gapit/cmd_screen_positionals.py +43 -0
- gapit/cmd_screen_reads_args.py +102 -0
- gapit/cmd_summary.py +130 -0
- gapit/cmd_typing.py +125 -0
- gapit/config.py +111 -0
- gapit/data/dbs/cje_cps/bundled.json +50 -0
- gapit/data/dbs/cje_cps/floors.json +8 -0
- gapit/data/dbs/cje_cps/screen.json +16 -0
- gapit/data/dbs/cje_cps/sequences +324 -0
- gapit/data/dbs/cje_cps/typing.json +1828 -0
- gapit/data/dbs/eco_dec/bundled.json +29 -0
- gapit/data/dbs/eco_dec/floors.json +1 -0
- gapit/data/dbs/eco_dec/records.jsonl +16 -0
- gapit/data/dbs/eco_dec/sequences +347 -0
- gapit/data/dbs/eco_dec/typing.json +125 -0
- gapit/data/dbs/lmo_sero/bundled.json +25 -0
- gapit/data/dbs/lmo_sero/sequences +58 -0
- gapit/data/dbs/lmo_sero/typing.json +72 -0
- gapit/data/dbs/lpn_sero/bundled.json +27 -0
- gapit/data/dbs/lpn_sero/cluster.json +4 -0
- gapit/data/dbs/lpn_sero/features.json +5381 -0
- gapit/data/dbs/lpn_sero/sequences +13403 -0
- gapit/data/dbs/lpn_sero/typing.json +337 -0
- gapit/data/dbs/lpn_vf/bundled.json +27 -0
- gapit/data/dbs/lpn_vf/sequences +2574 -0
- gapit/data/dbs/pau_sero/bundled.json +33 -0
- gapit/data/dbs/pau_sero/cluster.json +4 -0
- gapit/data/dbs/pau_sero/features.json +1636 -0
- gapit/data/dbs/pau_sero/sequences +4015 -0
- gapit/data/dbs/pau_sero/typing.json +352 -0
- gapit/data/dbs/sag_sero/bundled.json +34 -0
- gapit/data/dbs/sag_sero/floors.json +7 -0
- gapit/data/dbs/sag_sero/screen.json +16 -0
- gapit/data/dbs/sag_sero/sequences +1548 -0
- gapit/data/dbs/sag_sero/typing.json +234 -0
- gapit/data/dbs/shi_sero/bundled.json +29 -0
- gapit/data/dbs/shi_sero/markers.json +589 -0
- gapit/data/dbs/shi_sero/screen.json +12 -0
- gapit/data/dbs/shi_sero/sequences +2139 -0
- gapit/data/dbs/shi_sero/serotype.json +1239 -0
- gapit/data/dbs/str_lancefield/bundled.json +40 -0
- gapit/data/dbs/str_lancefield/floors.json +7 -0
- gapit/data/dbs/str_lancefield/screen.json +16 -0
- gapit/data/dbs/str_lancefield/sequences +853 -0
- gapit/data/dbs/str_lancefield/typing.json +193 -0
- gapit/data/dbs/vch_oagc/bundled.json +37 -0
- gapit/data/dbs/vch_oagc/features.json +40976 -0
- gapit/data/dbs/vch_oagc/sequences +105002 -0
- gapit/data/dbs/vch_oagc/typing.json +4593 -0
- gapit/data/dbs/vch_vf/bundled.json +37 -0
- gapit/data/dbs/vch_vf/screen.json +11 -0
- gapit/data/dbs/vch_vf/sequences +159 -0
- gapit/data/dbs/vpa_ok/bundled.json +30 -0
- gapit/data/dbs/vpa_ok/cluster.json +12 -0
- gapit/data/dbs/vpa_ok/features.json +47815 -0
- gapit/data/dbs/vpa_ok/gene_families.json +25451 -0
- gapit/data/dbs/vpa_ok/sequences +115266 -0
- gapit/data/dbs/vpa_ok/typing.json +4261 -0
- gapit/data/dbs/vpa_pai/bundled.json +29 -0
- gapit/data/dbs/vpa_pai/features.json +1937 -0
- gapit/data/dbs/vpa_pai/sequences +5419 -0
- gapit/db.py +363 -0
- gapit/db_build_meta.py +98 -0
- gapit/db_build_ops.py +316 -0
- gapit/db_info_manual.py +250 -0
- gapit/db_info_models.py +45 -0
- gapit/db_info_ops.py +266 -0
- gapit/db_ops.py +433 -0
- gapit/db_query_ops.py +231 -0
- gapit/db_rename_ops.py +166 -0
- gapit/db_setup_ops.py +144 -0
- gapit/db_txn.py +125 -0
- gapit/dbbuild.py +352 -0
- gapit/dbcodec.py +117 -0
- gapit/dispatch.py +108 -0
- gapit/engines.py +31 -0
- gapit/errors.py +177 -0
- gapit/fasta.py +123 -0
- gapit/formats/__init__.py +1 -0
- gapit/formats/cluster.py +250 -0
- gapit/formats/cluster2.py +252 -0
- gapit/formats/cluster_md.py +264 -0
- gapit/formats/csvutil.py +38 -0
- gapit/formats/json.py +163 -0
- gapit/formats/locus_map.py +76 -0
- gapit/formats/md.py +276 -0
- gapit/formats/reads_json.py +200 -0
- gapit/formats/reads_tsv.py +95 -0
- gapit/formats/schemas.py +51 -0
- gapit/formats/summary.py +118 -0
- gapit/formats/tsv.py +70 -0
- gapit/formats/typing_result.py +159 -0
- gapit/fragments.py +143 -0
- gapit/gbfeatures.py +329 -0
- gapit/gene_families.py +263 -0
- gapit/gene_floors.py +155 -0
- gapit/gffparse.py +211 -0
- gapit/gseek_engine.py +690 -0
- gapit/hits.py +155 -0
- gapit/kmerdb.py +258 -0
- gapit/locus_map.py +207 -0
- gapit/locus_map_models.py +79 -0
- gapit/mcp.py +161 -0
- gapit/mcp_schemas.py +170 -0
- gapit/mcp_tools.py +374 -0
- gapit/minimap.py +20 -0
- gapit/minimap2_run.py +211 -0
- gapit/paf.py +144 -0
- gapit/proctools.py +24 -0
- gapit/progress.py +180 -0
- gapit/providers/__init__.py +69 -0
- gapit/providers/ab_k.py +66 -0
- gapit/providers/ab_o.py +67 -0
- gapit/providers/argannot.py +115 -0
- gapit/providers/assets/vch_oagc/control.gbk +41 -0
- gapit/providers/assets/vch_oagc/typing.json +4603 -0
- gapit/providers/bacmet2.py +81 -0
- gapit/providers/card.py +179 -0
- gapit/providers/cluster_common.py +177 -0
- gapit/providers/common.py +309 -0
- gapit/providers/data/spn_sero_kmer.json +600 -0
- gapit/providers/data/spn_sero_typing.json +2910 -0
- gapit/providers/data/spy_emm_typing.json +9181 -0
- gapit/providers/data/tuf.fna +2 -0
- gapit/providers/ecoh.py +83 -0
- gapit/providers/ecoli_kps.py +73 -0
- gapit/providers/ecoli_vf.py +99 -0
- gapit/providers/emmtyper.py +137 -0
- gapit/providers/kosc_k.py +73 -0
- gapit/providers/kosc_o.py +67 -0
- gapit/providers/kpsc_k.py +65 -0
- gapit/providers/kpsc_o.py +65 -0
- gapit/providers/megares.py +94 -0
- gapit/providers/ncbi.py +125 -0
- gapit/providers/nme_sero.py +199 -0
- gapit/providers/plasmidfinder.py +90 -0
- gapit/providers/resfinder.py +143 -0
- gapit/providers/seroba.py +148 -0
- gapit/providers/staphscope.py +173 -0
- gapit/providers/upec_expec_vf.py +100 -0
- gapit/providers/vch_oagc.py +285 -0
- gapit/providers/vfdb.py +120 -0
- gapit/providers/victors.py +130 -0
- gapit/py.typed +0 -0
- gapit/readpairs.py +110 -0
- gapit/reads.py +278 -0
- gapit/records.py +186 -0
- gapit/report.py +30 -0
- gapit/screen_defaults.py +155 -0
- gapit/screening.py +271 -0
- gapit/screening_batch.py +74 -0
- gapit/screening_cluster.py +348 -0
- gapit/screening_kmer.py +136 -0
- gapit/screening_reads.py +324 -0
- gapit/screening_reads_positional.py +212 -0
- gapit/seqconvert.py +203 -0
- gapit/serotype_engine.py +125 -0
- gapit/serotype_models.py +150 -0
- gapit/summary.py +205 -0
- gapit/typing_cluster2.py +137 -0
- gapit/typing_cluster_input.py +115 -0
- gapit/typing_decide.py +153 -0
- gapit/typing_engine.py +304 -0
- gapit/typing_gene.py +134 -0
- gapit/typing_input.py +142 -0
- gapit/typing_models.py +327 -0
- gapit/typing_ops.py +155 -0
- gapit/typing_results.py +161 -0
- gapit/typing_rules.py +212 -0
- gapit/typing_table.py +256 -0
- gspec-0.8.0.dist-info/METADATA +124 -0
- gspec-0.8.0.dist-info/RECORD +194 -0
- gspec-0.8.0.dist-info/WHEEL +4 -0
- gspec-0.8.0.dist-info/entry_points.txt +5 -0
- gspec-0.8.0.dist-info/licenses/LICENSE +21 -0
gapit/__init__.py
ADDED
gapit/blast.py
ADDED
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
"""BLAST invocation, tabular parsing, and the screening pipeline (SPEC.md §3)."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import shlex
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel
|
|
12
|
+
|
|
13
|
+
from gapit.db import Database
|
|
14
|
+
from gapit.errors import DependencyError, GapitError, InputError
|
|
15
|
+
from gapit.fragments import merge_fragment_hits
|
|
16
|
+
from gapit.gene_floors import apply_hit_floors, gene_floors
|
|
17
|
+
from gapit.gseek_engine import engine, run_blast_gseek
|
|
18
|
+
from gapit.hits import process_rows, process_rows_with_partials
|
|
19
|
+
from gapit.report import Report, ScreeningParams
|
|
20
|
+
from gapit.seqconvert import SeqFormat, detect_format, to_fasta_lines
|
|
21
|
+
|
|
22
|
+
BLAST_FIELDS = [
|
|
23
|
+
"qseqid",
|
|
24
|
+
"qstart",
|
|
25
|
+
"qend",
|
|
26
|
+
"qlen",
|
|
27
|
+
"sseqid",
|
|
28
|
+
"sstart",
|
|
29
|
+
"send",
|
|
30
|
+
"slen",
|
|
31
|
+
"sstrand",
|
|
32
|
+
"evalue",
|
|
33
|
+
"length",
|
|
34
|
+
"pident",
|
|
35
|
+
"gaps",
|
|
36
|
+
"gapopen",
|
|
37
|
+
"stitle",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
_OUTFMT = "6 " + " ".join(BLAST_FIELDS)
|
|
41
|
+
_MIN_BLAST_VERSION = (2, 2, 30)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class BlastRow(BaseModel, frozen=True):
|
|
45
|
+
"""One outfmt-6 row, typed at the boundary (SPEC.md §3 field order)."""
|
|
46
|
+
|
|
47
|
+
qseqid: str
|
|
48
|
+
qstart: int
|
|
49
|
+
qend: int
|
|
50
|
+
qlen: int
|
|
51
|
+
sseqid: str
|
|
52
|
+
sstart: int
|
|
53
|
+
send: int
|
|
54
|
+
slen: int
|
|
55
|
+
sstrand: str
|
|
56
|
+
evalue: float
|
|
57
|
+
length: int
|
|
58
|
+
pident: float
|
|
59
|
+
gaps: int
|
|
60
|
+
gapopen: int
|
|
61
|
+
stitle: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def parse_blast_row(line: str) -> BlastRow:
|
|
65
|
+
"""Parse one tab-delimited outfmt-6 line; a row with != 15 columns is a
|
|
66
|
+
hard error (upstream wording)."""
|
|
67
|
+
fields = line.split("\t")
|
|
68
|
+
if len(fields) != 15:
|
|
69
|
+
raise GapitError("can not find sequence data", code="BLAST_PARSE_FAILED")
|
|
70
|
+
try:
|
|
71
|
+
return BlastRow(
|
|
72
|
+
qseqid=fields[0],
|
|
73
|
+
qstart=int(fields[1]),
|
|
74
|
+
qend=int(fields[2]),
|
|
75
|
+
qlen=int(fields[3]),
|
|
76
|
+
sseqid=fields[4],
|
|
77
|
+
sstart=int(fields[5]),
|
|
78
|
+
send=int(fields[6]),
|
|
79
|
+
slen=int(fields[7]),
|
|
80
|
+
sstrand=fields[8],
|
|
81
|
+
evalue=float(fields[9]),
|
|
82
|
+
length=int(fields[10]),
|
|
83
|
+
pident=float(fields[11]),
|
|
84
|
+
gaps=int(fields[12]),
|
|
85
|
+
gapopen=int(fields[13]),
|
|
86
|
+
stitle=fields[14],
|
|
87
|
+
)
|
|
88
|
+
except (TypeError, ValueError) as exc:
|
|
89
|
+
raise GapitError(
|
|
90
|
+
f"can not parse BLAST row (numeric field): {line!r}",
|
|
91
|
+
code="BLAST_PARSE_FAILED",
|
|
92
|
+
) from exc
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def ensure_blast() -> None:
|
|
96
|
+
"""Require blastn >= 2.2.30 on PATH (SPEC.md §3 version gate)."""
|
|
97
|
+
try:
|
|
98
|
+
result = subprocess.run(["blastn", "-version"], check=False, capture_output=True, text=True)
|
|
99
|
+
except FileNotFoundError as exc:
|
|
100
|
+
raise DependencyError(
|
|
101
|
+
"required binary not found on PATH: blastn",
|
|
102
|
+
code="MISSING_DEPENDENCY",
|
|
103
|
+
context={"binary": "blastn"},
|
|
104
|
+
) from exc
|
|
105
|
+
if result.returncode != 0:
|
|
106
|
+
raise DependencyError(
|
|
107
|
+
f"blastn -version failed: {result.stderr.strip()}",
|
|
108
|
+
code="BLAST_VERSION_CHECK_FAILED",
|
|
109
|
+
)
|
|
110
|
+
match = re.search(r"blastn:\s*(\d+)\.(\d+)\.(\d+)", result.stdout)
|
|
111
|
+
if match is None:
|
|
112
|
+
raise DependencyError(
|
|
113
|
+
f"could not parse blastn version from: {result.stdout!r}",
|
|
114
|
+
code="BLAST_VERSION_CHECK_FAILED",
|
|
115
|
+
)
|
|
116
|
+
major, minor, revision = (int(part) for part in match.groups())
|
|
117
|
+
if (major, minor, revision) < _MIN_BLAST_VERSION:
|
|
118
|
+
raise DependencyError(
|
|
119
|
+
f"blastn {major}.{minor}.{revision} is older than 2.2.30",
|
|
120
|
+
code="BLAST_VERSION_TOO_OLD",
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _normalize(query: Path) -> tuple[str, SeqFormat]:
|
|
125
|
+
"""Convert one input file to FASTA text in-process (seqconvert), wrapping
|
|
126
|
+
any parser failure in the blast-path error shape
|
|
127
|
+
``invalid input file {query}: <reason>``.
|
|
128
|
+
|
|
129
|
+
The whole normalized FASTA is held in memory and fed to blast via
|
|
130
|
+
``input=`` — genome-scale inputs are a few MB, a deliberate trade-off:
|
|
131
|
+
``subprocess.run``'s communicate() owns stdin/stdout concurrency, so the
|
|
132
|
+
retired ``any2fasta | blastn`` Popen chain needs no stderr-drain thread
|
|
133
|
+
here anymore.
|
|
134
|
+
"""
|
|
135
|
+
try:
|
|
136
|
+
fmt = detect_format(query)
|
|
137
|
+
text = "".join(to_fasta_lines(query, fmt))
|
|
138
|
+
except InputError as exc:
|
|
139
|
+
raise InputError(
|
|
140
|
+
f"invalid input file {query}: {exc}",
|
|
141
|
+
code="INVALID_INPUT",
|
|
142
|
+
context={"file": str(query)},
|
|
143
|
+
) from exc
|
|
144
|
+
return text, fmt
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _pipeline(fasta_text: str, argv: list[str]) -> str:
|
|
148
|
+
"""``<argv>`` reading the normalized FASTA on stdin; returns stdout as text."""
|
|
149
|
+
try:
|
|
150
|
+
result = subprocess.run(argv, input=fasta_text, capture_output=True, text=True)
|
|
151
|
+
except FileNotFoundError as exc:
|
|
152
|
+
raise DependencyError(
|
|
153
|
+
f"required binary not found on PATH: {argv[0]}",
|
|
154
|
+
code="MISSING_DEPENDENCY",
|
|
155
|
+
context={"binary": argv[0]},
|
|
156
|
+
) from exc
|
|
157
|
+
if result.returncode != 0:
|
|
158
|
+
raise GapitError(
|
|
159
|
+
f"{argv[0]} failed: {result.stderr.strip()}",
|
|
160
|
+
code="BLAST_FAILED",
|
|
161
|
+
context={"binary": argv[0]},
|
|
162
|
+
)
|
|
163
|
+
return result.stdout
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def run_blastn(
|
|
167
|
+
query: Path,
|
|
168
|
+
database: Database,
|
|
169
|
+
params: ScreeningParams,
|
|
170
|
+
*,
|
|
171
|
+
dbtype: Literal["nucl", "prot"],
|
|
172
|
+
debug: bool = False,
|
|
173
|
+
) -> list[BlastRow]:
|
|
174
|
+
"""Run the normalize (native seqconvert) -> blastn/blastx pipeline for one
|
|
175
|
+
query file.
|
|
176
|
+
|
|
177
|
+
The database must already be indexed; protein databases switch to blastx
|
|
178
|
+
without -perc_identity (upstream quirk, minid silently ignored). With
|
|
179
|
+
``debug``, echo the normalization step and the exact blast argv to stderr
|
|
180
|
+
(abricate --debug parity). ``dbtype`` comes from the caller resolving it
|
|
181
|
+
once per run, so dependency/index errors surface before any per-file work.
|
|
182
|
+
"""
|
|
183
|
+
if dbtype == "prot":
|
|
184
|
+
argv = [
|
|
185
|
+
"blastx",
|
|
186
|
+
"-task",
|
|
187
|
+
"blastx-fast",
|
|
188
|
+
"-seg",
|
|
189
|
+
"no",
|
|
190
|
+
"-db",
|
|
191
|
+
str(database.sequences_path),
|
|
192
|
+
"-outfmt",
|
|
193
|
+
_OUTFMT,
|
|
194
|
+
"-num_threads",
|
|
195
|
+
str(params.threads),
|
|
196
|
+
"-evalue",
|
|
197
|
+
"1E-20",
|
|
198
|
+
"-culling_limit",
|
|
199
|
+
"1",
|
|
200
|
+
"-max_target_seqs",
|
|
201
|
+
"10000",
|
|
202
|
+
]
|
|
203
|
+
sys.stderr.write("--minid is not applied to protein databases (abricate parity)\n")
|
|
204
|
+
else:
|
|
205
|
+
argv = [
|
|
206
|
+
"blastn",
|
|
207
|
+
"-task",
|
|
208
|
+
"blastn",
|
|
209
|
+
"-dust",
|
|
210
|
+
"no",
|
|
211
|
+
"-perc_identity",
|
|
212
|
+
str(params.minid),
|
|
213
|
+
"-db",
|
|
214
|
+
str(database.sequences_path),
|
|
215
|
+
"-outfmt",
|
|
216
|
+
_OUTFMT,
|
|
217
|
+
"-num_threads",
|
|
218
|
+
str(params.threads),
|
|
219
|
+
"-evalue",
|
|
220
|
+
"1E-20",
|
|
221
|
+
"-culling_limit",
|
|
222
|
+
"1",
|
|
223
|
+
"-max_target_seqs",
|
|
224
|
+
"10000",
|
|
225
|
+
]
|
|
226
|
+
fasta_text, fmt = _normalize(query)
|
|
227
|
+
switch = engine()
|
|
228
|
+
mode = switch.mode.value
|
|
229
|
+
if debug:
|
|
230
|
+
sys.stderr.write(f"gspec: normalize: {query} ({fmt.value})\n")
|
|
231
|
+
if mode in {"legacy", "shadow"}:
|
|
232
|
+
sys.stderr.write(f"gspec: run: {shlex.join(argv)}\n")
|
|
233
|
+
|
|
234
|
+
def legacy_rows() -> list[BlastRow]:
|
|
235
|
+
output = _pipeline(fasta_text, argv)
|
|
236
|
+
return [parse_blast_row(line) for line in output.splitlines() if line.strip()]
|
|
237
|
+
|
|
238
|
+
def gseek_rows() -> list[BlastRow]:
|
|
239
|
+
return run_blast_gseek(fasta_text, database, params, dbtype=dbtype, debug=debug)
|
|
240
|
+
|
|
241
|
+
return switch.run(legacy_rows, gseek_rows)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def report_from_rows(
|
|
245
|
+
rows: Iterable[BlastRow],
|
|
246
|
+
query: Path,
|
|
247
|
+
database: Database,
|
|
248
|
+
params: ScreeningParams,
|
|
249
|
+
*,
|
|
250
|
+
merge_fragments: bool = False,
|
|
251
|
+
) -> Report:
|
|
252
|
+
"""Fold typed BLAST rows into one sorted Report.
|
|
253
|
+
|
|
254
|
+
Shared by the per-file and batch gseek paths; the caller owns the BLAST
|
|
255
|
+
run and the per-run ``ensure_blast``/dbtype gates.
|
|
256
|
+
"""
|
|
257
|
+
floors = gene_floors(database)
|
|
258
|
+
if merge_fragments:
|
|
259
|
+
hits, partials = process_rows_with_partials(
|
|
260
|
+
rows, mincov=params.mincov, default_db=params.db
|
|
261
|
+
)
|
|
262
|
+
if floors is not None:
|
|
263
|
+
hits = apply_hit_floors(hits, floors)
|
|
264
|
+
partials = apply_hit_floors(partials, floors)
|
|
265
|
+
hits = merge_fragment_hits(hits, partials, mincov=params.mincov)
|
|
266
|
+
else:
|
|
267
|
+
hits = process_rows(rows, mincov=params.mincov, default_db=params.db)
|
|
268
|
+
if floors is not None:
|
|
269
|
+
hits = apply_hit_floors(hits, floors)
|
|
270
|
+
ordered = sorted(hits, key=lambda hit: (hit.sequence, hit.start))
|
|
271
|
+
return Report(file=str(query), hits=tuple(ordered))
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def screen_file(
|
|
275
|
+
query: Path,
|
|
276
|
+
database: Database,
|
|
277
|
+
params: ScreeningParams,
|
|
278
|
+
*,
|
|
279
|
+
dbtype: Literal["nucl", "prot"],
|
|
280
|
+
debug: bool = False,
|
|
281
|
+
merge_fragments: bool = False,
|
|
282
|
+
) -> Report:
|
|
283
|
+
"""Screen one input file against one database into a sorted Report.
|
|
284
|
+
|
|
285
|
+
The caller owns the per-run gates (``ensure_blast`` and the one-shot
|
|
286
|
+
``dbtype`` resolution) so a multi-file run pays each probe once. With
|
|
287
|
+
``merge_fragments`` (opt-in gspec extension), sub-mincov dedup survivors
|
|
288
|
+
are folded into one hit per gene when their union coverage reaches mincov.
|
|
289
|
+
"""
|
|
290
|
+
rows = run_blastn(query, database, params, dbtype=dbtype, debug=debug)
|
|
291
|
+
return report_from_rows(rows, query, database, params, merge_fragments=merge_fragments)
|
gapit/bundled.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"""Bundled databases: license-clean content shipped inside the wheel.
|
|
2
|
+
|
|
3
|
+
A bundled database is a directory ``gapit/data/dbs/<name>/`` holding
|
|
4
|
+
``sequences``, a ``bundled.json`` metadata file, and kind-dependent
|
|
5
|
+
companions: gene bundles an optional ``typing.json`` and ``floors.json``
|
|
6
|
+
(gapit.floors/1), cluster bundles a REQUIRED ``features.json``
|
|
7
|
+
(gapit.features/1) plus their typing document. Materialization replays the
|
|
8
|
+
standard build pipelines into the user's datadir — the gene path through
|
|
9
|
+
:func:`gapit.db_build_ops.perform_build`, the cluster path through
|
|
10
|
+
:func:`gapit.bundled_cluster.materialize_cluster_bundle` (which rebuilds
|
|
11
|
+
from the committed locus FASTA + resolved feature table via
|
|
12
|
+
:func:`gapit.clusterbuild.build_cluster_database`) — deterministic, zero
|
|
13
|
+
network, so a bundled database is ready at install time with no
|
|
14
|
+
``db fetch``.
|
|
15
|
+
|
|
16
|
+
Only gspec-curated content with clean provenance may live here (the wheel
|
|
17
|
+
is MIT): our own panels and public-domain / permissive sources, never GPL,
|
|
18
|
+
non-commercial, or third-party provider snapshots whose redistribution terms
|
|
19
|
+
are uncertain. Every provider database — ncbi, card, ecoh, resfinder, ... —
|
|
20
|
+
is fetch-on-demand only, even when its upstream license would permit a
|
|
21
|
+
snapshot. The typed cluster snapshot vpa_ok is a pure built-in (no provider
|
|
22
|
+
behind it) carrying the resolved gapit.features/1 table a completed /2
|
|
23
|
+
build produced; a refresh means a new snapshot riding a wheel release.
|
|
24
|
+
The gene build pipeline re-headers any bundled FASTA into gapit/v1 like a
|
|
25
|
+
custom build, so header styles round-trip.
|
|
26
|
+
|
|
27
|
+
Bundling is the exception, not the pattern, and every new bundle clears
|
|
28
|
+
two gates in order (the full policy lives in AGENTS.md §6): the license
|
|
29
|
+
gate above, then the size gate — the whole bundled root stays within a
|
|
30
|
+
30 MiB wheel budget, mechanically enforced by tests/test_bundled.py.
|
|
31
|
+
Past the budget, split a separate data wheel or keep the database
|
|
32
|
+
fetch-on-demand instead of inflating the ceiling.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
import shutil
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
from typing import Literal
|
|
38
|
+
|
|
39
|
+
from pydantic import BaseModel
|
|
40
|
+
|
|
41
|
+
from gapit import config
|
|
42
|
+
from gapit.bundled_cluster import materialize_cluster_bundle
|
|
43
|
+
from gapit.db_build_ops import perform_build
|
|
44
|
+
from gapit.db_info_models import DbGuidance
|
|
45
|
+
from gapit.errors import DatabaseError
|
|
46
|
+
from gapit.fasta import iter_fasta
|
|
47
|
+
from gapit.proctools import note
|
|
48
|
+
|
|
49
|
+
BUNDLED_ROOT = Path(__file__).resolve().parent / "data" / "dbs"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class BundledMetadata(BaseModel, frozen=True):
|
|
53
|
+
"""Parsed ``bundled.json`` — the listing-facing facts for one bundle.
|
|
54
|
+
|
|
55
|
+
``snapshotted`` is the ISO date the wheel copy was produced from
|
|
56
|
+
upstream (provenance for humans; ``db fetch`` refreshes past it).
|
|
57
|
+
``kind`` routes materialization (gene vs cluster pipeline; cluster
|
|
58
|
+
bundles additionally require ``features.json``). ``license`` and
|
|
59
|
+
``note`` carry content provenance into the materialized manifest
|
|
60
|
+
(cluster bundles); they default to empty so the gene bundles'
|
|
61
|
+
metadata stays minimal. ``guidance`` is what `gspec db info` reports:
|
|
62
|
+
the literature references, the intended usage recipe, and the caveats
|
|
63
|
+
a caller must not skip.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
name: str
|
|
67
|
+
description: str
|
|
68
|
+
vendor: str
|
|
69
|
+
dbtype: Literal["nucl", "prot"]
|
|
70
|
+
kind: Literal["gene", "cluster"] = "gene"
|
|
71
|
+
snapshotted: str
|
|
72
|
+
license: str = ""
|
|
73
|
+
note: str = ""
|
|
74
|
+
guidance: DbGuidance = DbGuidance()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class BundledDatabase(BaseModel, frozen=True):
|
|
78
|
+
"""One bundled database directory plus its parsed metadata."""
|
|
79
|
+
|
|
80
|
+
metadata: BundledMetadata
|
|
81
|
+
path: Path
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def name(self) -> str:
|
|
85
|
+
return self.metadata.name
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def sequences_path(self) -> Path:
|
|
89
|
+
return self.path / "sequences"
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def typing_path(self) -> Path | None:
|
|
93
|
+
"""The bundled typing spec, when the database ships one."""
|
|
94
|
+
typing = self.path / "typing.json"
|
|
95
|
+
return typing if typing.is_file() else None
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def floors_path(self) -> Path | None:
|
|
99
|
+
"""The bundled gapit.floors/1 sidecar, when the database ships one."""
|
|
100
|
+
floors = self.path / "floors.json"
|
|
101
|
+
return floors if floors.is_file() else None
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def screen_path(self) -> Path | None:
|
|
105
|
+
"""The bundled gapit.screen/1 defaults, when the database ships one."""
|
|
106
|
+
screen = self.path / "screen.json"
|
|
107
|
+
return screen if screen.is_file() else None
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def features_path(self) -> Path | None:
|
|
111
|
+
"""The bundled gapit.features/1 table — required (and checked at
|
|
112
|
+
discovery) for cluster bundles, absent from gene bundles."""
|
|
113
|
+
features = self.path / "features.json"
|
|
114
|
+
return features if features.is_file() else None
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def profile_path(self) -> Path | None:
|
|
118
|
+
"""The optional gapit.cluster_profile/1 sidecar for cluster bundles."""
|
|
119
|
+
profile = self.path / "cluster.json"
|
|
120
|
+
return profile if profile.is_file() else None
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def gene_families_path(self) -> Path | None:
|
|
124
|
+
"""The optional gapit.gene_families/1 sidecar for cluster bundles."""
|
|
125
|
+
families = self.path / "gene_families.json"
|
|
126
|
+
return families if families.is_file() else None
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def markers_path(self) -> Path | None:
|
|
130
|
+
"""The optional marker metadata sidecar for a gene bundle."""
|
|
131
|
+
markers = self.path / "markers.json"
|
|
132
|
+
return markers if markers.is_file() else None
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def serotype_path(self) -> Path | None:
|
|
136
|
+
"""The optional gapit.serotype/1 profile sidecar for a gene bundle."""
|
|
137
|
+
serotype = self.path / "serotype.json"
|
|
138
|
+
return serotype if serotype.is_file() else None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def bundled_databases() -> list[BundledDatabase]:
|
|
142
|
+
"""Every bundled database, sorted by name. A directory qualifies when it
|
|
143
|
+
carries both ``sequences`` and ``bundled.json`` — plus ``features.json``
|
|
144
|
+
when the metadata declares ``kind: cluster`` (a cluster bundle without
|
|
145
|
+
its feature table cannot materialize; typed error, no silent
|
|
146
|
+
degradation). Malformed metadata propagates, and a wheel without the
|
|
147
|
+
data dir lists nothing."""
|
|
148
|
+
if not BUNDLED_ROOT.is_dir():
|
|
149
|
+
return []
|
|
150
|
+
databases: list[BundledDatabase] = []
|
|
151
|
+
for child in sorted(BUNDLED_ROOT.iterdir()):
|
|
152
|
+
metadata_path = child / "bundled.json"
|
|
153
|
+
if not (child.is_dir() and (child / "sequences").is_file() and metadata_path.is_file()):
|
|
154
|
+
continue
|
|
155
|
+
try:
|
|
156
|
+
metadata = BundledMetadata.model_validate_json(
|
|
157
|
+
metadata_path.read_text(encoding="utf-8")
|
|
158
|
+
)
|
|
159
|
+
except (OSError, UnicodeDecodeError, ValueError) as exc:
|
|
160
|
+
raise DatabaseError(
|
|
161
|
+
f"bundled metadata is malformed: {metadata_path}: {exc}",
|
|
162
|
+
code="BUNDLED_MALFORMED",
|
|
163
|
+
context={"db": child.name, "file": str(metadata_path)},
|
|
164
|
+
) from exc
|
|
165
|
+
if metadata.kind == "cluster" and not (child / "features.json").is_file():
|
|
166
|
+
raise DatabaseError(
|
|
167
|
+
f"bundled cluster database {metadata.name} lacks features.json",
|
|
168
|
+
code="BUNDLED_MALFORMED",
|
|
169
|
+
context={"db": metadata.name},
|
|
170
|
+
)
|
|
171
|
+
databases.append(BundledDatabase(metadata=metadata, path=child))
|
|
172
|
+
return databases
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def bundled_names() -> list[str]:
|
|
176
|
+
"""Sorted names of every bundled database."""
|
|
177
|
+
return [database.name for database in bundled_databases()]
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def find_bundled(name: str) -> BundledDatabase | None:
|
|
181
|
+
"""The bundled database called ``name``, or None when none ships."""
|
|
182
|
+
return next((database for database in bundled_databases() if database.name == name), None)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def materialize_bundled(name: str, datadir: Path, *, quiet: bool = True) -> None:
|
|
186
|
+
"""Build a bundled database into ``<datadir>/name`` via its kind's
|
|
187
|
+
standard pipeline (records.jsonl/sequences for gene bundles, locus
|
|
188
|
+
FASTA + features.json for cluster bundles; BLAST index, typing copy,
|
|
189
|
+
manifest stamped ``source: "bundled"`` — written last, as always).
|
|
190
|
+
Zero network: every input travels inside the wheel.
|
|
191
|
+
|
|
192
|
+
An already-materialized database (manifest present) is left untouched —
|
|
193
|
+
callers short-circuit on the manifest, and each path's
|
|
194
|
+
DB_ALREADY_EXISTS guard turns a racing double build into a typed error.
|
|
195
|
+
"""
|
|
196
|
+
bundled = find_bundled(name)
|
|
197
|
+
if bundled is None:
|
|
198
|
+
raise DatabaseError(
|
|
199
|
+
f"unknown bundled database: {name}",
|
|
200
|
+
code="BUNDLED_NOT_FOUND",
|
|
201
|
+
context={"db": name},
|
|
202
|
+
)
|
|
203
|
+
if bundled.metadata.kind == "cluster":
|
|
204
|
+
materialize_cluster_bundle(bundled, datadir, quiet=quiet)
|
|
205
|
+
return
|
|
206
|
+
n_records = sum(1 for _ in iter_fasta(bundled.sequences_path))
|
|
207
|
+
note(quiet, f"materializing bundled database {name} ({n_records} records) into {datadir}")
|
|
208
|
+
perform_build(
|
|
209
|
+
name,
|
|
210
|
+
bundled.sequences_path,
|
|
211
|
+
None,
|
|
212
|
+
None,
|
|
213
|
+
bundled.metadata.description,
|
|
214
|
+
datadir,
|
|
215
|
+
force=False,
|
|
216
|
+
warn=lambda message: note(quiet, message),
|
|
217
|
+
quiet=True,
|
|
218
|
+
typing=bundled.typing_path,
|
|
219
|
+
floors=bundled.floors_path,
|
|
220
|
+
screen=bundled.screen_path,
|
|
221
|
+
source="bundled",
|
|
222
|
+
)
|
|
223
|
+
if bundled.markers_path is not None:
|
|
224
|
+
shutil.copyfile(bundled.markers_path, datadir / name / "markers.json")
|
|
225
|
+
if bundled.serotype_path is not None:
|
|
226
|
+
shutil.copyfile(bundled.serotype_path, datadir / name / "serotype.json")
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def resolve_screen_datadir(cli_value: Path | None, db_name: str) -> Path:
|
|
230
|
+
"""``resolve_datadir`` for the screening lookup — except that a MISSING
|
|
231
|
+
datadir is bootstrapped when (and only when) the requested database is
|
|
232
|
+
bundled: the zero-network first-use path must work on a fresh machine
|
|
233
|
+
where ``~/.local/share/gspec/db`` does not exist yet. Every other
|
|
234
|
+
missing-datadir case keeps the typed DATADIR_NOT_FOUND error."""
|
|
235
|
+
try:
|
|
236
|
+
return config.resolve_datadir(cli_value)
|
|
237
|
+
except DatabaseError as exc:
|
|
238
|
+
if exc.code != "DATADIR_NOT_FOUND" or find_bundled(db_name) is None:
|
|
239
|
+
raise
|
|
240
|
+
return config.ensure_datadir(cli_value)
|
gapit/bundled_cluster.py
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""Kind-aware materialization for CLUSTER bundles (the bundled.py sibling).
|
|
2
|
+
|
|
3
|
+
A cluster bundle ships its build INPUTS committed: the locus FASTA plus the
|
|
4
|
+
``gapit.features/1`` table already carrying RESOLVED gene ids (the snapshot
|
|
5
|
+
was produced by a completed gapit.typing/2 build, so its rules reference
|
|
6
|
+
the resolved locus_tag-style ids). Materialization reconstructs
|
|
7
|
+
:class:`LocusSequence` objects from those artifacts and replays
|
|
8
|
+
:func:`gapit.clusterbuild.build_cluster_database` — the entry taking
|
|
9
|
+
pre-parsed loci, the SAME code a provider fetch runs — into the user's
|
|
10
|
+
datadir, deterministic and zero network.
|
|
11
|
+
|
|
12
|
+
Gene-id resolution deliberately does NOT re-run: the bundled feature table
|
|
13
|
+
is already resolved, and re-running would be a no-op at best. The typing
|
|
14
|
+
document IS validated against the reconstructed loci (mirroring the build
|
|
15
|
+
path's pre-write gate), and the screen-time /2 guard re-checks afterwards
|
|
16
|
+
— belt and braces, like the fetch path.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import shutil
|
|
20
|
+
from datetime import UTC, datetime
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import TYPE_CHECKING
|
|
23
|
+
|
|
24
|
+
from pydantic import ValidationError
|
|
25
|
+
|
|
26
|
+
from gapit.clusterbuild import build_cluster_database
|
|
27
|
+
from gapit.db_txn import staged_database
|
|
28
|
+
from gapit.errors import DatabaseError
|
|
29
|
+
from gapit.fasta import iter_fasta
|
|
30
|
+
from gapit.gbfeatures import FeaturesDocument, LocusSequence
|
|
31
|
+
from gapit.proctools import note
|
|
32
|
+
from gapit.typing_models import (
|
|
33
|
+
read_typing_document,
|
|
34
|
+
typing_schema_of,
|
|
35
|
+
validate_references,
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
if TYPE_CHECKING: # bundled.py imports this module at runtime; no cycle.
|
|
39
|
+
from gapit.bundled import BundledDatabase
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _malformed(db_name: str, reason: str) -> DatabaseError:
|
|
43
|
+
"""The broken-wheel-cluster-bundle failure: located, stable code."""
|
|
44
|
+
return DatabaseError(
|
|
45
|
+
f"bundled cluster database {db_name} is malformed ({reason})",
|
|
46
|
+
code="BUNDLED_MALFORMED",
|
|
47
|
+
context={"db": db_name, "reason": reason},
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _reconstruct_loci(bundled: "BundledDatabase") -> list[LocusSequence]:
|
|
52
|
+
"""The bundle's committed artifacts back into LocusSequence objects.
|
|
53
|
+
|
|
54
|
+
features.json supplies the feature tables (loci in document order), the
|
|
55
|
+
FASTA the sequences (one bare-id record per locus); every locus must
|
|
56
|
+
pair with the same-order same-id record — the reconstruction mirror of
|
|
57
|
+
the build self-check, BUNDLED_MALFORMED on any drift."""
|
|
58
|
+
features_path = bundled.path / "features.json"
|
|
59
|
+
try:
|
|
60
|
+
document = FeaturesDocument.model_validate_json(features_path.read_text(encoding="utf-8"))
|
|
61
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
62
|
+
raise _malformed(bundled.name, f"unreadable features.json: {exc}") from exc
|
|
63
|
+
except ValidationError as exc:
|
|
64
|
+
raise _malformed(bundled.name, f"unparseable features.json: {exc}") from exc
|
|
65
|
+
records = list(iter_fasta(bundled.sequences_path))
|
|
66
|
+
if len(records) != len(document.loci):
|
|
67
|
+
raise _malformed(
|
|
68
|
+
bundled.name,
|
|
69
|
+
f"{len(document.loci)} loci but {len(records)} sequence records",
|
|
70
|
+
)
|
|
71
|
+
loci: list[LocusSequence] = []
|
|
72
|
+
for locus, record in zip(document.loci, records, strict=True):
|
|
73
|
+
if locus.id != record.id:
|
|
74
|
+
raise _malformed(bundled.name, f"locus {locus.id!r} paired with record {record.id!r}")
|
|
75
|
+
loci.append(LocusSequence(features=locus, sequence=record.sequence))
|
|
76
|
+
return loci
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def materialize_cluster_bundle(
|
|
80
|
+
bundled: "BundledDatabase", datadir: Path, *, quiet: bool = True
|
|
81
|
+
) -> None:
|
|
82
|
+
"""Build a cluster-kind bundle into ``<datadir>/name`` through the
|
|
83
|
+
cluster pipeline (locus FASTA, features.json, typing copy, BLAST index,
|
|
84
|
+
manifest stamped ``source: "bundled"`` and written last, as always).
|
|
85
|
+
The typing spec validates BEFORE any artifact is written, so a spec
|
|
86
|
+
drifting from the committed table fails with no half-built database;
|
|
87
|
+
an existing manifest is a typed DB_ALREADY_EXISTS (the racing
|
|
88
|
+
double-build guard perform_build gives the gene path)."""
|
|
89
|
+
name = bundled.name
|
|
90
|
+
db_dir = datadir / name
|
|
91
|
+
if (db_dir / "gapit-manifest.json").is_file():
|
|
92
|
+
raise DatabaseError(
|
|
93
|
+
f"won't overwrite existing database {name} (use --force)",
|
|
94
|
+
code="DB_ALREADY_EXISTS",
|
|
95
|
+
context={"db": name},
|
|
96
|
+
)
|
|
97
|
+
n_loci = sum(1 for _ in iter_fasta(bundled.sequences_path))
|
|
98
|
+
note(quiet, f"materializing bundled database {name} ({n_loci} loci) into {datadir}")
|
|
99
|
+
loci = _reconstruct_loci(bundled)
|
|
100
|
+
typing_schema = ""
|
|
101
|
+
if bundled.typing_path is not None:
|
|
102
|
+
document = read_typing_document(bundled.typing_path)
|
|
103
|
+
validate_references(
|
|
104
|
+
document, FeaturesDocument(loci=tuple(locus.features for locus in loci))
|
|
105
|
+
)
|
|
106
|
+
typing_schema = typing_schema_of(bundled.typing_path)
|
|
107
|
+
with staged_database(db_dir) as build_dir:
|
|
108
|
+
if (db_dir / "gapit-manifest.json").is_file():
|
|
109
|
+
raise DatabaseError(
|
|
110
|
+
f"won't overwrite existing database {name} (use --force)",
|
|
111
|
+
code="DB_ALREADY_EXISTS",
|
|
112
|
+
context={"db": name},
|
|
113
|
+
)
|
|
114
|
+
if bundled.profile_path is not None:
|
|
115
|
+
shutil.copyfile(bundled.profile_path, build_dir / "cluster.json")
|
|
116
|
+
if bundled.gene_families_path is not None:
|
|
117
|
+
shutil.copyfile(bundled.gene_families_path, build_dir / "gene_families.json")
|
|
118
|
+
build_cluster_database(
|
|
119
|
+
build_dir,
|
|
120
|
+
name=name,
|
|
121
|
+
loci=loci,
|
|
122
|
+
typing=bundled.typing_path,
|
|
123
|
+
source_urls=("local",),
|
|
124
|
+
fetched_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
125
|
+
typing_schema=typing_schema,
|
|
126
|
+
quiet=True, # the materializing note above is the only stderr line (gene-path parity)
|
|
127
|
+
content_license=bundled.metadata.license,
|
|
128
|
+
content_note=bundled.metadata.note,
|
|
129
|
+
source="bundled",
|
|
130
|
+
)
|