gspec 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. gapit/__init__.py +3 -0
  2. gapit/blast.py +291 -0
  3. gapit/bundled.py +240 -0
  4. gapit/bundled_cluster.py +130 -0
  5. gapit/cli.py +136 -0
  6. gapit/cluster.py +367 -0
  7. gapit/cluster_alignment.py +322 -0
  8. gapit/cluster_math.py +163 -0
  9. gapit/cluster_profile.py +97 -0
  10. gapit/clusterbuild.py +424 -0
  11. gapit/cmd_db.py +203 -0
  12. gapit/cmd_db_build.py +142 -0
  13. gapit/cmd_db_info.py +63 -0
  14. gapit/cmd_db_install.py +126 -0
  15. gapit/cmd_db_outdated.py +50 -0
  16. gapit/cmd_db_rename.py +35 -0
  17. gapit/cmd_db_search.py +82 -0
  18. gapit/cmd_db_setup.py +87 -0
  19. gapit/cmd_screen.py +474 -0
  20. gapit/cmd_screen_positionals.py +43 -0
  21. gapit/cmd_screen_reads_args.py +102 -0
  22. gapit/cmd_summary.py +130 -0
  23. gapit/cmd_typing.py +125 -0
  24. gapit/config.py +111 -0
  25. gapit/data/dbs/cje_cps/bundled.json +50 -0
  26. gapit/data/dbs/cje_cps/floors.json +8 -0
  27. gapit/data/dbs/cje_cps/screen.json +16 -0
  28. gapit/data/dbs/cje_cps/sequences +324 -0
  29. gapit/data/dbs/cje_cps/typing.json +1828 -0
  30. gapit/data/dbs/eco_dec/bundled.json +29 -0
  31. gapit/data/dbs/eco_dec/floors.json +1 -0
  32. gapit/data/dbs/eco_dec/records.jsonl +16 -0
  33. gapit/data/dbs/eco_dec/sequences +347 -0
  34. gapit/data/dbs/eco_dec/typing.json +125 -0
  35. gapit/data/dbs/lmo_sero/bundled.json +25 -0
  36. gapit/data/dbs/lmo_sero/sequences +58 -0
  37. gapit/data/dbs/lmo_sero/typing.json +72 -0
  38. gapit/data/dbs/lpn_sero/bundled.json +27 -0
  39. gapit/data/dbs/lpn_sero/cluster.json +4 -0
  40. gapit/data/dbs/lpn_sero/features.json +5381 -0
  41. gapit/data/dbs/lpn_sero/sequences +13403 -0
  42. gapit/data/dbs/lpn_sero/typing.json +337 -0
  43. gapit/data/dbs/lpn_vf/bundled.json +27 -0
  44. gapit/data/dbs/lpn_vf/sequences +2574 -0
  45. gapit/data/dbs/pau_sero/bundled.json +33 -0
  46. gapit/data/dbs/pau_sero/cluster.json +4 -0
  47. gapit/data/dbs/pau_sero/features.json +1636 -0
  48. gapit/data/dbs/pau_sero/sequences +4015 -0
  49. gapit/data/dbs/pau_sero/typing.json +352 -0
  50. gapit/data/dbs/sag_sero/bundled.json +34 -0
  51. gapit/data/dbs/sag_sero/floors.json +7 -0
  52. gapit/data/dbs/sag_sero/screen.json +16 -0
  53. gapit/data/dbs/sag_sero/sequences +1548 -0
  54. gapit/data/dbs/sag_sero/typing.json +234 -0
  55. gapit/data/dbs/shi_sero/bundled.json +29 -0
  56. gapit/data/dbs/shi_sero/markers.json +589 -0
  57. gapit/data/dbs/shi_sero/screen.json +12 -0
  58. gapit/data/dbs/shi_sero/sequences +2139 -0
  59. gapit/data/dbs/shi_sero/serotype.json +1239 -0
  60. gapit/data/dbs/str_lancefield/bundled.json +40 -0
  61. gapit/data/dbs/str_lancefield/floors.json +7 -0
  62. gapit/data/dbs/str_lancefield/screen.json +16 -0
  63. gapit/data/dbs/str_lancefield/sequences +853 -0
  64. gapit/data/dbs/str_lancefield/typing.json +193 -0
  65. gapit/data/dbs/vch_oagc/bundled.json +37 -0
  66. gapit/data/dbs/vch_oagc/features.json +40976 -0
  67. gapit/data/dbs/vch_oagc/sequences +105002 -0
  68. gapit/data/dbs/vch_oagc/typing.json +4593 -0
  69. gapit/data/dbs/vch_vf/bundled.json +37 -0
  70. gapit/data/dbs/vch_vf/screen.json +11 -0
  71. gapit/data/dbs/vch_vf/sequences +159 -0
  72. gapit/data/dbs/vpa_ok/bundled.json +30 -0
  73. gapit/data/dbs/vpa_ok/cluster.json +12 -0
  74. gapit/data/dbs/vpa_ok/features.json +47815 -0
  75. gapit/data/dbs/vpa_ok/gene_families.json +25451 -0
  76. gapit/data/dbs/vpa_ok/sequences +115266 -0
  77. gapit/data/dbs/vpa_ok/typing.json +4261 -0
  78. gapit/data/dbs/vpa_pai/bundled.json +29 -0
  79. gapit/data/dbs/vpa_pai/features.json +1937 -0
  80. gapit/data/dbs/vpa_pai/sequences +5419 -0
  81. gapit/db.py +363 -0
  82. gapit/db_build_meta.py +98 -0
  83. gapit/db_build_ops.py +316 -0
  84. gapit/db_info_manual.py +250 -0
  85. gapit/db_info_models.py +45 -0
  86. gapit/db_info_ops.py +266 -0
  87. gapit/db_ops.py +433 -0
  88. gapit/db_query_ops.py +231 -0
  89. gapit/db_rename_ops.py +166 -0
  90. gapit/db_setup_ops.py +144 -0
  91. gapit/db_txn.py +125 -0
  92. gapit/dbbuild.py +352 -0
  93. gapit/dbcodec.py +117 -0
  94. gapit/dispatch.py +108 -0
  95. gapit/engines.py +31 -0
  96. gapit/errors.py +177 -0
  97. gapit/fasta.py +123 -0
  98. gapit/formats/__init__.py +1 -0
  99. gapit/formats/cluster.py +250 -0
  100. gapit/formats/cluster2.py +252 -0
  101. gapit/formats/cluster_md.py +264 -0
  102. gapit/formats/csvutil.py +38 -0
  103. gapit/formats/json.py +163 -0
  104. gapit/formats/locus_map.py +76 -0
  105. gapit/formats/md.py +276 -0
  106. gapit/formats/reads_json.py +200 -0
  107. gapit/formats/reads_tsv.py +95 -0
  108. gapit/formats/schemas.py +51 -0
  109. gapit/formats/summary.py +118 -0
  110. gapit/formats/tsv.py +70 -0
  111. gapit/formats/typing_result.py +159 -0
  112. gapit/fragments.py +143 -0
  113. gapit/gbfeatures.py +329 -0
  114. gapit/gene_families.py +263 -0
  115. gapit/gene_floors.py +155 -0
  116. gapit/gffparse.py +211 -0
  117. gapit/gseek_engine.py +690 -0
  118. gapit/hits.py +155 -0
  119. gapit/kmerdb.py +258 -0
  120. gapit/locus_map.py +207 -0
  121. gapit/locus_map_models.py +79 -0
  122. gapit/mcp.py +161 -0
  123. gapit/mcp_schemas.py +170 -0
  124. gapit/mcp_tools.py +374 -0
  125. gapit/minimap.py +20 -0
  126. gapit/minimap2_run.py +211 -0
  127. gapit/paf.py +144 -0
  128. gapit/proctools.py +24 -0
  129. gapit/progress.py +180 -0
  130. gapit/providers/__init__.py +69 -0
  131. gapit/providers/ab_k.py +66 -0
  132. gapit/providers/ab_o.py +67 -0
  133. gapit/providers/argannot.py +115 -0
  134. gapit/providers/assets/vch_oagc/control.gbk +41 -0
  135. gapit/providers/assets/vch_oagc/typing.json +4603 -0
  136. gapit/providers/bacmet2.py +81 -0
  137. gapit/providers/card.py +179 -0
  138. gapit/providers/cluster_common.py +177 -0
  139. gapit/providers/common.py +309 -0
  140. gapit/providers/data/spn_sero_kmer.json +600 -0
  141. gapit/providers/data/spn_sero_typing.json +2910 -0
  142. gapit/providers/data/spy_emm_typing.json +9181 -0
  143. gapit/providers/data/tuf.fna +2 -0
  144. gapit/providers/ecoh.py +83 -0
  145. gapit/providers/ecoli_kps.py +73 -0
  146. gapit/providers/ecoli_vf.py +99 -0
  147. gapit/providers/emmtyper.py +137 -0
  148. gapit/providers/kosc_k.py +73 -0
  149. gapit/providers/kosc_o.py +67 -0
  150. gapit/providers/kpsc_k.py +65 -0
  151. gapit/providers/kpsc_o.py +65 -0
  152. gapit/providers/megares.py +94 -0
  153. gapit/providers/ncbi.py +125 -0
  154. gapit/providers/nme_sero.py +199 -0
  155. gapit/providers/plasmidfinder.py +90 -0
  156. gapit/providers/resfinder.py +143 -0
  157. gapit/providers/seroba.py +148 -0
  158. gapit/providers/staphscope.py +173 -0
  159. gapit/providers/upec_expec_vf.py +100 -0
  160. gapit/providers/vch_oagc.py +285 -0
  161. gapit/providers/vfdb.py +120 -0
  162. gapit/providers/victors.py +130 -0
  163. gapit/py.typed +0 -0
  164. gapit/readpairs.py +110 -0
  165. gapit/reads.py +278 -0
  166. gapit/records.py +186 -0
  167. gapit/report.py +30 -0
  168. gapit/screen_defaults.py +155 -0
  169. gapit/screening.py +271 -0
  170. gapit/screening_batch.py +74 -0
  171. gapit/screening_cluster.py +348 -0
  172. gapit/screening_kmer.py +136 -0
  173. gapit/screening_reads.py +324 -0
  174. gapit/screening_reads_positional.py +212 -0
  175. gapit/seqconvert.py +203 -0
  176. gapit/serotype_engine.py +125 -0
  177. gapit/serotype_models.py +150 -0
  178. gapit/summary.py +205 -0
  179. gapit/typing_cluster2.py +137 -0
  180. gapit/typing_cluster_input.py +115 -0
  181. gapit/typing_decide.py +153 -0
  182. gapit/typing_engine.py +304 -0
  183. gapit/typing_gene.py +134 -0
  184. gapit/typing_input.py +142 -0
  185. gapit/typing_models.py +327 -0
  186. gapit/typing_ops.py +155 -0
  187. gapit/typing_results.py +161 -0
  188. gapit/typing_rules.py +212 -0
  189. gapit/typing_table.py +256 -0
  190. gspec-0.8.0.dist-info/METADATA +124 -0
  191. gspec-0.8.0.dist-info/RECORD +194 -0
  192. gspec-0.8.0.dist-info/WHEEL +4 -0
  193. gspec-0.8.0.dist-info/entry_points.txt +5 -0
  194. gspec-0.8.0.dist-info/licenses/LICENSE +21 -0
gapit/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """gspec — agent-first Python reimplementation of abricate."""
2
+
3
+ __version__ = "0.8.0"
gapit/blast.py ADDED
@@ -0,0 +1,291 @@
1
+ """BLAST invocation, tabular parsing, and the screening pipeline (SPEC.md §3)."""
2
+
3
+ import re
4
+ import shlex
5
+ import subprocess
6
+ import sys
7
+ from collections.abc import Iterable
8
+ from pathlib import Path
9
+ from typing import Literal
10
+
11
+ from pydantic import BaseModel
12
+
13
+ from gapit.db import Database
14
+ from gapit.errors import DependencyError, GapitError, InputError
15
+ from gapit.fragments import merge_fragment_hits
16
+ from gapit.gene_floors import apply_hit_floors, gene_floors
17
+ from gapit.gseek_engine import engine, run_blast_gseek
18
+ from gapit.hits import process_rows, process_rows_with_partials
19
+ from gapit.report import Report, ScreeningParams
20
+ from gapit.seqconvert import SeqFormat, detect_format, to_fasta_lines
21
+
22
+ BLAST_FIELDS = [
23
+ "qseqid",
24
+ "qstart",
25
+ "qend",
26
+ "qlen",
27
+ "sseqid",
28
+ "sstart",
29
+ "send",
30
+ "slen",
31
+ "sstrand",
32
+ "evalue",
33
+ "length",
34
+ "pident",
35
+ "gaps",
36
+ "gapopen",
37
+ "stitle",
38
+ ]
39
+
40
+ _OUTFMT = "6 " + " ".join(BLAST_FIELDS)
41
+ _MIN_BLAST_VERSION = (2, 2, 30)
42
+
43
+
44
+ class BlastRow(BaseModel, frozen=True):
45
+ """One outfmt-6 row, typed at the boundary (SPEC.md §3 field order)."""
46
+
47
+ qseqid: str
48
+ qstart: int
49
+ qend: int
50
+ qlen: int
51
+ sseqid: str
52
+ sstart: int
53
+ send: int
54
+ slen: int
55
+ sstrand: str
56
+ evalue: float
57
+ length: int
58
+ pident: float
59
+ gaps: int
60
+ gapopen: int
61
+ stitle: str
62
+
63
+
64
+ def parse_blast_row(line: str) -> BlastRow:
65
+ """Parse one tab-delimited outfmt-6 line; a row with != 15 columns is a
66
+ hard error (upstream wording)."""
67
+ fields = line.split("\t")
68
+ if len(fields) != 15:
69
+ raise GapitError("can not find sequence data", code="BLAST_PARSE_FAILED")
70
+ try:
71
+ return BlastRow(
72
+ qseqid=fields[0],
73
+ qstart=int(fields[1]),
74
+ qend=int(fields[2]),
75
+ qlen=int(fields[3]),
76
+ sseqid=fields[4],
77
+ sstart=int(fields[5]),
78
+ send=int(fields[6]),
79
+ slen=int(fields[7]),
80
+ sstrand=fields[8],
81
+ evalue=float(fields[9]),
82
+ length=int(fields[10]),
83
+ pident=float(fields[11]),
84
+ gaps=int(fields[12]),
85
+ gapopen=int(fields[13]),
86
+ stitle=fields[14],
87
+ )
88
+ except (TypeError, ValueError) as exc:
89
+ raise GapitError(
90
+ f"can not parse BLAST row (numeric field): {line!r}",
91
+ code="BLAST_PARSE_FAILED",
92
+ ) from exc
93
+
94
+
95
+ def ensure_blast() -> None:
96
+ """Require blastn >= 2.2.30 on PATH (SPEC.md §3 version gate)."""
97
+ try:
98
+ result = subprocess.run(["blastn", "-version"], check=False, capture_output=True, text=True)
99
+ except FileNotFoundError as exc:
100
+ raise DependencyError(
101
+ "required binary not found on PATH: blastn",
102
+ code="MISSING_DEPENDENCY",
103
+ context={"binary": "blastn"},
104
+ ) from exc
105
+ if result.returncode != 0:
106
+ raise DependencyError(
107
+ f"blastn -version failed: {result.stderr.strip()}",
108
+ code="BLAST_VERSION_CHECK_FAILED",
109
+ )
110
+ match = re.search(r"blastn:\s*(\d+)\.(\d+)\.(\d+)", result.stdout)
111
+ if match is None:
112
+ raise DependencyError(
113
+ f"could not parse blastn version from: {result.stdout!r}",
114
+ code="BLAST_VERSION_CHECK_FAILED",
115
+ )
116
+ major, minor, revision = (int(part) for part in match.groups())
117
+ if (major, minor, revision) < _MIN_BLAST_VERSION:
118
+ raise DependencyError(
119
+ f"blastn {major}.{minor}.{revision} is older than 2.2.30",
120
+ code="BLAST_VERSION_TOO_OLD",
121
+ )
122
+
123
+
124
+ def _normalize(query: Path) -> tuple[str, SeqFormat]:
125
+ """Convert one input file to FASTA text in-process (seqconvert), wrapping
126
+ any parser failure in the blast-path error shape
127
+ ``invalid input file {query}: <reason>``.
128
+
129
+ The whole normalized FASTA is held in memory and fed to blast via
130
+ ``input=`` — genome-scale inputs are a few MB, a deliberate trade-off:
131
+ ``subprocess.run``'s communicate() owns stdin/stdout concurrency, so the
132
+ retired ``any2fasta | blastn`` Popen chain needs no stderr-drain thread
133
+ here anymore.
134
+ """
135
+ try:
136
+ fmt = detect_format(query)
137
+ text = "".join(to_fasta_lines(query, fmt))
138
+ except InputError as exc:
139
+ raise InputError(
140
+ f"invalid input file {query}: {exc}",
141
+ code="INVALID_INPUT",
142
+ context={"file": str(query)},
143
+ ) from exc
144
+ return text, fmt
145
+
146
+
147
+ def _pipeline(fasta_text: str, argv: list[str]) -> str:
148
+ """``<argv>`` reading the normalized FASTA on stdin; returns stdout as text."""
149
+ try:
150
+ result = subprocess.run(argv, input=fasta_text, capture_output=True, text=True)
151
+ except FileNotFoundError as exc:
152
+ raise DependencyError(
153
+ f"required binary not found on PATH: {argv[0]}",
154
+ code="MISSING_DEPENDENCY",
155
+ context={"binary": argv[0]},
156
+ ) from exc
157
+ if result.returncode != 0:
158
+ raise GapitError(
159
+ f"{argv[0]} failed: {result.stderr.strip()}",
160
+ code="BLAST_FAILED",
161
+ context={"binary": argv[0]},
162
+ )
163
+ return result.stdout
164
+
165
+
166
+ def run_blastn(
167
+ query: Path,
168
+ database: Database,
169
+ params: ScreeningParams,
170
+ *,
171
+ dbtype: Literal["nucl", "prot"],
172
+ debug: bool = False,
173
+ ) -> list[BlastRow]:
174
+ """Run the normalize (native seqconvert) -> blastn/blastx pipeline for one
175
+ query file.
176
+
177
+ The database must already be indexed; protein databases switch to blastx
178
+ without -perc_identity (upstream quirk, minid silently ignored). With
179
+ ``debug``, echo the normalization step and the exact blast argv to stderr
180
+ (abricate --debug parity). ``dbtype`` comes from the caller resolving it
181
+ once per run, so dependency/index errors surface before any per-file work.
182
+ """
183
+ if dbtype == "prot":
184
+ argv = [
185
+ "blastx",
186
+ "-task",
187
+ "blastx-fast",
188
+ "-seg",
189
+ "no",
190
+ "-db",
191
+ str(database.sequences_path),
192
+ "-outfmt",
193
+ _OUTFMT,
194
+ "-num_threads",
195
+ str(params.threads),
196
+ "-evalue",
197
+ "1E-20",
198
+ "-culling_limit",
199
+ "1",
200
+ "-max_target_seqs",
201
+ "10000",
202
+ ]
203
+ sys.stderr.write("--minid is not applied to protein databases (abricate parity)\n")
204
+ else:
205
+ argv = [
206
+ "blastn",
207
+ "-task",
208
+ "blastn",
209
+ "-dust",
210
+ "no",
211
+ "-perc_identity",
212
+ str(params.minid),
213
+ "-db",
214
+ str(database.sequences_path),
215
+ "-outfmt",
216
+ _OUTFMT,
217
+ "-num_threads",
218
+ str(params.threads),
219
+ "-evalue",
220
+ "1E-20",
221
+ "-culling_limit",
222
+ "1",
223
+ "-max_target_seqs",
224
+ "10000",
225
+ ]
226
+ fasta_text, fmt = _normalize(query)
227
+ switch = engine()
228
+ mode = switch.mode.value
229
+ if debug:
230
+ sys.stderr.write(f"gspec: normalize: {query} ({fmt.value})\n")
231
+ if mode in {"legacy", "shadow"}:
232
+ sys.stderr.write(f"gspec: run: {shlex.join(argv)}\n")
233
+
234
+ def legacy_rows() -> list[BlastRow]:
235
+ output = _pipeline(fasta_text, argv)
236
+ return [parse_blast_row(line) for line in output.splitlines() if line.strip()]
237
+
238
+ def gseek_rows() -> list[BlastRow]:
239
+ return run_blast_gseek(fasta_text, database, params, dbtype=dbtype, debug=debug)
240
+
241
+ return switch.run(legacy_rows, gseek_rows)
242
+
243
+
244
+ def report_from_rows(
245
+ rows: Iterable[BlastRow],
246
+ query: Path,
247
+ database: Database,
248
+ params: ScreeningParams,
249
+ *,
250
+ merge_fragments: bool = False,
251
+ ) -> Report:
252
+ """Fold typed BLAST rows into one sorted Report.
253
+
254
+ Shared by the per-file and batch gseek paths; the caller owns the BLAST
255
+ run and the per-run ``ensure_blast``/dbtype gates.
256
+ """
257
+ floors = gene_floors(database)
258
+ if merge_fragments:
259
+ hits, partials = process_rows_with_partials(
260
+ rows, mincov=params.mincov, default_db=params.db
261
+ )
262
+ if floors is not None:
263
+ hits = apply_hit_floors(hits, floors)
264
+ partials = apply_hit_floors(partials, floors)
265
+ hits = merge_fragment_hits(hits, partials, mincov=params.mincov)
266
+ else:
267
+ hits = process_rows(rows, mincov=params.mincov, default_db=params.db)
268
+ if floors is not None:
269
+ hits = apply_hit_floors(hits, floors)
270
+ ordered = sorted(hits, key=lambda hit: (hit.sequence, hit.start))
271
+ return Report(file=str(query), hits=tuple(ordered))
272
+
273
+
274
+ def screen_file(
275
+ query: Path,
276
+ database: Database,
277
+ params: ScreeningParams,
278
+ *,
279
+ dbtype: Literal["nucl", "prot"],
280
+ debug: bool = False,
281
+ merge_fragments: bool = False,
282
+ ) -> Report:
283
+ """Screen one input file against one database into a sorted Report.
284
+
285
+ The caller owns the per-run gates (``ensure_blast`` and the one-shot
286
+ ``dbtype`` resolution) so a multi-file run pays each probe once. With
287
+ ``merge_fragments`` (opt-in gspec extension), sub-mincov dedup survivors
288
+ are folded into one hit per gene when their union coverage reaches mincov.
289
+ """
290
+ rows = run_blastn(query, database, params, dbtype=dbtype, debug=debug)
291
+ return report_from_rows(rows, query, database, params, merge_fragments=merge_fragments)
gapit/bundled.py ADDED
@@ -0,0 +1,240 @@
1
+ """Bundled databases: license-clean content shipped inside the wheel.
2
+
3
+ A bundled database is a directory ``gapit/data/dbs/<name>/`` holding
4
+ ``sequences``, a ``bundled.json`` metadata file, and kind-dependent
5
+ companions: gene bundles an optional ``typing.json`` and ``floors.json``
6
+ (gapit.floors/1), cluster bundles a REQUIRED ``features.json``
7
+ (gapit.features/1) plus their typing document. Materialization replays the
8
+ standard build pipelines into the user's datadir — the gene path through
9
+ :func:`gapit.db_build_ops.perform_build`, the cluster path through
10
+ :func:`gapit.bundled_cluster.materialize_cluster_bundle` (which rebuilds
11
+ from the committed locus FASTA + resolved feature table via
12
+ :func:`gapit.clusterbuild.build_cluster_database`) — deterministic, zero
13
+ network, so a bundled database is ready at install time with no
14
+ ``db fetch``.
15
+
16
+ Only gspec-curated content with clean provenance may live here (the wheel
17
+ is MIT): our own panels and public-domain / permissive sources, never GPL,
18
+ non-commercial, or third-party provider snapshots whose redistribution terms
19
+ are uncertain. Every provider database — ncbi, card, ecoh, resfinder, ... —
20
+ is fetch-on-demand only, even when its upstream license would permit a
21
+ snapshot. The typed cluster snapshot vpa_ok is a pure built-in (no provider
22
+ behind it) carrying the resolved gapit.features/1 table a completed /2
23
+ build produced; a refresh means a new snapshot riding a wheel release.
24
+ The gene build pipeline re-headers any bundled FASTA into gapit/v1 like a
25
+ custom build, so header styles round-trip.
26
+
27
+ Bundling is the exception, not the pattern, and every new bundle clears
28
+ two gates in order (the full policy lives in AGENTS.md §6): the license
29
+ gate above, then the size gate — the whole bundled root stays within a
30
+ 30 MiB wheel budget, mechanically enforced by tests/test_bundled.py.
31
+ Past the budget, split a separate data wheel or keep the database
32
+ fetch-on-demand instead of inflating the ceiling.
33
+ """
34
+
35
+ import shutil
36
+ from pathlib import Path
37
+ from typing import Literal
38
+
39
+ from pydantic import BaseModel
40
+
41
+ from gapit import config
42
+ from gapit.bundled_cluster import materialize_cluster_bundle
43
+ from gapit.db_build_ops import perform_build
44
+ from gapit.db_info_models import DbGuidance
45
+ from gapit.errors import DatabaseError
46
+ from gapit.fasta import iter_fasta
47
+ from gapit.proctools import note
48
+
49
+ BUNDLED_ROOT = Path(__file__).resolve().parent / "data" / "dbs"
50
+
51
+
52
+ class BundledMetadata(BaseModel, frozen=True):
53
+ """Parsed ``bundled.json`` — the listing-facing facts for one bundle.
54
+
55
+ ``snapshotted`` is the ISO date the wheel copy was produced from
56
+ upstream (provenance for humans; ``db fetch`` refreshes past it).
57
+ ``kind`` routes materialization (gene vs cluster pipeline; cluster
58
+ bundles additionally require ``features.json``). ``license`` and
59
+ ``note`` carry content provenance into the materialized manifest
60
+ (cluster bundles); they default to empty so the gene bundles'
61
+ metadata stays minimal. ``guidance`` is what `gspec db info` reports:
62
+ the literature references, the intended usage recipe, and the caveats
63
+ a caller must not skip.
64
+ """
65
+
66
+ name: str
67
+ description: str
68
+ vendor: str
69
+ dbtype: Literal["nucl", "prot"]
70
+ kind: Literal["gene", "cluster"] = "gene"
71
+ snapshotted: str
72
+ license: str = ""
73
+ note: str = ""
74
+ guidance: DbGuidance = DbGuidance()
75
+
76
+
77
+ class BundledDatabase(BaseModel, frozen=True):
78
+ """One bundled database directory plus its parsed metadata."""
79
+
80
+ metadata: BundledMetadata
81
+ path: Path
82
+
83
+ @property
84
+ def name(self) -> str:
85
+ return self.metadata.name
86
+
87
+ @property
88
+ def sequences_path(self) -> Path:
89
+ return self.path / "sequences"
90
+
91
+ @property
92
+ def typing_path(self) -> Path | None:
93
+ """The bundled typing spec, when the database ships one."""
94
+ typing = self.path / "typing.json"
95
+ return typing if typing.is_file() else None
96
+
97
+ @property
98
+ def floors_path(self) -> Path | None:
99
+ """The bundled gapit.floors/1 sidecar, when the database ships one."""
100
+ floors = self.path / "floors.json"
101
+ return floors if floors.is_file() else None
102
+
103
+ @property
104
+ def screen_path(self) -> Path | None:
105
+ """The bundled gapit.screen/1 defaults, when the database ships one."""
106
+ screen = self.path / "screen.json"
107
+ return screen if screen.is_file() else None
108
+
109
+ @property
110
+ def features_path(self) -> Path | None:
111
+ """The bundled gapit.features/1 table — required (and checked at
112
+ discovery) for cluster bundles, absent from gene bundles."""
113
+ features = self.path / "features.json"
114
+ return features if features.is_file() else None
115
+
116
+ @property
117
+ def profile_path(self) -> Path | None:
118
+ """The optional gapit.cluster_profile/1 sidecar for cluster bundles."""
119
+ profile = self.path / "cluster.json"
120
+ return profile if profile.is_file() else None
121
+
122
+ @property
123
+ def gene_families_path(self) -> Path | None:
124
+ """The optional gapit.gene_families/1 sidecar for cluster bundles."""
125
+ families = self.path / "gene_families.json"
126
+ return families if families.is_file() else None
127
+
128
+ @property
129
+ def markers_path(self) -> Path | None:
130
+ """The optional marker metadata sidecar for a gene bundle."""
131
+ markers = self.path / "markers.json"
132
+ return markers if markers.is_file() else None
133
+
134
+ @property
135
+ def serotype_path(self) -> Path | None:
136
+ """The optional gapit.serotype/1 profile sidecar for a gene bundle."""
137
+ serotype = self.path / "serotype.json"
138
+ return serotype if serotype.is_file() else None
139
+
140
+
141
+ def bundled_databases() -> list[BundledDatabase]:
142
+ """Every bundled database, sorted by name. A directory qualifies when it
143
+ carries both ``sequences`` and ``bundled.json`` — plus ``features.json``
144
+ when the metadata declares ``kind: cluster`` (a cluster bundle without
145
+ its feature table cannot materialize; typed error, no silent
146
+ degradation). Malformed metadata propagates, and a wheel without the
147
+ data dir lists nothing."""
148
+ if not BUNDLED_ROOT.is_dir():
149
+ return []
150
+ databases: list[BundledDatabase] = []
151
+ for child in sorted(BUNDLED_ROOT.iterdir()):
152
+ metadata_path = child / "bundled.json"
153
+ if not (child.is_dir() and (child / "sequences").is_file() and metadata_path.is_file()):
154
+ continue
155
+ try:
156
+ metadata = BundledMetadata.model_validate_json(
157
+ metadata_path.read_text(encoding="utf-8")
158
+ )
159
+ except (OSError, UnicodeDecodeError, ValueError) as exc:
160
+ raise DatabaseError(
161
+ f"bundled metadata is malformed: {metadata_path}: {exc}",
162
+ code="BUNDLED_MALFORMED",
163
+ context={"db": child.name, "file": str(metadata_path)},
164
+ ) from exc
165
+ if metadata.kind == "cluster" and not (child / "features.json").is_file():
166
+ raise DatabaseError(
167
+ f"bundled cluster database {metadata.name} lacks features.json",
168
+ code="BUNDLED_MALFORMED",
169
+ context={"db": metadata.name},
170
+ )
171
+ databases.append(BundledDatabase(metadata=metadata, path=child))
172
+ return databases
173
+
174
+
175
+ def bundled_names() -> list[str]:
176
+ """Sorted names of every bundled database."""
177
+ return [database.name for database in bundled_databases()]
178
+
179
+
180
+ def find_bundled(name: str) -> BundledDatabase | None:
181
+ """The bundled database called ``name``, or None when none ships."""
182
+ return next((database for database in bundled_databases() if database.name == name), None)
183
+
184
+
185
+ def materialize_bundled(name: str, datadir: Path, *, quiet: bool = True) -> None:
186
+ """Build a bundled database into ``<datadir>/name`` via its kind's
187
+ standard pipeline (records.jsonl/sequences for gene bundles, locus
188
+ FASTA + features.json for cluster bundles; BLAST index, typing copy,
189
+ manifest stamped ``source: "bundled"`` — written last, as always).
190
+ Zero network: every input travels inside the wheel.
191
+
192
+ An already-materialized database (manifest present) is left untouched —
193
+ callers short-circuit on the manifest, and each path's
194
+ DB_ALREADY_EXISTS guard turns a racing double build into a typed error.
195
+ """
196
+ bundled = find_bundled(name)
197
+ if bundled is None:
198
+ raise DatabaseError(
199
+ f"unknown bundled database: {name}",
200
+ code="BUNDLED_NOT_FOUND",
201
+ context={"db": name},
202
+ )
203
+ if bundled.metadata.kind == "cluster":
204
+ materialize_cluster_bundle(bundled, datadir, quiet=quiet)
205
+ return
206
+ n_records = sum(1 for _ in iter_fasta(bundled.sequences_path))
207
+ note(quiet, f"materializing bundled database {name} ({n_records} records) into {datadir}")
208
+ perform_build(
209
+ name,
210
+ bundled.sequences_path,
211
+ None,
212
+ None,
213
+ bundled.metadata.description,
214
+ datadir,
215
+ force=False,
216
+ warn=lambda message: note(quiet, message),
217
+ quiet=True,
218
+ typing=bundled.typing_path,
219
+ floors=bundled.floors_path,
220
+ screen=bundled.screen_path,
221
+ source="bundled",
222
+ )
223
+ if bundled.markers_path is not None:
224
+ shutil.copyfile(bundled.markers_path, datadir / name / "markers.json")
225
+ if bundled.serotype_path is not None:
226
+ shutil.copyfile(bundled.serotype_path, datadir / name / "serotype.json")
227
+
228
+
229
+ def resolve_screen_datadir(cli_value: Path | None, db_name: str) -> Path:
230
+ """``resolve_datadir`` for the screening lookup — except that a MISSING
231
+ datadir is bootstrapped when (and only when) the requested database is
232
+ bundled: the zero-network first-use path must work on a fresh machine
233
+ where ``~/.local/share/gspec/db`` does not exist yet. Every other
234
+ missing-datadir case keeps the typed DATADIR_NOT_FOUND error."""
235
+ try:
236
+ return config.resolve_datadir(cli_value)
237
+ except DatabaseError as exc:
238
+ if exc.code != "DATADIR_NOT_FOUND" or find_bundled(db_name) is None:
239
+ raise
240
+ return config.ensure_datadir(cli_value)
@@ -0,0 +1,130 @@
1
+ """Kind-aware materialization for CLUSTER bundles (the bundled.py sibling).
2
+
3
+ A cluster bundle ships its build INPUTS committed: the locus FASTA plus the
4
+ ``gapit.features/1`` table already carrying RESOLVED gene ids (the snapshot
5
+ was produced by a completed gapit.typing/2 build, so its rules reference
6
+ the resolved locus_tag-style ids). Materialization reconstructs
7
+ :class:`LocusSequence` objects from those artifacts and replays
8
+ :func:`gapit.clusterbuild.build_cluster_database` — the entry taking
9
+ pre-parsed loci, the SAME code a provider fetch runs — into the user's
10
+ datadir, deterministic and zero network.
11
+
12
+ Gene-id resolution deliberately does NOT re-run: the bundled feature table
13
+ is already resolved, and re-running would be a no-op at best. The typing
14
+ document IS validated against the reconstructed loci (mirroring the build
15
+ path's pre-write gate), and the screen-time /2 guard re-checks afterwards
16
+ — belt and braces, like the fetch path.
17
+ """
18
+
19
+ import shutil
20
+ from datetime import UTC, datetime
21
+ from pathlib import Path
22
+ from typing import TYPE_CHECKING
23
+
24
+ from pydantic import ValidationError
25
+
26
+ from gapit.clusterbuild import build_cluster_database
27
+ from gapit.db_txn import staged_database
28
+ from gapit.errors import DatabaseError
29
+ from gapit.fasta import iter_fasta
30
+ from gapit.gbfeatures import FeaturesDocument, LocusSequence
31
+ from gapit.proctools import note
32
+ from gapit.typing_models import (
33
+ read_typing_document,
34
+ typing_schema_of,
35
+ validate_references,
36
+ )
37
+
38
+ if TYPE_CHECKING: # bundled.py imports this module at runtime; no cycle.
39
+ from gapit.bundled import BundledDatabase
40
+
41
+
42
+ def _malformed(db_name: str, reason: str) -> DatabaseError:
43
+ """The broken-wheel-cluster-bundle failure: located, stable code."""
44
+ return DatabaseError(
45
+ f"bundled cluster database {db_name} is malformed ({reason})",
46
+ code="BUNDLED_MALFORMED",
47
+ context={"db": db_name, "reason": reason},
48
+ )
49
+
50
+
51
+ def _reconstruct_loci(bundled: "BundledDatabase") -> list[LocusSequence]:
52
+ """The bundle's committed artifacts back into LocusSequence objects.
53
+
54
+ features.json supplies the feature tables (loci in document order), the
55
+ FASTA the sequences (one bare-id record per locus); every locus must
56
+ pair with the same-order same-id record — the reconstruction mirror of
57
+ the build self-check, BUNDLED_MALFORMED on any drift."""
58
+ features_path = bundled.path / "features.json"
59
+ try:
60
+ document = FeaturesDocument.model_validate_json(features_path.read_text(encoding="utf-8"))
61
+ except (OSError, UnicodeDecodeError) as exc:
62
+ raise _malformed(bundled.name, f"unreadable features.json: {exc}") from exc
63
+ except ValidationError as exc:
64
+ raise _malformed(bundled.name, f"unparseable features.json: {exc}") from exc
65
+ records = list(iter_fasta(bundled.sequences_path))
66
+ if len(records) != len(document.loci):
67
+ raise _malformed(
68
+ bundled.name,
69
+ f"{len(document.loci)} loci but {len(records)} sequence records",
70
+ )
71
+ loci: list[LocusSequence] = []
72
+ for locus, record in zip(document.loci, records, strict=True):
73
+ if locus.id != record.id:
74
+ raise _malformed(bundled.name, f"locus {locus.id!r} paired with record {record.id!r}")
75
+ loci.append(LocusSequence(features=locus, sequence=record.sequence))
76
+ return loci
77
+
78
+
79
+ def materialize_cluster_bundle(
80
+ bundled: "BundledDatabase", datadir: Path, *, quiet: bool = True
81
+ ) -> None:
82
+ """Build a cluster-kind bundle into ``<datadir>/name`` through the
83
+ cluster pipeline (locus FASTA, features.json, typing copy, BLAST index,
84
+ manifest stamped ``source: "bundled"`` and written last, as always).
85
+ The typing spec validates BEFORE any artifact is written, so a spec
86
+ drifting from the committed table fails with no half-built database;
87
+ an existing manifest is a typed DB_ALREADY_EXISTS (the racing
88
+ double-build guard perform_build gives the gene path)."""
89
+ name = bundled.name
90
+ db_dir = datadir / name
91
+ if (db_dir / "gapit-manifest.json").is_file():
92
+ raise DatabaseError(
93
+ f"won't overwrite existing database {name} (use --force)",
94
+ code="DB_ALREADY_EXISTS",
95
+ context={"db": name},
96
+ )
97
+ n_loci = sum(1 for _ in iter_fasta(bundled.sequences_path))
98
+ note(quiet, f"materializing bundled database {name} ({n_loci} loci) into {datadir}")
99
+ loci = _reconstruct_loci(bundled)
100
+ typing_schema = ""
101
+ if bundled.typing_path is not None:
102
+ document = read_typing_document(bundled.typing_path)
103
+ validate_references(
104
+ document, FeaturesDocument(loci=tuple(locus.features for locus in loci))
105
+ )
106
+ typing_schema = typing_schema_of(bundled.typing_path)
107
+ with staged_database(db_dir) as build_dir:
108
+ if (db_dir / "gapit-manifest.json").is_file():
109
+ raise DatabaseError(
110
+ f"won't overwrite existing database {name} (use --force)",
111
+ code="DB_ALREADY_EXISTS",
112
+ context={"db": name},
113
+ )
114
+ if bundled.profile_path is not None:
115
+ shutil.copyfile(bundled.profile_path, build_dir / "cluster.json")
116
+ if bundled.gene_families_path is not None:
117
+ shutil.copyfile(bundled.gene_families_path, build_dir / "gene_families.json")
118
+ build_cluster_database(
119
+ build_dir,
120
+ name=name,
121
+ loci=loci,
122
+ typing=bundled.typing_path,
123
+ source_urls=("local",),
124
+ fetched_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
125
+ typing_schema=typing_schema,
126
+ quiet=True, # the materializing note above is the only stderr line (gene-path parity)
127
+ content_license=bundled.metadata.license,
128
+ content_note=bundled.metadata.note,
129
+ source="bundled",
130
+ )