dnakit 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. dnakit/__init__.py +108 -0
  2. dnakit/_version.py +5 -0
  3. dnakit/alignment/__init__.py +6 -0
  4. dnakit/alignment/pairwise.py +512 -0
  5. dnakit/alignment/results.py +188 -0
  6. dnakit/backends/__init__.py +80 -0
  7. dnakit/backends/external.py +517 -0
  8. dnakit/backends/registry.py +207 -0
  9. dnakit/batch.py +452 -0
  10. dnakit/cache/__init__.py +5 -0
  11. dnakit/cache/store.py +424 -0
  12. dnakit/chunking.py +920 -0
  13. dnakit/cli/__init__.py +5 -0
  14. dnakit/cli/__main__.py +6 -0
  15. dnakit/cli/app.py +814 -0
  16. dnakit/cli/workflow.py +94 -0
  17. dnakit/config/__init__.py +5 -0
  18. dnakit/config/loader.py +381 -0
  19. dnakit/core/__init__.py +80 -0
  20. dnakit/core/_json.py +206 -0
  21. dnakit/core/backend_info.py +70 -0
  22. dnakit/core/collection.py +163 -0
  23. dnakit/core/coordinates.py +295 -0
  24. dnakit/core/enums.py +95 -0
  25. dnakit/core/facade.py +719 -0
  26. dnakit/core/feature.py +90 -0
  27. dnakit/core/gap.py +82 -0
  28. dnakit/core/issues.py +62 -0
  29. dnakit/core/provenance.py +305 -0
  30. dnakit/core/record.py +161 -0
  31. dnakit/core/results.py +134 -0
  32. dnakit/core/sequence.py +327 -0
  33. dnakit/datasets/__init__.py +100 -0
  34. dnakit/datasets/_advanced_shared.py +198 -0
  35. dnakit/datasets/_metadata.py +33 -0
  36. dnakit/datasets/advanced_deduplicate.py +183 -0
  37. dnakit/datasets/advanced_split.py +302 -0
  38. dnakit/datasets/clustering.py +286 -0
  39. dnakit/datasets/config.py +531 -0
  40. dnakit/datasets/deduplicate.py +303 -0
  41. dnakit/datasets/evaluation.py +244 -0
  42. dnakit/datasets/filter.py +135 -0
  43. dnakit/datasets/neural_clustering.py +221 -0
  44. dnakit/datasets/results.py +520 -0
  45. dnakit/datasets/split.py +450 -0
  46. dnakit/descriptors/__init__.py +102 -0
  47. dnakit/descriptors/_dinucleotide.py +345 -0
  48. dnakit/descriptors/_shared.py +180 -0
  49. dnakit/descriptors/all.py +917 -0
  50. dnakit/descriptors/basic.py +193 -0
  51. dnakit/descriptors/codon.py +195 -0
  52. dnakit/descriptors/complexity.py +201 -0
  53. dnakit/descriptors/entropy.py +101 -0
  54. dnakit/descriptors/homopolymer.py +105 -0
  55. dnakit/descriptors/kmer.py +88 -0
  56. dnakit/descriptors/results.py +286 -0
  57. dnakit/descriptors/schema.py +414 -0
  58. dnakit/descriptors/window.py +203 -0
  59. dnakit/download/__init__.py +54 -0
  60. dnakit/download/catalogs.py +166 -0
  61. dnakit/download/ena.py +131 -0
  62. dnakit/download/encode.py +103 -0
  63. dnakit/download/files.py +469 -0
  64. dnakit/download/indexes.py +233 -0
  65. dnakit/download/metadata.py +164 -0
  66. dnakit/download/models.py +227 -0
  67. dnakit/download/ncbi.py +333 -0
  68. dnakit/download/sequence.py +133 -0
  69. dnakit/download/ucsc.py +99 -0
  70. dnakit/evaluation/__init__.py +81 -0
  71. dnakit/evaluation/_shared.py +288 -0
  72. dnakit/evaluation/collection.py +270 -0
  73. dnakit/evaluation/config.py +591 -0
  74. dnakit/evaluation/distribution.py +287 -0
  75. dnakit/evaluation/frechet.py +223 -0
  76. dnakit/evaluation/generative.py +381 -0
  77. dnakit/evaluation/reference.py +409 -0
  78. dnakit/evaluation/results.py +137 -0
  79. dnakit/evaluation/scorecard.py +211 -0
  80. dnakit/evaluation/sequence.py +403 -0
  81. dnakit/evaluation/synthesis.py +249 -0
  82. dnakit/exceptions.py +189 -0
  83. dnakit/fingerprints/__init__.py +109 -0
  84. dnakit/fingerprints/_shared.py +116 -0
  85. dnakit/fingerprints/advanced.py +829 -0
  86. dnakit/fingerprints/bit.py +229 -0
  87. dnakit/fingerprints/encoding.py +351 -0
  88. dnakit/fingerprints/kmer.py +170 -0
  89. dnakit/fingerprints/results.py +531 -0
  90. dnakit/fingerprints/sketch.py +246 -0
  91. dnakit/io/__init__.py +112 -0
  92. dnakit/io/_advanced_common.py +105 -0
  93. dnakit/io/_formats.py +994 -0
  94. dnakit/io/annotations.py +1179 -0
  95. dnakit/io/api.py +759 -0
  96. dnakit/io/config.py +315 -0
  97. dnakit/io/genbank.py +523 -0
  98. dnakit/io/indexed.py +1146 -0
  99. dnakit/io/metadata.py +452 -0
  100. dnakit/io/parquet.py +56 -0
  101. dnakit/io/results.py +131 -0
  102. dnakit/io/source.py +138 -0
  103. dnakit/io/tables.py +1237 -0
  104. dnakit/molbio/__init__.py +101 -0
  105. dnakit/molbio/_shared.py +266 -0
  106. dnakit/molbio/assembly.py +279 -0
  107. dnakit/molbio/crispr.py +181 -0
  108. dnakit/molbio/optimization.py +897 -0
  109. dnakit/molbio/primer3_design.py +667 -0
  110. dnakit/molbio/primers.py +899 -0
  111. dnakit/molbio/restriction.py +573 -0
  112. dnakit/molbio/results.py +583 -0
  113. dnakit/ops/__init__.py +588 -0
  114. dnakit/ops/_common.py +201 -0
  115. dnakit/ops/circular.py +193 -0
  116. dnakit/ops/concat.py +204 -0
  117. dnakit/ops/direction.py +27 -0
  118. dnakit/ops/edit.py +443 -0
  119. dnakit/ops/evolution.py +991 -0
  120. dnakit/ops/mutation.py +151 -0
  121. dnakit/ops/records.py +1012 -0
  122. dnakit/ops/translation.py +257 -0
  123. dnakit/patterns/__init__.py +81 -0
  124. dnakit/patterns/_shared.py +282 -0
  125. dnakit/patterns/coding.py +360 -0
  126. dnakit/patterns/crispr.py +337 -0
  127. dnakit/patterns/motif.py +721 -0
  128. dnakit/patterns/regions.py +341 -0
  129. dnakit/patterns/repeats.py +480 -0
  130. dnakit/patterns/restriction.py +281 -0
  131. dnakit/patterns/results.py +411 -0
  132. dnakit/py.typed +1 -0
  133. dnakit/references/__init__.py +27 -0
  134. dnakit/references/models.py +155 -0
  135. dnakit/references/ncbi.py +785 -0
  136. dnakit/representations/__init__.py +29 -0
  137. dnakit/representations/backends.py +928 -0
  138. dnakit/representations/checkpoints.py +356 -0
  139. dnakit/representations/extraction.py +220 -0
  140. dnakit/representations/models.py +395 -0
  141. dnakit/representations/results.py +79 -0
  142. dnakit/search/__init__.py +132 -0
  143. dnakit/search/_http.py +271 -0
  144. dnakit/search/_shared.py +108 -0
  145. dnakit/search/blast.py +516 -0
  146. dnakit/search/ena.py +245 -0
  147. dnakit/search/encode.py +102 -0
  148. dnakit/search/ensembl.py +874 -0
  149. dnakit/search/models.py +159 -0
  150. dnakit/search/ncbi.py +756 -0
  151. dnakit/search/ucsc.py +388 -0
  152. dnakit/secondary_structure/__init__.py +38 -0
  153. dnakit/secondary_structure/dotbracket.py +410 -0
  154. dnakit/secondary_structure/nupack.py +589 -0
  155. dnakit/secondary_structure/results.py +142 -0
  156. dnakit/similarity/__init__.py +125 -0
  157. dnakit/similarity/_shared.py +152 -0
  158. dnakit/similarity/approximate.py +258 -0
  159. dnakit/similarity/compare.py +162 -0
  160. dnakit/similarity/dashing.py +936 -0
  161. dnakit/similarity/distance.py +279 -0
  162. dnakit/similarity/index.py +365 -0
  163. dnakit/similarity/matrix.py +254 -0
  164. dnakit/similarity/results.py +676 -0
  165. dnakit/similarity/search.py +286 -0
  166. dnakit/similarity/sketch.py +96 -0
  167. dnakit/similarity/vector.py +662 -0
  168. dnakit/standardize/__init__.py +77 -0
  169. dnakit/standardize/_shared.py +100 -0
  170. dnakit/standardize/config.py +411 -0
  171. dnakit/standardize/gaps.py +1060 -0
  172. dnakit/standardize/normalize.py +565 -0
  173. dnakit/standardize/results.py +247 -0
  174. dnakit/standardize/validate.py +487 -0
  175. dnakit/structure3d/__init__.py +47 -0
  176. dnakit/structure3d/_shared.py +47 -0
  177. dnakit/structure3d/geometry.py +773 -0
  178. dnakit/structure3d/pdb.py +253 -0
  179. dnakit/structure3d/results.py +233 -0
  180. dnakit/structure3d/threedna.py +174 -0
  181. dnakit/thermodynamics/__init__.py +112 -0
  182. dnakit/thermodynamics/_shared.py +92 -0
  183. dnakit/thermodynamics/backends.py +653 -0
  184. dnakit/thermodynamics/calculations.py +837 -0
  185. dnakit/thermodynamics/config.py +175 -0
  186. dnakit/thermodynamics/equilibrium.py +340 -0
  187. dnakit/thermodynamics/optics.py +385 -0
  188. dnakit/thermodynamics/parameters.py +79 -0
  189. dnakit/thermodynamics/results.py +445 -0
  190. dnakit/visualization/__init__.py +43 -0
  191. dnakit/visualization/_svg.py +185 -0
  192. dnakit/visualization/advanced.py +314 -0
  193. dnakit/visualization/config.py +255 -0
  194. dnakit/visualization/export.py +349 -0
  195. dnakit/visualization/matrix.py +279 -0
  196. dnakit/visualization/report.py +431 -0
  197. dnakit/visualization/results.py +236 -0
  198. dnakit/visualization/sequence.py +459 -0
  199. dnakit/workflows/__init__.py +45 -0
  200. dnakit/workflows/manifest.py +365 -0
  201. dnakit/workflows/runner.py +1548 -0
  202. dnakit/workflows/schema.py +898 -0
  203. dnakit-0.1.0.dev0.dist-info/METADATA +126 -0
  204. dnakit-0.1.0.dev0.dist-info/RECORD +210 -0
  205. dnakit-0.1.0.dev0.dist-info/WHEEL +5 -0
  206. dnakit-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  207. dnakit-0.1.0.dev0.dist-info/licenses/DISCLAIMER.md +26 -0
  208. dnakit-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
  209. dnakit-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.md +73 -0
  210. dnakit-0.1.0.dev0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,785 @@
1
+ """NCBI Datasets genome-reference download adapter."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import os
9
+ import re
10
+ import tempfile
11
+ import zipfile
12
+ from collections.abc import Callable, Iterable, Mapping
13
+ from pathlib import Path, PurePosixPath
14
+ from typing import Any, BinaryIO, TypeAlias, cast
15
+ from urllib.error import HTTPError, URLError
16
+ from urllib.parse import quote, urlencode
17
+ from urllib.request import Request, urlopen
18
+
19
+ from dnakit._version import __version__
20
+ from dnakit.core import (
21
+ Citation,
22
+ ExecutionMode,
23
+ ImplementationInfo,
24
+ ImplementationLabel,
25
+ OriginClass,
26
+ Provenance,
27
+ ReferenceInfo,
28
+ )
29
+ from dnakit.exceptions import ConfigurationError, DownloadError
30
+
31
+ from .models import DownloadProgress, GenomeAssembly, GenomeDownloadResult
32
+
33
+ DEFAULT_NCBI_API_BASE_URL = "https://api.ncbi.nlm.nih.gov/datasets/v2"
34
+ DEFAULT_TIMEOUT = 60.0
35
+ DEFAULT_CHUNK_SIZE = 1024 * 1024
36
+ DEFAULT_MAX_DOWNLOAD_BYTES = 20_000_000_000
37
+ _ACCESSION_PATTERN = re.compile(r"^GC[AF]_\d+\.\d+$", flags=re.IGNORECASE)
38
+
39
+ # These aliases are intentionally versioned. Use an explicit accession when exact
40
+ # reproducibility matters; taxon names are resolved against NCBI's current report.
41
+ _FIXED_ASSEMBLY_ALIASES: dict[str, str] = {
42
+ "hg38": "GCF_000001405.40",
43
+ "grch38": "GCF_000001405.40",
44
+ "grch38.p14": "GCF_000001405.40",
45
+ "mm39": "GCF_000001635.27",
46
+ "grcm39": "GCF_000001635.27",
47
+ "tair10": "GCF_000001735.4",
48
+ }
49
+ _TAXON_ALIASES: dict[str, str] = {
50
+ "arabidopsis": "Arabidopsis thaliana",
51
+ "arabidopsis thaliana": "Arabidopsis thaliana",
52
+ "arabidopsis_thaliana": "Arabidopsis thaliana",
53
+ "human": "human",
54
+ "homo sapiens": "Homo sapiens",
55
+ "mouse": "mouse",
56
+ "mice": "mouse",
57
+ "mus musculus": "Mus musculus",
58
+ }
59
+ ProgressCallback: TypeAlias = Callable[[DownloadProgress], None]
60
+
61
+
62
+ def supported_genome_aliases() -> dict[str, str]:
63
+ """Return the fixed aliases supported without a metadata lookup."""
64
+
65
+ return dict(_FIXED_ASSEMBLY_ALIASES)
66
+
67
+
68
+ def _validate_positive(value: int | float, name: str) -> None:
69
+ if (
70
+ isinstance(value, bool)
71
+ or not isinstance(value, (int, float))
72
+ or not math.isfinite(value)
73
+ or value <= 0
74
+ ):
75
+ raise ConfigurationError(f"{name} must be positive.", code="INVALID_DOWNLOAD_CONFIG")
76
+
77
+
78
+ def _new_digest(name: str) -> Any:
79
+ if name == "md5":
80
+ return hashlib.md5(usedforsecurity=False)
81
+ return hashlib.new(name)
82
+
83
+
84
+ def _validate_common(
85
+ *,
86
+ timeout: float,
87
+ chunk_size: int,
88
+ max_download_bytes: int,
89
+ api_base_url: str,
90
+ api_key: str | None,
91
+ progress: ProgressCallback | None,
92
+ ) -> None:
93
+ _validate_positive(timeout, "timeout")
94
+ if isinstance(chunk_size, bool) or not isinstance(chunk_size, int) or chunk_size < 1:
95
+ raise ConfigurationError("chunk_size must be positive.", code="INVALID_DOWNLOAD_CONFIG")
96
+ if (
97
+ isinstance(max_download_bytes, bool)
98
+ or not isinstance(max_download_bytes, int)
99
+ or max_download_bytes < 1
100
+ ):
101
+ raise ConfigurationError(
102
+ "max_download_bytes must be positive.", code="INVALID_DOWNLOAD_CONFIG"
103
+ )
104
+ if not isinstance(api_base_url, str) or not api_base_url.startswith("https://"):
105
+ raise ConfigurationError(
106
+ "api_base_url must be an HTTPS URL.", code="INVALID_DOWNLOAD_CONFIG"
107
+ )
108
+ if api_key is not None and (not isinstance(api_key, str) or not api_key.strip()):
109
+ raise ConfigurationError(
110
+ "api_key must be None or non-empty text.", code="INVALID_DOWNLOAD_CONFIG"
111
+ )
112
+ if progress is not None and not callable(progress):
113
+ raise ConfigurationError(
114
+ "progress must be callable or None.", code="INVALID_DOWNLOAD_CONFIG"
115
+ )
116
+
117
+
118
+ def _request_headers(api_key: str | None) -> dict[str, str]:
119
+ headers = {
120
+ "Accept": "application/json",
121
+ "User-Agent": f"DNAKit/{__version__} NCBI-reference-downloader",
122
+ }
123
+ if api_key is not None:
124
+ headers["api-key"] = api_key
125
+ return headers
126
+
127
+
128
+ def _request_json(
129
+ url: str,
130
+ *,
131
+ timeout: float,
132
+ api_key: str | None,
133
+ max_bytes: int = 20_000_000,
134
+ ) -> dict[str, Any]:
135
+ request = Request(url, headers=_request_headers(api_key), method="GET")
136
+ try:
137
+ with urlopen(request, timeout=timeout) as response:
138
+ payload = _read_limited(response, max_bytes=max_bytes)
139
+ except HTTPError as exc:
140
+ raise DownloadError(
141
+ "NCBI Datasets returned an HTTP error.",
142
+ code="NCBI_HTTP_ERROR",
143
+ context={"url": url, "status": exc.code},
144
+ ) from exc
145
+ except (OSError, TimeoutError, URLError) as exc:
146
+ raise DownloadError(
147
+ "Could not reach NCBI Datasets.",
148
+ code="NCBI_NETWORK_ERROR",
149
+ context={"url": url},
150
+ ) from exc
151
+ try:
152
+ decoded = json.loads(payload.decode("utf-8"))
153
+ except (UnicodeDecodeError, json.JSONDecodeError) as exc:
154
+ raise DownloadError(
155
+ "NCBI Datasets returned invalid JSON.",
156
+ code="NCBI_RESPONSE_ERROR",
157
+ context={"url": url},
158
+ ) from exc
159
+ if not isinstance(decoded, dict):
160
+ raise DownloadError(
161
+ "NCBI Datasets returned an unexpected JSON shape.",
162
+ code="NCBI_RESPONSE_ERROR",
163
+ context={"url": url},
164
+ )
165
+ return cast(dict[str, Any], decoded)
166
+
167
+
168
+ def _read_limited(stream: BinaryIO, *, max_bytes: int) -> bytes:
169
+ chunks: list[bytes] = []
170
+ count = 0
171
+ while True:
172
+ chunk = stream.read(min(DEFAULT_CHUNK_SIZE, max_bytes - count + 1))
173
+ if not chunk:
174
+ break
175
+ count += len(chunk)
176
+ if count > max_bytes:
177
+ raise DownloadError(
178
+ "NCBI metadata response exceeds the configured limit.",
179
+ code="NCBI_RESPONSE_SIZE_LIMIT",
180
+ context={"max_bytes": max_bytes},
181
+ )
182
+ chunks.append(chunk)
183
+ return b"".join(chunks)
184
+
185
+
186
+ def _join_url(api_base_url: str, path: str, params: Iterable[tuple[str, str]]) -> str:
187
+ query = urlencode(tuple(params), doseq=True)
188
+ return f"{api_base_url.rstrip('/')}/{path.lstrip('/')}" + (f"?{query}" if query else "")
189
+
190
+
191
+ def _resolve_taxon_alias(query: str) -> str:
192
+ return _TAXON_ALIASES.get(query.casefold(), query)
193
+
194
+
195
+ def resolve_genome_assembly(
196
+ query: str,
197
+ *,
198
+ timeout: float = DEFAULT_TIMEOUT,
199
+ api_key: str | None = None,
200
+ api_base_url: str = DEFAULT_NCBI_API_BASE_URL,
201
+ ) -> GenomeAssembly:
202
+ """Resolve an accession, fixed alias, or taxon name to one NCBI assembly.
203
+
204
+ Taxon resolution selects one current RefSeq reference assembly. If NCBI
205
+ returns multiple candidates, callers must use an explicit accession.
206
+ """
207
+
208
+ if not isinstance(query, str) or not query.strip():
209
+ raise ConfigurationError("query must be non-empty text.", code="INVALID_GENOME_QUERY")
210
+ _validate_common(
211
+ timeout=timeout,
212
+ chunk_size=DEFAULT_CHUNK_SIZE,
213
+ max_download_bytes=DEFAULT_MAX_DOWNLOAD_BYTES,
214
+ api_base_url=api_base_url,
215
+ api_key=api_key,
216
+ progress=None,
217
+ )
218
+ normalized = query.strip()
219
+ if _ACCESSION_PATTERN.fullmatch(normalized):
220
+ return GenomeAssembly(query=normalized, accession=normalized.upper())
221
+ fixed_accession = _FIXED_ASSEMBLY_ALIASES.get(normalized.casefold())
222
+ if fixed_accession is not None:
223
+ return GenomeAssembly(query=normalized, accession=fixed_accession)
224
+
225
+ taxon = _resolve_taxon_alias(normalized)
226
+ url = _join_url(
227
+ api_base_url,
228
+ f"/genome/taxon/{quote(taxon, safe='')}/dataset_report",
229
+ (
230
+ ("filters.reference_only", "true"),
231
+ ("filters.assembly_source", "refseq"),
232
+ ("filters.assembly_version", "current"),
233
+ ("filters.exclude_paired_reports", "true"),
234
+ ("filters.exclude_atypical", "true"),
235
+ ("tax_exact_match", "true"),
236
+ ("returned_content", "ASSM_ACC"),
237
+ ("page_size", "1000"),
238
+ ),
239
+ )
240
+ payload = _request_json(url, timeout=timeout, api_key=api_key)
241
+ reports = payload.get("reports")
242
+ if not isinstance(reports, list):
243
+ raise DownloadError(
244
+ "NCBI Datasets taxon response did not contain reports.",
245
+ code="NCBI_RESPONSE_ERROR",
246
+ context={"query": query},
247
+ )
248
+ accessions: list[str] = []
249
+ for item in reports:
250
+ if not isinstance(item, Mapping):
251
+ continue
252
+ candidate = item.get("accession")
253
+ if isinstance(candidate, str):
254
+ accessions.append(candidate)
255
+ if not accessions:
256
+ raise ConfigurationError(
257
+ f"No current RefSeq reference assembly was found for {query!r}.",
258
+ code="GENOME_ASSEMBLY_NOT_FOUND",
259
+ context={"query": query},
260
+ hint="Use an explicit GCA_/GCF_ assembly accession.",
261
+ )
262
+ if len(accessions) != 1:
263
+ raise ConfigurationError(
264
+ f"Taxon {query!r} resolved to multiple reference assemblies.",
265
+ code="AMBIGUOUS_GENOME_ASSEMBLY",
266
+ context={"query": query, "accessions": accessions},
267
+ hint="Use an explicit GCA_/GCF_ assembly accession.",
268
+ )
269
+ accession = accessions[0]
270
+ if _ACCESSION_PATTERN.fullmatch(accession) is None:
271
+ raise DownloadError(
272
+ "NCBI returned an invalid assembly accession.",
273
+ code="NCBI_RESPONSE_ERROR",
274
+ context={"query": query, "accession": accession},
275
+ )
276
+ source_database = None
277
+ selected_report = next(
278
+ (
279
+ item
280
+ for item in reports
281
+ if isinstance(item, Mapping) and item.get("accession") == accession
282
+ ),
283
+ None,
284
+ )
285
+ if isinstance(selected_report, Mapping) and isinstance(
286
+ selected_report.get("source_database"), str
287
+ ):
288
+ source_database = selected_report["source_database"]
289
+ return GenomeAssembly(
290
+ query=normalized,
291
+ accession=accession,
292
+ source_database=source_database,
293
+ )
294
+
295
+
296
+ def _content_length(response: Any) -> int | None:
297
+ headers = getattr(response, "headers", None)
298
+ if headers is None:
299
+ return None
300
+ value = headers.get("Content-Length")
301
+ if value is None:
302
+ return None
303
+ try:
304
+ parsed = int(value)
305
+ except (TypeError, ValueError):
306
+ return None
307
+ return parsed if parsed >= 0 else None
308
+
309
+
310
+ def _download_package(
311
+ url: str,
312
+ target: Path,
313
+ *,
314
+ timeout: float,
315
+ api_key: str | None,
316
+ chunk_size: int,
317
+ max_download_bytes: int,
318
+ assembly: GenomeAssembly,
319
+ progress: ProgressCallback | None,
320
+ ) -> tuple[int, str]:
321
+ request = Request(url, headers={**_request_headers(api_key), "Accept": "application/zip"})
322
+ try:
323
+ with urlopen(request, timeout=timeout) as response:
324
+ total_bytes = _content_length(response)
325
+ if total_bytes is not None and total_bytes > max_download_bytes:
326
+ raise DownloadError(
327
+ "The NCBI genome package exceeds max_download_bytes.",
328
+ code="DOWNLOAD_SIZE_LIMIT",
329
+ context={
330
+ "accession": assembly.accession,
331
+ "content_length": total_bytes,
332
+ "max_download_bytes": max_download_bytes,
333
+ },
334
+ )
335
+ completed = 0
336
+ digest = hashlib.sha256()
337
+ with target.open("wb") as output:
338
+ while True:
339
+ chunk = response.read(chunk_size)
340
+ if not chunk:
341
+ break
342
+ completed += len(chunk)
343
+ if completed > max_download_bytes:
344
+ raise DownloadError(
345
+ "The NCBI genome package exceeds max_download_bytes.",
346
+ code="DOWNLOAD_SIZE_LIMIT",
347
+ context={
348
+ "accession": assembly.accession,
349
+ "downloaded_bytes": completed,
350
+ "max_download_bytes": max_download_bytes,
351
+ },
352
+ )
353
+ output.write(chunk)
354
+ digest.update(chunk)
355
+ if progress is not None:
356
+ progress(
357
+ DownloadProgress(
358
+ phase="download",
359
+ query=assembly.query,
360
+ accession=assembly.accession,
361
+ bytes_completed=completed,
362
+ total_bytes=total_bytes,
363
+ )
364
+ )
365
+ if completed < 1:
366
+ raise DownloadError(
367
+ "NCBI returned an empty genome package.",
368
+ code="EMPTY_DOWNLOAD",
369
+ context={"accession": assembly.accession},
370
+ )
371
+ return completed, digest.hexdigest()
372
+ except DownloadError:
373
+ raise
374
+ except HTTPError as exc:
375
+ raise DownloadError(
376
+ "NCBI Datasets returned an HTTP error while downloading the genome.",
377
+ code="NCBI_HTTP_ERROR",
378
+ context={"url": url, "status": exc.code},
379
+ ) from exc
380
+ except (OSError, TimeoutError, URLError) as exc:
381
+ raise DownloadError(
382
+ "The NCBI genome package could not be downloaded.",
383
+ code="NCBI_NETWORK_ERROR",
384
+ context={"url": url},
385
+ ) from exc
386
+
387
+
388
+ def _safe_member_name(name: str) -> str:
389
+ if not isinstance(name, str) or not name or "\x00" in name or "\\" in name:
390
+ raise DownloadError("NCBI package contains an unsafe member name.", code="UNSAFE_ARCHIVE")
391
+ path = PurePosixPath(name)
392
+ if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
393
+ raise DownloadError("NCBI package contains an unsafe member path.", code="UNSAFE_ARCHIVE")
394
+ return "/".join(path.parts)
395
+
396
+
397
+ def _validate_member(info: zipfile.ZipInfo) -> str:
398
+ name = _safe_member_name(info.filename)
399
+ mode = (info.external_attr >> 16) & 0o170000
400
+ if mode == 0o120000:
401
+ raise DownloadError("NCBI package contains a symbolic link.", code="UNSAFE_ARCHIVE")
402
+ if info.is_dir():
403
+ raise DownloadError(
404
+ "Expected NCBI package file but found a directory.", code="INVALID_ARCHIVE"
405
+ )
406
+ return name
407
+
408
+
409
+ def _find_member(infos: list[zipfile.ZipInfo], filename: str) -> zipfile.ZipInfo:
410
+ matches = [info for info in infos if PurePosixPath(info.filename).name == filename]
411
+ if len(matches) != 1:
412
+ raise DownloadError(
413
+ f"NCBI package must contain exactly one {filename} file.",
414
+ code="INVALID_ARCHIVE",
415
+ context={"matches": len(matches)},
416
+ )
417
+ _validate_member(matches[0])
418
+ return matches[0]
419
+
420
+
421
+ def _copy_member(
422
+ archive: zipfile.ZipFile,
423
+ info: zipfile.ZipInfo,
424
+ target: Path,
425
+ *,
426
+ chunk_size: int,
427
+ assembly: GenomeAssembly,
428
+ progress: ProgressCallback | None,
429
+ hash_algorithms: tuple[str, ...] = (),
430
+ ) -> dict[str, str]:
431
+ digests = {name: _new_digest(name) for name in hash_algorithms}
432
+ completed = 0
433
+ try:
434
+ with archive.open(info, "r") as source, target.open("wb") as output:
435
+ while True:
436
+ chunk = source.read(chunk_size)
437
+ if not chunk:
438
+ break
439
+ output.write(chunk)
440
+ completed += len(chunk)
441
+ for digest in digests.values():
442
+ digest.update(chunk)
443
+ if progress is not None:
444
+ progress(
445
+ DownloadProgress(
446
+ phase="extract",
447
+ query=assembly.query,
448
+ accession=assembly.accession,
449
+ bytes_completed=completed,
450
+ total_bytes=info.file_size,
451
+ current_file=PurePosixPath(info.filename).name,
452
+ )
453
+ )
454
+ except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
455
+ raise DownloadError(
456
+ "Could not extract the NCBI genome package.",
457
+ code="ARCHIVE_EXTRACTION_ERROR",
458
+ context={"member": info.filename},
459
+ ) from exc
460
+ return {name: digest.hexdigest() for name, digest in digests.items()}
461
+
462
+
463
+ def _read_member_text(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> str:
464
+ try:
465
+ with archive.open(info, "r") as source:
466
+ payload = source.read(20_000_001)
467
+ except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
468
+ raise DownloadError(
469
+ "Could not read an NCBI package metadata file.",
470
+ code="ARCHIVE_EXTRACTION_ERROR",
471
+ context={"member": info.filename},
472
+ ) from exc
473
+ if len(payload) > 20_000_000:
474
+ raise DownloadError(
475
+ "NCBI package metadata exceeds the configured limit.",
476
+ code="ARCHIVE_METADATA_SIZE_LIMIT",
477
+ context={"member": info.filename},
478
+ )
479
+ try:
480
+ return payload.decode("utf-8")
481
+ except UnicodeDecodeError as exc:
482
+ raise DownloadError(
483
+ "NCBI package metadata is not UTF-8.",
484
+ code="ARCHIVE_METADATA_ERROR",
485
+ context={"member": info.filename},
486
+ ) from exc
487
+
488
+
489
+ def _extract_report_details(text: str, accession: str) -> tuple[str | None, str | None]:
490
+ for line in text.splitlines():
491
+ if not line.strip():
492
+ continue
493
+ try:
494
+ payload = json.loads(line)
495
+ except json.JSONDecodeError as exc:
496
+ raise DownloadError(
497
+ "NCBI assembly_data_report.jsonl is invalid JSON.",
498
+ code="ARCHIVE_METADATA_ERROR",
499
+ ) from exc
500
+ if not isinstance(payload, Mapping):
501
+ continue
502
+ reported_accession = payload.get("accession")
503
+ assembly = payload.get("assembly")
504
+ organism = payload.get("organism")
505
+ if isinstance(assembly, Mapping):
506
+ reported_accession = assembly.get("accession", reported_accession)
507
+ if reported_accession is not None and reported_accession != accession:
508
+ continue
509
+ assembly_name = None
510
+ if isinstance(assembly, Mapping):
511
+ for key in ("name", "assembly_name"):
512
+ if isinstance(assembly.get(key), str) and assembly[key].strip():
513
+ assembly_name = assembly[key]
514
+ break
515
+ for key in ("assembly_name", "assminfo_name"):
516
+ if assembly_name is None and isinstance(payload.get(key), str):
517
+ assembly_name = payload[key]
518
+ organism_name = None
519
+ if isinstance(organism, Mapping):
520
+ for key in ("organism_name", "name"):
521
+ if isinstance(organism.get(key), str) and organism[key].strip():
522
+ organism_name = organism[key]
523
+ break
524
+ for key in ("organism_name", "organism"):
525
+ if organism_name is None and isinstance(payload.get(key), str):
526
+ organism_name = payload[key]
527
+ return organism_name, assembly_name
528
+ return None, None
529
+
530
+
531
+ def _expected_md5(text: str, member_name: str) -> str:
532
+ normalized_member = member_name.lstrip("./")
533
+ basename = PurePosixPath(member_name).name
534
+ for line in text.splitlines():
535
+ fields = line.strip().split(maxsplit=1)
536
+ if len(fields) != 2:
537
+ continue
538
+ digest, path = fields
539
+ normalized_path = path.lstrip("*").lstrip("./")
540
+ if (
541
+ normalized_path == normalized_member or PurePosixPath(normalized_path).name == basename
542
+ ) and re.fullmatch(r"[0-9a-fA-F]{32}", digest):
543
+ return digest.lower()
544
+ raise DownloadError(
545
+ "NCBI md5sum.txt does not contain the genomic FASTA entry.",
546
+ code="CHECKSUM_MISSING",
547
+ context={"member": member_name},
548
+ )
549
+
550
+
551
+ def _install_staged(staged_to_target: Iterable[tuple[Path, Path]], *, overwrite: bool) -> None:
552
+ pairs = tuple(staged_to_target)
553
+ targets = tuple(target for _, target in pairs)
554
+ if len(targets) != len(set(targets)):
555
+ raise ConfigurationError("Download output paths collide.", code="OUTPUT_COLLISION")
556
+ existing = [target for target in targets if target.exists()]
557
+ if existing and not overwrite:
558
+ raise FileExistsError(f"Refusing to overwrite existing download outputs: {existing}")
559
+ if any(target.exists() and not target.is_file() for target in targets):
560
+ raise IsADirectoryError("Download output targets must be files.")
561
+ installed: list[Path] = []
562
+ backups: list[tuple[Path, Path]] = []
563
+ try:
564
+ if overwrite:
565
+ backup_paths = [
566
+ target.with_name(f".{target.name}.dnakit-backup-{index}")
567
+ for index, target in enumerate(targets)
568
+ if target.exists()
569
+ ]
570
+ if any(path.exists() for path in backup_paths):
571
+ raise FileExistsError("A stale DNAKit download backup already exists.")
572
+ for index, (staged, target) in enumerate(pairs):
573
+ if overwrite:
574
+ if target.exists():
575
+ backup = target.with_name(f".{target.name}.dnakit-backup-{index}")
576
+ os.replace(target, backup)
577
+ backups.append((target, backup))
578
+ os.replace(staged, target)
579
+ else:
580
+ os.link(staged, target)
581
+ staged.unlink()
582
+ installed.append(target)
583
+ except BaseException:
584
+ for target in installed:
585
+ target.unlink(missing_ok=True)
586
+ for target, backup in reversed(backups):
587
+ if backup.exists():
588
+ os.replace(backup, target)
589
+ raise
590
+ for _, backup in backups:
591
+ backup.unlink(missing_ok=True)
592
+
593
+
594
+ def download_genome(
595
+ query: str,
596
+ output_dir: str | os.PathLike[str],
597
+ *,
598
+ overwrite: bool = False,
599
+ keep_package: bool = False,
600
+ timeout: float = DEFAULT_TIMEOUT,
601
+ api_key: str | None = None,
602
+ api_base_url: str = DEFAULT_NCBI_API_BASE_URL,
603
+ chunk_size: int = DEFAULT_CHUNK_SIZE,
604
+ max_download_bytes: int = DEFAULT_MAX_DOWNLOAD_BYTES,
605
+ progress: ProgressCallback | None = None,
606
+ ) -> GenomeDownloadResult:
607
+ """Download one NCBI genomic FASTA and its provenance files.
608
+
609
+ ``query`` may be an explicit GCA_/GCF_ assembly accession, a fixed alias
610
+ such as ``hg38`` or ``mm39``, or a taxon name. Taxon names are resolved
611
+ to one current RefSeq reference assembly and fail when ambiguous.
612
+ """
613
+
614
+ _validate_common(
615
+ timeout=timeout,
616
+ chunk_size=chunk_size,
617
+ max_download_bytes=max_download_bytes,
618
+ api_base_url=api_base_url,
619
+ api_key=api_key,
620
+ progress=progress,
621
+ )
622
+ if not isinstance(overwrite, bool) or not isinstance(keep_package, bool):
623
+ raise ConfigurationError(
624
+ "overwrite and keep_package must be booleans.", code="INVALID_DOWNLOAD_CONFIG"
625
+ )
626
+ assembly = resolve_genome_assembly(
627
+ query,
628
+ timeout=timeout,
629
+ api_key=api_key,
630
+ api_base_url=api_base_url,
631
+ )
632
+ if progress is not None:
633
+ progress(
634
+ DownloadProgress(
635
+ phase="resolve",
636
+ query=assembly.query,
637
+ accession=assembly.accession,
638
+ bytes_completed=0,
639
+ )
640
+ )
641
+ output = Path(output_dir).expanduser().resolve()
642
+ if output.exists() and not output.is_dir():
643
+ raise NotADirectoryError(f"output_dir must be a directory: {output}")
644
+ output.mkdir(parents=True, exist_ok=True)
645
+ url = _join_url(
646
+ api_base_url,
647
+ f"/genome/accession/{quote(assembly.accession, safe='')}/download",
648
+ (("include_annotation_type", "GENOME_FASTA"), ("hydrated", "FULLY_HYDRATED")),
649
+ )
650
+
651
+ with tempfile.TemporaryDirectory(prefix=".dnakit-ncbi-", dir=str(output)) as raw_work:
652
+ work = Path(raw_work)
653
+ package = work / "ncbi_dataset.zip"
654
+ downloaded_bytes, package_sha256 = _download_package(
655
+ url,
656
+ package,
657
+ timeout=timeout,
658
+ api_key=api_key,
659
+ chunk_size=chunk_size,
660
+ max_download_bytes=max_download_bytes,
661
+ assembly=assembly,
662
+ progress=progress,
663
+ )
664
+ try:
665
+ with zipfile.ZipFile(package, "r") as archive:
666
+ infos = archive.infolist()
667
+ for info in infos:
668
+ _safe_member_name(info.filename)
669
+ fasta_info = next(
670
+ (
671
+ info
672
+ for info in infos
673
+ if PurePosixPath(info.filename).name.endswith("_genomic.fna")
674
+ ),
675
+ None,
676
+ )
677
+ if fasta_info is None:
678
+ raise DownloadError(
679
+ "NCBI package does not contain a genomic FASTA file.",
680
+ code="INVALID_ARCHIVE",
681
+ )
682
+ fasta_member_name = _validate_member(fasta_info)
683
+ report_info = _find_member(infos, "assembly_data_report.jsonl")
684
+ checksum_info = _find_member(infos, "md5sum.txt")
685
+ report_text = _read_member_text(archive, report_info)
686
+ checksum_text = _read_member_text(archive, checksum_info)
687
+ expected_md5 = _expected_md5(checksum_text, fasta_member_name)
688
+ staged_fasta = work / PurePosixPath(fasta_member_name).name
689
+ digests = _copy_member(
690
+ archive,
691
+ fasta_info,
692
+ staged_fasta,
693
+ chunk_size=chunk_size,
694
+ assembly=assembly,
695
+ progress=progress,
696
+ hash_algorithms=("md5", "sha256"),
697
+ )
698
+ if digests["md5"] != expected_md5:
699
+ raise DownloadError(
700
+ "The downloaded genomic FASTA failed the NCBI MD5 check.",
701
+ code="CHECKSUM_MISMATCH",
702
+ context={"expected_md5": expected_md5, "actual_md5": digests["md5"]},
703
+ )
704
+ organism, assembly_name = _extract_report_details(report_text, assembly.accession)
705
+ staged_report = work / f"{assembly.accession}_assembly_data_report.jsonl"
706
+ staged_report.write_text(report_text, encoding="utf-8")
707
+ staged_checksum = work / f"{assembly.accession}_md5sum.txt"
708
+ staged_checksum.write_text(checksum_text, encoding="utf-8")
709
+ staged_package = package
710
+ fasta_target = output / staged_fasta.name
711
+ report_target = output / staged_report.name
712
+ checksum_target = output / staged_checksum.name
713
+ package_target = output / f"{assembly.accession}_ncbi_dataset.zip"
714
+ pairs: list[tuple[Path, Path]] = [
715
+ (staged_fasta, fasta_target),
716
+ (staged_report, report_target),
717
+ (staged_checksum, checksum_target),
718
+ ]
719
+ if keep_package:
720
+ pairs.append((staged_package, package_target))
721
+ _install_staged(pairs, overwrite=overwrite)
722
+ except DownloadError:
723
+ raise
724
+ except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
725
+ raise DownloadError(
726
+ "The NCBI genome package is invalid or could not be extracted.",
727
+ code="INVALID_ARCHIVE",
728
+ context={"accession": assembly.accession},
729
+ ) from exc
730
+
731
+ return GenomeDownloadResult(
732
+ query=assembly.query,
733
+ accession=assembly.accession,
734
+ organism=organism,
735
+ assembly_name=assembly_name,
736
+ output_directory=str(output),
737
+ fasta_path=str(fasta_target),
738
+ metadata_path=str(report_target),
739
+ checksum_path=str(checksum_target),
740
+ package_path=str(package_target) if keep_package else None,
741
+ download_url=url,
742
+ downloaded_bytes=downloaded_bytes,
743
+ package_sha256=package_sha256,
744
+ fasta_sha256=digests["sha256"],
745
+ fasta_md5=digests["md5"],
746
+ provenance=Provenance(
747
+ implementation=ImplementationInfo(
748
+ label=ImplementationLabel.ADAPTER,
749
+ execution_mode=ExecutionMode.EXTERNAL,
750
+ origin_class=OriginClass.INTEGRATION,
751
+ citations=(
752
+ Citation(
753
+ "ncbi-datasets",
754
+ title="NCBI Datasets Genome Package",
755
+ url=(
756
+ "https://www.ncbi.nlm.nih.gov/datasets/docs/v2/"
757
+ "reference-docs/data-packages/genome/"
758
+ ),
759
+ ),
760
+ ),
761
+ ),
762
+ reference=ReferenceInfo(
763
+ name="NCBI genome assembly",
764
+ version=assembly.accession,
765
+ checksum=digests["sha256"],
766
+ filters={
767
+ "query": assembly.query,
768
+ "include_annotation_type": "GENOME_FASTA",
769
+ "hydrated": "FULLY_HYDRATED",
770
+ },
771
+ ),
772
+ ),
773
+ )
774
+
775
+
776
+ __all__ = [
777
+ "DEFAULT_CHUNK_SIZE",
778
+ "DEFAULT_MAX_DOWNLOAD_BYTES",
779
+ "DEFAULT_NCBI_API_BASE_URL",
780
+ "DEFAULT_TIMEOUT",
781
+ "ProgressCallback",
782
+ "download_genome",
783
+ "resolve_genome_assembly",
784
+ "supported_genome_aliases",
785
+ ]