dnakit 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dnakit/__init__.py +108 -0
- dnakit/_version.py +5 -0
- dnakit/alignment/__init__.py +6 -0
- dnakit/alignment/pairwise.py +512 -0
- dnakit/alignment/results.py +188 -0
- dnakit/backends/__init__.py +80 -0
- dnakit/backends/external.py +517 -0
- dnakit/backends/registry.py +207 -0
- dnakit/batch.py +452 -0
- dnakit/cache/__init__.py +5 -0
- dnakit/cache/store.py +424 -0
- dnakit/chunking.py +920 -0
- dnakit/cli/__init__.py +5 -0
- dnakit/cli/__main__.py +6 -0
- dnakit/cli/app.py +814 -0
- dnakit/cli/workflow.py +94 -0
- dnakit/config/__init__.py +5 -0
- dnakit/config/loader.py +381 -0
- dnakit/core/__init__.py +80 -0
- dnakit/core/_json.py +206 -0
- dnakit/core/backend_info.py +70 -0
- dnakit/core/collection.py +163 -0
- dnakit/core/coordinates.py +295 -0
- dnakit/core/enums.py +95 -0
- dnakit/core/facade.py +719 -0
- dnakit/core/feature.py +90 -0
- dnakit/core/gap.py +82 -0
- dnakit/core/issues.py +62 -0
- dnakit/core/provenance.py +305 -0
- dnakit/core/record.py +161 -0
- dnakit/core/results.py +134 -0
- dnakit/core/sequence.py +327 -0
- dnakit/datasets/__init__.py +100 -0
- dnakit/datasets/_advanced_shared.py +198 -0
- dnakit/datasets/_metadata.py +33 -0
- dnakit/datasets/advanced_deduplicate.py +183 -0
- dnakit/datasets/advanced_split.py +302 -0
- dnakit/datasets/clustering.py +286 -0
- dnakit/datasets/config.py +531 -0
- dnakit/datasets/deduplicate.py +303 -0
- dnakit/datasets/evaluation.py +244 -0
- dnakit/datasets/filter.py +135 -0
- dnakit/datasets/neural_clustering.py +221 -0
- dnakit/datasets/results.py +520 -0
- dnakit/datasets/split.py +450 -0
- dnakit/descriptors/__init__.py +102 -0
- dnakit/descriptors/_dinucleotide.py +345 -0
- dnakit/descriptors/_shared.py +180 -0
- dnakit/descriptors/all.py +917 -0
- dnakit/descriptors/basic.py +193 -0
- dnakit/descriptors/codon.py +195 -0
- dnakit/descriptors/complexity.py +201 -0
- dnakit/descriptors/entropy.py +101 -0
- dnakit/descriptors/homopolymer.py +105 -0
- dnakit/descriptors/kmer.py +88 -0
- dnakit/descriptors/results.py +286 -0
- dnakit/descriptors/schema.py +414 -0
- dnakit/descriptors/window.py +203 -0
- dnakit/download/__init__.py +54 -0
- dnakit/download/catalogs.py +166 -0
- dnakit/download/ena.py +131 -0
- dnakit/download/encode.py +103 -0
- dnakit/download/files.py +469 -0
- dnakit/download/indexes.py +233 -0
- dnakit/download/metadata.py +164 -0
- dnakit/download/models.py +227 -0
- dnakit/download/ncbi.py +333 -0
- dnakit/download/sequence.py +133 -0
- dnakit/download/ucsc.py +99 -0
- dnakit/evaluation/__init__.py +81 -0
- dnakit/evaluation/_shared.py +288 -0
- dnakit/evaluation/collection.py +270 -0
- dnakit/evaluation/config.py +591 -0
- dnakit/evaluation/distribution.py +287 -0
- dnakit/evaluation/frechet.py +223 -0
- dnakit/evaluation/generative.py +381 -0
- dnakit/evaluation/reference.py +409 -0
- dnakit/evaluation/results.py +137 -0
- dnakit/evaluation/scorecard.py +211 -0
- dnakit/evaluation/sequence.py +403 -0
- dnakit/evaluation/synthesis.py +249 -0
- dnakit/exceptions.py +189 -0
- dnakit/fingerprints/__init__.py +109 -0
- dnakit/fingerprints/_shared.py +116 -0
- dnakit/fingerprints/advanced.py +829 -0
- dnakit/fingerprints/bit.py +229 -0
- dnakit/fingerprints/encoding.py +351 -0
- dnakit/fingerprints/kmer.py +170 -0
- dnakit/fingerprints/results.py +531 -0
- dnakit/fingerprints/sketch.py +246 -0
- dnakit/io/__init__.py +112 -0
- dnakit/io/_advanced_common.py +105 -0
- dnakit/io/_formats.py +994 -0
- dnakit/io/annotations.py +1179 -0
- dnakit/io/api.py +759 -0
- dnakit/io/config.py +315 -0
- dnakit/io/genbank.py +523 -0
- dnakit/io/indexed.py +1146 -0
- dnakit/io/metadata.py +452 -0
- dnakit/io/parquet.py +56 -0
- dnakit/io/results.py +131 -0
- dnakit/io/source.py +138 -0
- dnakit/io/tables.py +1237 -0
- dnakit/molbio/__init__.py +101 -0
- dnakit/molbio/_shared.py +266 -0
- dnakit/molbio/assembly.py +279 -0
- dnakit/molbio/crispr.py +181 -0
- dnakit/molbio/optimization.py +897 -0
- dnakit/molbio/primer3_design.py +667 -0
- dnakit/molbio/primers.py +899 -0
- dnakit/molbio/restriction.py +573 -0
- dnakit/molbio/results.py +583 -0
- dnakit/ops/__init__.py +588 -0
- dnakit/ops/_common.py +201 -0
- dnakit/ops/circular.py +193 -0
- dnakit/ops/concat.py +204 -0
- dnakit/ops/direction.py +27 -0
- dnakit/ops/edit.py +443 -0
- dnakit/ops/evolution.py +991 -0
- dnakit/ops/mutation.py +151 -0
- dnakit/ops/records.py +1012 -0
- dnakit/ops/translation.py +257 -0
- dnakit/patterns/__init__.py +81 -0
- dnakit/patterns/_shared.py +282 -0
- dnakit/patterns/coding.py +360 -0
- dnakit/patterns/crispr.py +337 -0
- dnakit/patterns/motif.py +721 -0
- dnakit/patterns/regions.py +341 -0
- dnakit/patterns/repeats.py +480 -0
- dnakit/patterns/restriction.py +281 -0
- dnakit/patterns/results.py +411 -0
- dnakit/py.typed +1 -0
- dnakit/references/__init__.py +27 -0
- dnakit/references/models.py +155 -0
- dnakit/references/ncbi.py +785 -0
- dnakit/representations/__init__.py +29 -0
- dnakit/representations/backends.py +928 -0
- dnakit/representations/checkpoints.py +356 -0
- dnakit/representations/extraction.py +220 -0
- dnakit/representations/models.py +395 -0
- dnakit/representations/results.py +79 -0
- dnakit/search/__init__.py +132 -0
- dnakit/search/_http.py +271 -0
- dnakit/search/_shared.py +108 -0
- dnakit/search/blast.py +516 -0
- dnakit/search/ena.py +245 -0
- dnakit/search/encode.py +102 -0
- dnakit/search/ensembl.py +874 -0
- dnakit/search/models.py +159 -0
- dnakit/search/ncbi.py +756 -0
- dnakit/search/ucsc.py +388 -0
- dnakit/secondary_structure/__init__.py +38 -0
- dnakit/secondary_structure/dotbracket.py +410 -0
- dnakit/secondary_structure/nupack.py +589 -0
- dnakit/secondary_structure/results.py +142 -0
- dnakit/similarity/__init__.py +125 -0
- dnakit/similarity/_shared.py +152 -0
- dnakit/similarity/approximate.py +258 -0
- dnakit/similarity/compare.py +162 -0
- dnakit/similarity/dashing.py +936 -0
- dnakit/similarity/distance.py +279 -0
- dnakit/similarity/index.py +365 -0
- dnakit/similarity/matrix.py +254 -0
- dnakit/similarity/results.py +676 -0
- dnakit/similarity/search.py +286 -0
- dnakit/similarity/sketch.py +96 -0
- dnakit/similarity/vector.py +662 -0
- dnakit/standardize/__init__.py +77 -0
- dnakit/standardize/_shared.py +100 -0
- dnakit/standardize/config.py +411 -0
- dnakit/standardize/gaps.py +1060 -0
- dnakit/standardize/normalize.py +565 -0
- dnakit/standardize/results.py +247 -0
- dnakit/standardize/validate.py +487 -0
- dnakit/structure3d/__init__.py +47 -0
- dnakit/structure3d/_shared.py +47 -0
- dnakit/structure3d/geometry.py +773 -0
- dnakit/structure3d/pdb.py +253 -0
- dnakit/structure3d/results.py +233 -0
- dnakit/structure3d/threedna.py +174 -0
- dnakit/thermodynamics/__init__.py +112 -0
- dnakit/thermodynamics/_shared.py +92 -0
- dnakit/thermodynamics/backends.py +653 -0
- dnakit/thermodynamics/calculations.py +837 -0
- dnakit/thermodynamics/config.py +175 -0
- dnakit/thermodynamics/equilibrium.py +340 -0
- dnakit/thermodynamics/optics.py +385 -0
- dnakit/thermodynamics/parameters.py +79 -0
- dnakit/thermodynamics/results.py +445 -0
- dnakit/visualization/__init__.py +43 -0
- dnakit/visualization/_svg.py +185 -0
- dnakit/visualization/advanced.py +314 -0
- dnakit/visualization/config.py +255 -0
- dnakit/visualization/export.py +349 -0
- dnakit/visualization/matrix.py +279 -0
- dnakit/visualization/report.py +431 -0
- dnakit/visualization/results.py +236 -0
- dnakit/visualization/sequence.py +459 -0
- dnakit/workflows/__init__.py +45 -0
- dnakit/workflows/manifest.py +365 -0
- dnakit/workflows/runner.py +1548 -0
- dnakit/workflows/schema.py +898 -0
- dnakit-0.1.0.dev0.dist-info/METADATA +126 -0
- dnakit-0.1.0.dev0.dist-info/RECORD +210 -0
- dnakit-0.1.0.dev0.dist-info/WHEEL +5 -0
- dnakit-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- dnakit-0.1.0.dev0.dist-info/licenses/DISCLAIMER.md +26 -0
- dnakit-0.1.0.dev0.dist-info/licenses/LICENSE +21 -0
- dnakit-0.1.0.dev0.dist-info/licenses/THIRD_PARTY_NOTICES.md +73 -0
- dnakit-0.1.0.dev0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,785 @@
|
|
|
1
|
+
"""NCBI Datasets genome-reference download adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import tempfile
|
|
11
|
+
import zipfile
|
|
12
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
13
|
+
from pathlib import Path, PurePosixPath
|
|
14
|
+
from typing import Any, BinaryIO, TypeAlias, cast
|
|
15
|
+
from urllib.error import HTTPError, URLError
|
|
16
|
+
from urllib.parse import quote, urlencode
|
|
17
|
+
from urllib.request import Request, urlopen
|
|
18
|
+
|
|
19
|
+
from dnakit._version import __version__
|
|
20
|
+
from dnakit.core import (
|
|
21
|
+
Citation,
|
|
22
|
+
ExecutionMode,
|
|
23
|
+
ImplementationInfo,
|
|
24
|
+
ImplementationLabel,
|
|
25
|
+
OriginClass,
|
|
26
|
+
Provenance,
|
|
27
|
+
ReferenceInfo,
|
|
28
|
+
)
|
|
29
|
+
from dnakit.exceptions import ConfigurationError, DownloadError
|
|
30
|
+
|
|
31
|
+
from .models import DownloadProgress, GenomeAssembly, GenomeDownloadResult
|
|
32
|
+
|
|
33
|
+
DEFAULT_NCBI_API_BASE_URL = "https://api.ncbi.nlm.nih.gov/datasets/v2"
|
|
34
|
+
DEFAULT_TIMEOUT = 60.0
|
|
35
|
+
DEFAULT_CHUNK_SIZE = 1024 * 1024
|
|
36
|
+
DEFAULT_MAX_DOWNLOAD_BYTES = 20_000_000_000
|
|
37
|
+
_ACCESSION_PATTERN = re.compile(r"^GC[AF]_\d+\.\d+$", flags=re.IGNORECASE)
|
|
38
|
+
|
|
39
|
+
# These aliases are intentionally versioned. Use an explicit accession when exact
|
|
40
|
+
# reproducibility matters; taxon names are resolved against NCBI's current report.
|
|
41
|
+
_FIXED_ASSEMBLY_ALIASES: dict[str, str] = {
|
|
42
|
+
"hg38": "GCF_000001405.40",
|
|
43
|
+
"grch38": "GCF_000001405.40",
|
|
44
|
+
"grch38.p14": "GCF_000001405.40",
|
|
45
|
+
"mm39": "GCF_000001635.27",
|
|
46
|
+
"grcm39": "GCF_000001635.27",
|
|
47
|
+
"tair10": "GCF_000001735.4",
|
|
48
|
+
}
|
|
49
|
+
_TAXON_ALIASES: dict[str, str] = {
|
|
50
|
+
"arabidopsis": "Arabidopsis thaliana",
|
|
51
|
+
"arabidopsis thaliana": "Arabidopsis thaliana",
|
|
52
|
+
"arabidopsis_thaliana": "Arabidopsis thaliana",
|
|
53
|
+
"human": "human",
|
|
54
|
+
"homo sapiens": "Homo sapiens",
|
|
55
|
+
"mouse": "mouse",
|
|
56
|
+
"mice": "mouse",
|
|
57
|
+
"mus musculus": "Mus musculus",
|
|
58
|
+
}
|
|
59
|
+
ProgressCallback: TypeAlias = Callable[[DownloadProgress], None]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def supported_genome_aliases() -> dict[str, str]:
|
|
63
|
+
"""Return the fixed aliases supported without a metadata lookup."""
|
|
64
|
+
|
|
65
|
+
return dict(_FIXED_ASSEMBLY_ALIASES)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _validate_positive(value: int | float, name: str) -> None:
|
|
69
|
+
if (
|
|
70
|
+
isinstance(value, bool)
|
|
71
|
+
or not isinstance(value, (int, float))
|
|
72
|
+
or not math.isfinite(value)
|
|
73
|
+
or value <= 0
|
|
74
|
+
):
|
|
75
|
+
raise ConfigurationError(f"{name} must be positive.", code="INVALID_DOWNLOAD_CONFIG")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _new_digest(name: str) -> Any:
|
|
79
|
+
if name == "md5":
|
|
80
|
+
return hashlib.md5(usedforsecurity=False)
|
|
81
|
+
return hashlib.new(name)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _validate_common(
|
|
85
|
+
*,
|
|
86
|
+
timeout: float,
|
|
87
|
+
chunk_size: int,
|
|
88
|
+
max_download_bytes: int,
|
|
89
|
+
api_base_url: str,
|
|
90
|
+
api_key: str | None,
|
|
91
|
+
progress: ProgressCallback | None,
|
|
92
|
+
) -> None:
|
|
93
|
+
_validate_positive(timeout, "timeout")
|
|
94
|
+
if isinstance(chunk_size, bool) or not isinstance(chunk_size, int) or chunk_size < 1:
|
|
95
|
+
raise ConfigurationError("chunk_size must be positive.", code="INVALID_DOWNLOAD_CONFIG")
|
|
96
|
+
if (
|
|
97
|
+
isinstance(max_download_bytes, bool)
|
|
98
|
+
or not isinstance(max_download_bytes, int)
|
|
99
|
+
or max_download_bytes < 1
|
|
100
|
+
):
|
|
101
|
+
raise ConfigurationError(
|
|
102
|
+
"max_download_bytes must be positive.", code="INVALID_DOWNLOAD_CONFIG"
|
|
103
|
+
)
|
|
104
|
+
if not isinstance(api_base_url, str) or not api_base_url.startswith("https://"):
|
|
105
|
+
raise ConfigurationError(
|
|
106
|
+
"api_base_url must be an HTTPS URL.", code="INVALID_DOWNLOAD_CONFIG"
|
|
107
|
+
)
|
|
108
|
+
if api_key is not None and (not isinstance(api_key, str) or not api_key.strip()):
|
|
109
|
+
raise ConfigurationError(
|
|
110
|
+
"api_key must be None or non-empty text.", code="INVALID_DOWNLOAD_CONFIG"
|
|
111
|
+
)
|
|
112
|
+
if progress is not None and not callable(progress):
|
|
113
|
+
raise ConfigurationError(
|
|
114
|
+
"progress must be callable or None.", code="INVALID_DOWNLOAD_CONFIG"
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _request_headers(api_key: str | None) -> dict[str, str]:
|
|
119
|
+
headers = {
|
|
120
|
+
"Accept": "application/json",
|
|
121
|
+
"User-Agent": f"DNAKit/{__version__} NCBI-reference-downloader",
|
|
122
|
+
}
|
|
123
|
+
if api_key is not None:
|
|
124
|
+
headers["api-key"] = api_key
|
|
125
|
+
return headers
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _request_json(
|
|
129
|
+
url: str,
|
|
130
|
+
*,
|
|
131
|
+
timeout: float,
|
|
132
|
+
api_key: str | None,
|
|
133
|
+
max_bytes: int = 20_000_000,
|
|
134
|
+
) -> dict[str, Any]:
|
|
135
|
+
request = Request(url, headers=_request_headers(api_key), method="GET")
|
|
136
|
+
try:
|
|
137
|
+
with urlopen(request, timeout=timeout) as response:
|
|
138
|
+
payload = _read_limited(response, max_bytes=max_bytes)
|
|
139
|
+
except HTTPError as exc:
|
|
140
|
+
raise DownloadError(
|
|
141
|
+
"NCBI Datasets returned an HTTP error.",
|
|
142
|
+
code="NCBI_HTTP_ERROR",
|
|
143
|
+
context={"url": url, "status": exc.code},
|
|
144
|
+
) from exc
|
|
145
|
+
except (OSError, TimeoutError, URLError) as exc:
|
|
146
|
+
raise DownloadError(
|
|
147
|
+
"Could not reach NCBI Datasets.",
|
|
148
|
+
code="NCBI_NETWORK_ERROR",
|
|
149
|
+
context={"url": url},
|
|
150
|
+
) from exc
|
|
151
|
+
try:
|
|
152
|
+
decoded = json.loads(payload.decode("utf-8"))
|
|
153
|
+
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
154
|
+
raise DownloadError(
|
|
155
|
+
"NCBI Datasets returned invalid JSON.",
|
|
156
|
+
code="NCBI_RESPONSE_ERROR",
|
|
157
|
+
context={"url": url},
|
|
158
|
+
) from exc
|
|
159
|
+
if not isinstance(decoded, dict):
|
|
160
|
+
raise DownloadError(
|
|
161
|
+
"NCBI Datasets returned an unexpected JSON shape.",
|
|
162
|
+
code="NCBI_RESPONSE_ERROR",
|
|
163
|
+
context={"url": url},
|
|
164
|
+
)
|
|
165
|
+
return cast(dict[str, Any], decoded)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _read_limited(stream: BinaryIO, *, max_bytes: int) -> bytes:
|
|
169
|
+
chunks: list[bytes] = []
|
|
170
|
+
count = 0
|
|
171
|
+
while True:
|
|
172
|
+
chunk = stream.read(min(DEFAULT_CHUNK_SIZE, max_bytes - count + 1))
|
|
173
|
+
if not chunk:
|
|
174
|
+
break
|
|
175
|
+
count += len(chunk)
|
|
176
|
+
if count > max_bytes:
|
|
177
|
+
raise DownloadError(
|
|
178
|
+
"NCBI metadata response exceeds the configured limit.",
|
|
179
|
+
code="NCBI_RESPONSE_SIZE_LIMIT",
|
|
180
|
+
context={"max_bytes": max_bytes},
|
|
181
|
+
)
|
|
182
|
+
chunks.append(chunk)
|
|
183
|
+
return b"".join(chunks)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _join_url(api_base_url: str, path: str, params: Iterable[tuple[str, str]]) -> str:
|
|
187
|
+
query = urlencode(tuple(params), doseq=True)
|
|
188
|
+
return f"{api_base_url.rstrip('/')}/{path.lstrip('/')}" + (f"?{query}" if query else "")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _resolve_taxon_alias(query: str) -> str:
|
|
192
|
+
return _TAXON_ALIASES.get(query.casefold(), query)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def resolve_genome_assembly(
|
|
196
|
+
query: str,
|
|
197
|
+
*,
|
|
198
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
199
|
+
api_key: str | None = None,
|
|
200
|
+
api_base_url: str = DEFAULT_NCBI_API_BASE_URL,
|
|
201
|
+
) -> GenomeAssembly:
|
|
202
|
+
"""Resolve an accession, fixed alias, or taxon name to one NCBI assembly.
|
|
203
|
+
|
|
204
|
+
Taxon resolution selects one current RefSeq reference assembly. If NCBI
|
|
205
|
+
returns multiple candidates, callers must use an explicit accession.
|
|
206
|
+
"""
|
|
207
|
+
|
|
208
|
+
if not isinstance(query, str) or not query.strip():
|
|
209
|
+
raise ConfigurationError("query must be non-empty text.", code="INVALID_GENOME_QUERY")
|
|
210
|
+
_validate_common(
|
|
211
|
+
timeout=timeout,
|
|
212
|
+
chunk_size=DEFAULT_CHUNK_SIZE,
|
|
213
|
+
max_download_bytes=DEFAULT_MAX_DOWNLOAD_BYTES,
|
|
214
|
+
api_base_url=api_base_url,
|
|
215
|
+
api_key=api_key,
|
|
216
|
+
progress=None,
|
|
217
|
+
)
|
|
218
|
+
normalized = query.strip()
|
|
219
|
+
if _ACCESSION_PATTERN.fullmatch(normalized):
|
|
220
|
+
return GenomeAssembly(query=normalized, accession=normalized.upper())
|
|
221
|
+
fixed_accession = _FIXED_ASSEMBLY_ALIASES.get(normalized.casefold())
|
|
222
|
+
if fixed_accession is not None:
|
|
223
|
+
return GenomeAssembly(query=normalized, accession=fixed_accession)
|
|
224
|
+
|
|
225
|
+
taxon = _resolve_taxon_alias(normalized)
|
|
226
|
+
url = _join_url(
|
|
227
|
+
api_base_url,
|
|
228
|
+
f"/genome/taxon/{quote(taxon, safe='')}/dataset_report",
|
|
229
|
+
(
|
|
230
|
+
("filters.reference_only", "true"),
|
|
231
|
+
("filters.assembly_source", "refseq"),
|
|
232
|
+
("filters.assembly_version", "current"),
|
|
233
|
+
("filters.exclude_paired_reports", "true"),
|
|
234
|
+
("filters.exclude_atypical", "true"),
|
|
235
|
+
("tax_exact_match", "true"),
|
|
236
|
+
("returned_content", "ASSM_ACC"),
|
|
237
|
+
("page_size", "1000"),
|
|
238
|
+
),
|
|
239
|
+
)
|
|
240
|
+
payload = _request_json(url, timeout=timeout, api_key=api_key)
|
|
241
|
+
reports = payload.get("reports")
|
|
242
|
+
if not isinstance(reports, list):
|
|
243
|
+
raise DownloadError(
|
|
244
|
+
"NCBI Datasets taxon response did not contain reports.",
|
|
245
|
+
code="NCBI_RESPONSE_ERROR",
|
|
246
|
+
context={"query": query},
|
|
247
|
+
)
|
|
248
|
+
accessions: list[str] = []
|
|
249
|
+
for item in reports:
|
|
250
|
+
if not isinstance(item, Mapping):
|
|
251
|
+
continue
|
|
252
|
+
candidate = item.get("accession")
|
|
253
|
+
if isinstance(candidate, str):
|
|
254
|
+
accessions.append(candidate)
|
|
255
|
+
if not accessions:
|
|
256
|
+
raise ConfigurationError(
|
|
257
|
+
f"No current RefSeq reference assembly was found for {query!r}.",
|
|
258
|
+
code="GENOME_ASSEMBLY_NOT_FOUND",
|
|
259
|
+
context={"query": query},
|
|
260
|
+
hint="Use an explicit GCA_/GCF_ assembly accession.",
|
|
261
|
+
)
|
|
262
|
+
if len(accessions) != 1:
|
|
263
|
+
raise ConfigurationError(
|
|
264
|
+
f"Taxon {query!r} resolved to multiple reference assemblies.",
|
|
265
|
+
code="AMBIGUOUS_GENOME_ASSEMBLY",
|
|
266
|
+
context={"query": query, "accessions": accessions},
|
|
267
|
+
hint="Use an explicit GCA_/GCF_ assembly accession.",
|
|
268
|
+
)
|
|
269
|
+
accession = accessions[0]
|
|
270
|
+
if _ACCESSION_PATTERN.fullmatch(accession) is None:
|
|
271
|
+
raise DownloadError(
|
|
272
|
+
"NCBI returned an invalid assembly accession.",
|
|
273
|
+
code="NCBI_RESPONSE_ERROR",
|
|
274
|
+
context={"query": query, "accession": accession},
|
|
275
|
+
)
|
|
276
|
+
source_database = None
|
|
277
|
+
selected_report = next(
|
|
278
|
+
(
|
|
279
|
+
item
|
|
280
|
+
for item in reports
|
|
281
|
+
if isinstance(item, Mapping) and item.get("accession") == accession
|
|
282
|
+
),
|
|
283
|
+
None,
|
|
284
|
+
)
|
|
285
|
+
if isinstance(selected_report, Mapping) and isinstance(
|
|
286
|
+
selected_report.get("source_database"), str
|
|
287
|
+
):
|
|
288
|
+
source_database = selected_report["source_database"]
|
|
289
|
+
return GenomeAssembly(
|
|
290
|
+
query=normalized,
|
|
291
|
+
accession=accession,
|
|
292
|
+
source_database=source_database,
|
|
293
|
+
)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _content_length(response: Any) -> int | None:
|
|
297
|
+
headers = getattr(response, "headers", None)
|
|
298
|
+
if headers is None:
|
|
299
|
+
return None
|
|
300
|
+
value = headers.get("Content-Length")
|
|
301
|
+
if value is None:
|
|
302
|
+
return None
|
|
303
|
+
try:
|
|
304
|
+
parsed = int(value)
|
|
305
|
+
except (TypeError, ValueError):
|
|
306
|
+
return None
|
|
307
|
+
return parsed if parsed >= 0 else None
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _download_package(
|
|
311
|
+
url: str,
|
|
312
|
+
target: Path,
|
|
313
|
+
*,
|
|
314
|
+
timeout: float,
|
|
315
|
+
api_key: str | None,
|
|
316
|
+
chunk_size: int,
|
|
317
|
+
max_download_bytes: int,
|
|
318
|
+
assembly: GenomeAssembly,
|
|
319
|
+
progress: ProgressCallback | None,
|
|
320
|
+
) -> tuple[int, str]:
|
|
321
|
+
request = Request(url, headers={**_request_headers(api_key), "Accept": "application/zip"})
|
|
322
|
+
try:
|
|
323
|
+
with urlopen(request, timeout=timeout) as response:
|
|
324
|
+
total_bytes = _content_length(response)
|
|
325
|
+
if total_bytes is not None and total_bytes > max_download_bytes:
|
|
326
|
+
raise DownloadError(
|
|
327
|
+
"The NCBI genome package exceeds max_download_bytes.",
|
|
328
|
+
code="DOWNLOAD_SIZE_LIMIT",
|
|
329
|
+
context={
|
|
330
|
+
"accession": assembly.accession,
|
|
331
|
+
"content_length": total_bytes,
|
|
332
|
+
"max_download_bytes": max_download_bytes,
|
|
333
|
+
},
|
|
334
|
+
)
|
|
335
|
+
completed = 0
|
|
336
|
+
digest = hashlib.sha256()
|
|
337
|
+
with target.open("wb") as output:
|
|
338
|
+
while True:
|
|
339
|
+
chunk = response.read(chunk_size)
|
|
340
|
+
if not chunk:
|
|
341
|
+
break
|
|
342
|
+
completed += len(chunk)
|
|
343
|
+
if completed > max_download_bytes:
|
|
344
|
+
raise DownloadError(
|
|
345
|
+
"The NCBI genome package exceeds max_download_bytes.",
|
|
346
|
+
code="DOWNLOAD_SIZE_LIMIT",
|
|
347
|
+
context={
|
|
348
|
+
"accession": assembly.accession,
|
|
349
|
+
"downloaded_bytes": completed,
|
|
350
|
+
"max_download_bytes": max_download_bytes,
|
|
351
|
+
},
|
|
352
|
+
)
|
|
353
|
+
output.write(chunk)
|
|
354
|
+
digest.update(chunk)
|
|
355
|
+
if progress is not None:
|
|
356
|
+
progress(
|
|
357
|
+
DownloadProgress(
|
|
358
|
+
phase="download",
|
|
359
|
+
query=assembly.query,
|
|
360
|
+
accession=assembly.accession,
|
|
361
|
+
bytes_completed=completed,
|
|
362
|
+
total_bytes=total_bytes,
|
|
363
|
+
)
|
|
364
|
+
)
|
|
365
|
+
if completed < 1:
|
|
366
|
+
raise DownloadError(
|
|
367
|
+
"NCBI returned an empty genome package.",
|
|
368
|
+
code="EMPTY_DOWNLOAD",
|
|
369
|
+
context={"accession": assembly.accession},
|
|
370
|
+
)
|
|
371
|
+
return completed, digest.hexdigest()
|
|
372
|
+
except DownloadError:
|
|
373
|
+
raise
|
|
374
|
+
except HTTPError as exc:
|
|
375
|
+
raise DownloadError(
|
|
376
|
+
"NCBI Datasets returned an HTTP error while downloading the genome.",
|
|
377
|
+
code="NCBI_HTTP_ERROR",
|
|
378
|
+
context={"url": url, "status": exc.code},
|
|
379
|
+
) from exc
|
|
380
|
+
except (OSError, TimeoutError, URLError) as exc:
|
|
381
|
+
raise DownloadError(
|
|
382
|
+
"The NCBI genome package could not be downloaded.",
|
|
383
|
+
code="NCBI_NETWORK_ERROR",
|
|
384
|
+
context={"url": url},
|
|
385
|
+
) from exc
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _safe_member_name(name: str) -> str:
|
|
389
|
+
if not isinstance(name, str) or not name or "\x00" in name or "\\" in name:
|
|
390
|
+
raise DownloadError("NCBI package contains an unsafe member name.", code="UNSAFE_ARCHIVE")
|
|
391
|
+
path = PurePosixPath(name)
|
|
392
|
+
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
|
|
393
|
+
raise DownloadError("NCBI package contains an unsafe member path.", code="UNSAFE_ARCHIVE")
|
|
394
|
+
return "/".join(path.parts)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _validate_member(info: zipfile.ZipInfo) -> str:
|
|
398
|
+
name = _safe_member_name(info.filename)
|
|
399
|
+
mode = (info.external_attr >> 16) & 0o170000
|
|
400
|
+
if mode == 0o120000:
|
|
401
|
+
raise DownloadError("NCBI package contains a symbolic link.", code="UNSAFE_ARCHIVE")
|
|
402
|
+
if info.is_dir():
|
|
403
|
+
raise DownloadError(
|
|
404
|
+
"Expected NCBI package file but found a directory.", code="INVALID_ARCHIVE"
|
|
405
|
+
)
|
|
406
|
+
return name
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _find_member(infos: list[zipfile.ZipInfo], filename: str) -> zipfile.ZipInfo:
|
|
410
|
+
matches = [info for info in infos if PurePosixPath(info.filename).name == filename]
|
|
411
|
+
if len(matches) != 1:
|
|
412
|
+
raise DownloadError(
|
|
413
|
+
f"NCBI package must contain exactly one {filename} file.",
|
|
414
|
+
code="INVALID_ARCHIVE",
|
|
415
|
+
context={"matches": len(matches)},
|
|
416
|
+
)
|
|
417
|
+
_validate_member(matches[0])
|
|
418
|
+
return matches[0]
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _copy_member(
|
|
422
|
+
archive: zipfile.ZipFile,
|
|
423
|
+
info: zipfile.ZipInfo,
|
|
424
|
+
target: Path,
|
|
425
|
+
*,
|
|
426
|
+
chunk_size: int,
|
|
427
|
+
assembly: GenomeAssembly,
|
|
428
|
+
progress: ProgressCallback | None,
|
|
429
|
+
hash_algorithms: tuple[str, ...] = (),
|
|
430
|
+
) -> dict[str, str]:
|
|
431
|
+
digests = {name: _new_digest(name) for name in hash_algorithms}
|
|
432
|
+
completed = 0
|
|
433
|
+
try:
|
|
434
|
+
with archive.open(info, "r") as source, target.open("wb") as output:
|
|
435
|
+
while True:
|
|
436
|
+
chunk = source.read(chunk_size)
|
|
437
|
+
if not chunk:
|
|
438
|
+
break
|
|
439
|
+
output.write(chunk)
|
|
440
|
+
completed += len(chunk)
|
|
441
|
+
for digest in digests.values():
|
|
442
|
+
digest.update(chunk)
|
|
443
|
+
if progress is not None:
|
|
444
|
+
progress(
|
|
445
|
+
DownloadProgress(
|
|
446
|
+
phase="extract",
|
|
447
|
+
query=assembly.query,
|
|
448
|
+
accession=assembly.accession,
|
|
449
|
+
bytes_completed=completed,
|
|
450
|
+
total_bytes=info.file_size,
|
|
451
|
+
current_file=PurePosixPath(info.filename).name,
|
|
452
|
+
)
|
|
453
|
+
)
|
|
454
|
+
except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
|
|
455
|
+
raise DownloadError(
|
|
456
|
+
"Could not extract the NCBI genome package.",
|
|
457
|
+
code="ARCHIVE_EXTRACTION_ERROR",
|
|
458
|
+
context={"member": info.filename},
|
|
459
|
+
) from exc
|
|
460
|
+
return {name: digest.hexdigest() for name, digest in digests.items()}
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _read_member_text(archive: zipfile.ZipFile, info: zipfile.ZipInfo) -> str:
|
|
464
|
+
try:
|
|
465
|
+
with archive.open(info, "r") as source:
|
|
466
|
+
payload = source.read(20_000_001)
|
|
467
|
+
except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
|
|
468
|
+
raise DownloadError(
|
|
469
|
+
"Could not read an NCBI package metadata file.",
|
|
470
|
+
code="ARCHIVE_EXTRACTION_ERROR",
|
|
471
|
+
context={"member": info.filename},
|
|
472
|
+
) from exc
|
|
473
|
+
if len(payload) > 20_000_000:
|
|
474
|
+
raise DownloadError(
|
|
475
|
+
"NCBI package metadata exceeds the configured limit.",
|
|
476
|
+
code="ARCHIVE_METADATA_SIZE_LIMIT",
|
|
477
|
+
context={"member": info.filename},
|
|
478
|
+
)
|
|
479
|
+
try:
|
|
480
|
+
return payload.decode("utf-8")
|
|
481
|
+
except UnicodeDecodeError as exc:
|
|
482
|
+
raise DownloadError(
|
|
483
|
+
"NCBI package metadata is not UTF-8.",
|
|
484
|
+
code="ARCHIVE_METADATA_ERROR",
|
|
485
|
+
context={"member": info.filename},
|
|
486
|
+
) from exc
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _extract_report_details(text: str, accession: str) -> tuple[str | None, str | None]:
|
|
490
|
+
for line in text.splitlines():
|
|
491
|
+
if not line.strip():
|
|
492
|
+
continue
|
|
493
|
+
try:
|
|
494
|
+
payload = json.loads(line)
|
|
495
|
+
except json.JSONDecodeError as exc:
|
|
496
|
+
raise DownloadError(
|
|
497
|
+
"NCBI assembly_data_report.jsonl is invalid JSON.",
|
|
498
|
+
code="ARCHIVE_METADATA_ERROR",
|
|
499
|
+
) from exc
|
|
500
|
+
if not isinstance(payload, Mapping):
|
|
501
|
+
continue
|
|
502
|
+
reported_accession = payload.get("accession")
|
|
503
|
+
assembly = payload.get("assembly")
|
|
504
|
+
organism = payload.get("organism")
|
|
505
|
+
if isinstance(assembly, Mapping):
|
|
506
|
+
reported_accession = assembly.get("accession", reported_accession)
|
|
507
|
+
if reported_accession is not None and reported_accession != accession:
|
|
508
|
+
continue
|
|
509
|
+
assembly_name = None
|
|
510
|
+
if isinstance(assembly, Mapping):
|
|
511
|
+
for key in ("name", "assembly_name"):
|
|
512
|
+
if isinstance(assembly.get(key), str) and assembly[key].strip():
|
|
513
|
+
assembly_name = assembly[key]
|
|
514
|
+
break
|
|
515
|
+
for key in ("assembly_name", "assminfo_name"):
|
|
516
|
+
if assembly_name is None and isinstance(payload.get(key), str):
|
|
517
|
+
assembly_name = payload[key]
|
|
518
|
+
organism_name = None
|
|
519
|
+
if isinstance(organism, Mapping):
|
|
520
|
+
for key in ("organism_name", "name"):
|
|
521
|
+
if isinstance(organism.get(key), str) and organism[key].strip():
|
|
522
|
+
organism_name = organism[key]
|
|
523
|
+
break
|
|
524
|
+
for key in ("organism_name", "organism"):
|
|
525
|
+
if organism_name is None and isinstance(payload.get(key), str):
|
|
526
|
+
organism_name = payload[key]
|
|
527
|
+
return organism_name, assembly_name
|
|
528
|
+
return None, None
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _expected_md5(text: str, member_name: str) -> str:
|
|
532
|
+
normalized_member = member_name.lstrip("./")
|
|
533
|
+
basename = PurePosixPath(member_name).name
|
|
534
|
+
for line in text.splitlines():
|
|
535
|
+
fields = line.strip().split(maxsplit=1)
|
|
536
|
+
if len(fields) != 2:
|
|
537
|
+
continue
|
|
538
|
+
digest, path = fields
|
|
539
|
+
normalized_path = path.lstrip("*").lstrip("./")
|
|
540
|
+
if (
|
|
541
|
+
normalized_path == normalized_member or PurePosixPath(normalized_path).name == basename
|
|
542
|
+
) and re.fullmatch(r"[0-9a-fA-F]{32}", digest):
|
|
543
|
+
return digest.lower()
|
|
544
|
+
raise DownloadError(
|
|
545
|
+
"NCBI md5sum.txt does not contain the genomic FASTA entry.",
|
|
546
|
+
code="CHECKSUM_MISSING",
|
|
547
|
+
context={"member": member_name},
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _install_staged(staged_to_target: Iterable[tuple[Path, Path]], *, overwrite: bool) -> None:
|
|
552
|
+
pairs = tuple(staged_to_target)
|
|
553
|
+
targets = tuple(target for _, target in pairs)
|
|
554
|
+
if len(targets) != len(set(targets)):
|
|
555
|
+
raise ConfigurationError("Download output paths collide.", code="OUTPUT_COLLISION")
|
|
556
|
+
existing = [target for target in targets if target.exists()]
|
|
557
|
+
if existing and not overwrite:
|
|
558
|
+
raise FileExistsError(f"Refusing to overwrite existing download outputs: {existing}")
|
|
559
|
+
if any(target.exists() and not target.is_file() for target in targets):
|
|
560
|
+
raise IsADirectoryError("Download output targets must be files.")
|
|
561
|
+
installed: list[Path] = []
|
|
562
|
+
backups: list[tuple[Path, Path]] = []
|
|
563
|
+
try:
|
|
564
|
+
if overwrite:
|
|
565
|
+
backup_paths = [
|
|
566
|
+
target.with_name(f".{target.name}.dnakit-backup-{index}")
|
|
567
|
+
for index, target in enumerate(targets)
|
|
568
|
+
if target.exists()
|
|
569
|
+
]
|
|
570
|
+
if any(path.exists() for path in backup_paths):
|
|
571
|
+
raise FileExistsError("A stale DNAKit download backup already exists.")
|
|
572
|
+
for index, (staged, target) in enumerate(pairs):
|
|
573
|
+
if overwrite:
|
|
574
|
+
if target.exists():
|
|
575
|
+
backup = target.with_name(f".{target.name}.dnakit-backup-{index}")
|
|
576
|
+
os.replace(target, backup)
|
|
577
|
+
backups.append((target, backup))
|
|
578
|
+
os.replace(staged, target)
|
|
579
|
+
else:
|
|
580
|
+
os.link(staged, target)
|
|
581
|
+
staged.unlink()
|
|
582
|
+
installed.append(target)
|
|
583
|
+
except BaseException:
|
|
584
|
+
for target in installed:
|
|
585
|
+
target.unlink(missing_ok=True)
|
|
586
|
+
for target, backup in reversed(backups):
|
|
587
|
+
if backup.exists():
|
|
588
|
+
os.replace(backup, target)
|
|
589
|
+
raise
|
|
590
|
+
for _, backup in backups:
|
|
591
|
+
backup.unlink(missing_ok=True)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def download_genome(
|
|
595
|
+
query: str,
|
|
596
|
+
output_dir: str | os.PathLike[str],
|
|
597
|
+
*,
|
|
598
|
+
overwrite: bool = False,
|
|
599
|
+
keep_package: bool = False,
|
|
600
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
601
|
+
api_key: str | None = None,
|
|
602
|
+
api_base_url: str = DEFAULT_NCBI_API_BASE_URL,
|
|
603
|
+
chunk_size: int = DEFAULT_CHUNK_SIZE,
|
|
604
|
+
max_download_bytes: int = DEFAULT_MAX_DOWNLOAD_BYTES,
|
|
605
|
+
progress: ProgressCallback | None = None,
|
|
606
|
+
) -> GenomeDownloadResult:
|
|
607
|
+
"""Download one NCBI genomic FASTA and its provenance files.
|
|
608
|
+
|
|
609
|
+
``query`` may be an explicit GCA_/GCF_ assembly accession, a fixed alias
|
|
610
|
+
such as ``hg38`` or ``mm39``, or a taxon name. Taxon names are resolved
|
|
611
|
+
to one current RefSeq reference assembly and fail when ambiguous.
|
|
612
|
+
"""
|
|
613
|
+
|
|
614
|
+
_validate_common(
|
|
615
|
+
timeout=timeout,
|
|
616
|
+
chunk_size=chunk_size,
|
|
617
|
+
max_download_bytes=max_download_bytes,
|
|
618
|
+
api_base_url=api_base_url,
|
|
619
|
+
api_key=api_key,
|
|
620
|
+
progress=progress,
|
|
621
|
+
)
|
|
622
|
+
if not isinstance(overwrite, bool) or not isinstance(keep_package, bool):
|
|
623
|
+
raise ConfigurationError(
|
|
624
|
+
"overwrite and keep_package must be booleans.", code="INVALID_DOWNLOAD_CONFIG"
|
|
625
|
+
)
|
|
626
|
+
assembly = resolve_genome_assembly(
|
|
627
|
+
query,
|
|
628
|
+
timeout=timeout,
|
|
629
|
+
api_key=api_key,
|
|
630
|
+
api_base_url=api_base_url,
|
|
631
|
+
)
|
|
632
|
+
if progress is not None:
|
|
633
|
+
progress(
|
|
634
|
+
DownloadProgress(
|
|
635
|
+
phase="resolve",
|
|
636
|
+
query=assembly.query,
|
|
637
|
+
accession=assembly.accession,
|
|
638
|
+
bytes_completed=0,
|
|
639
|
+
)
|
|
640
|
+
)
|
|
641
|
+
output = Path(output_dir).expanduser().resolve()
|
|
642
|
+
if output.exists() and not output.is_dir():
|
|
643
|
+
raise NotADirectoryError(f"output_dir must be a directory: {output}")
|
|
644
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
645
|
+
url = _join_url(
|
|
646
|
+
api_base_url,
|
|
647
|
+
f"/genome/accession/{quote(assembly.accession, safe='')}/download",
|
|
648
|
+
(("include_annotation_type", "GENOME_FASTA"), ("hydrated", "FULLY_HYDRATED")),
|
|
649
|
+
)
|
|
650
|
+
|
|
651
|
+
with tempfile.TemporaryDirectory(prefix=".dnakit-ncbi-", dir=str(output)) as raw_work:
|
|
652
|
+
work = Path(raw_work)
|
|
653
|
+
package = work / "ncbi_dataset.zip"
|
|
654
|
+
downloaded_bytes, package_sha256 = _download_package(
|
|
655
|
+
url,
|
|
656
|
+
package,
|
|
657
|
+
timeout=timeout,
|
|
658
|
+
api_key=api_key,
|
|
659
|
+
chunk_size=chunk_size,
|
|
660
|
+
max_download_bytes=max_download_bytes,
|
|
661
|
+
assembly=assembly,
|
|
662
|
+
progress=progress,
|
|
663
|
+
)
|
|
664
|
+
try:
|
|
665
|
+
with zipfile.ZipFile(package, "r") as archive:
|
|
666
|
+
infos = archive.infolist()
|
|
667
|
+
for info in infos:
|
|
668
|
+
_safe_member_name(info.filename)
|
|
669
|
+
fasta_info = next(
|
|
670
|
+
(
|
|
671
|
+
info
|
|
672
|
+
for info in infos
|
|
673
|
+
if PurePosixPath(info.filename).name.endswith("_genomic.fna")
|
|
674
|
+
),
|
|
675
|
+
None,
|
|
676
|
+
)
|
|
677
|
+
if fasta_info is None:
|
|
678
|
+
raise DownloadError(
|
|
679
|
+
"NCBI package does not contain a genomic FASTA file.",
|
|
680
|
+
code="INVALID_ARCHIVE",
|
|
681
|
+
)
|
|
682
|
+
fasta_member_name = _validate_member(fasta_info)
|
|
683
|
+
report_info = _find_member(infos, "assembly_data_report.jsonl")
|
|
684
|
+
checksum_info = _find_member(infos, "md5sum.txt")
|
|
685
|
+
report_text = _read_member_text(archive, report_info)
|
|
686
|
+
checksum_text = _read_member_text(archive, checksum_info)
|
|
687
|
+
expected_md5 = _expected_md5(checksum_text, fasta_member_name)
|
|
688
|
+
staged_fasta = work / PurePosixPath(fasta_member_name).name
|
|
689
|
+
digests = _copy_member(
|
|
690
|
+
archive,
|
|
691
|
+
fasta_info,
|
|
692
|
+
staged_fasta,
|
|
693
|
+
chunk_size=chunk_size,
|
|
694
|
+
assembly=assembly,
|
|
695
|
+
progress=progress,
|
|
696
|
+
hash_algorithms=("md5", "sha256"),
|
|
697
|
+
)
|
|
698
|
+
if digests["md5"] != expected_md5:
|
|
699
|
+
raise DownloadError(
|
|
700
|
+
"The downloaded genomic FASTA failed the NCBI MD5 check.",
|
|
701
|
+
code="CHECKSUM_MISMATCH",
|
|
702
|
+
context={"expected_md5": expected_md5, "actual_md5": digests["md5"]},
|
|
703
|
+
)
|
|
704
|
+
organism, assembly_name = _extract_report_details(report_text, assembly.accession)
|
|
705
|
+
staged_report = work / f"{assembly.accession}_assembly_data_report.jsonl"
|
|
706
|
+
staged_report.write_text(report_text, encoding="utf-8")
|
|
707
|
+
staged_checksum = work / f"{assembly.accession}_md5sum.txt"
|
|
708
|
+
staged_checksum.write_text(checksum_text, encoding="utf-8")
|
|
709
|
+
staged_package = package
|
|
710
|
+
fasta_target = output / staged_fasta.name
|
|
711
|
+
report_target = output / staged_report.name
|
|
712
|
+
checksum_target = output / staged_checksum.name
|
|
713
|
+
package_target = output / f"{assembly.accession}_ncbi_dataset.zip"
|
|
714
|
+
pairs: list[tuple[Path, Path]] = [
|
|
715
|
+
(staged_fasta, fasta_target),
|
|
716
|
+
(staged_report, report_target),
|
|
717
|
+
(staged_checksum, checksum_target),
|
|
718
|
+
]
|
|
719
|
+
if keep_package:
|
|
720
|
+
pairs.append((staged_package, package_target))
|
|
721
|
+
_install_staged(pairs, overwrite=overwrite)
|
|
722
|
+
except DownloadError:
|
|
723
|
+
raise
|
|
724
|
+
except (OSError, RuntimeError, zipfile.BadZipFile) as exc:
|
|
725
|
+
raise DownloadError(
|
|
726
|
+
"The NCBI genome package is invalid or could not be extracted.",
|
|
727
|
+
code="INVALID_ARCHIVE",
|
|
728
|
+
context={"accession": assembly.accession},
|
|
729
|
+
) from exc
|
|
730
|
+
|
|
731
|
+
return GenomeDownloadResult(
|
|
732
|
+
query=assembly.query,
|
|
733
|
+
accession=assembly.accession,
|
|
734
|
+
organism=organism,
|
|
735
|
+
assembly_name=assembly_name,
|
|
736
|
+
output_directory=str(output),
|
|
737
|
+
fasta_path=str(fasta_target),
|
|
738
|
+
metadata_path=str(report_target),
|
|
739
|
+
checksum_path=str(checksum_target),
|
|
740
|
+
package_path=str(package_target) if keep_package else None,
|
|
741
|
+
download_url=url,
|
|
742
|
+
downloaded_bytes=downloaded_bytes,
|
|
743
|
+
package_sha256=package_sha256,
|
|
744
|
+
fasta_sha256=digests["sha256"],
|
|
745
|
+
fasta_md5=digests["md5"],
|
|
746
|
+
provenance=Provenance(
|
|
747
|
+
implementation=ImplementationInfo(
|
|
748
|
+
label=ImplementationLabel.ADAPTER,
|
|
749
|
+
execution_mode=ExecutionMode.EXTERNAL,
|
|
750
|
+
origin_class=OriginClass.INTEGRATION,
|
|
751
|
+
citations=(
|
|
752
|
+
Citation(
|
|
753
|
+
"ncbi-datasets",
|
|
754
|
+
title="NCBI Datasets Genome Package",
|
|
755
|
+
url=(
|
|
756
|
+
"https://www.ncbi.nlm.nih.gov/datasets/docs/v2/"
|
|
757
|
+
"reference-docs/data-packages/genome/"
|
|
758
|
+
),
|
|
759
|
+
),
|
|
760
|
+
),
|
|
761
|
+
),
|
|
762
|
+
reference=ReferenceInfo(
|
|
763
|
+
name="NCBI genome assembly",
|
|
764
|
+
version=assembly.accession,
|
|
765
|
+
checksum=digests["sha256"],
|
|
766
|
+
filters={
|
|
767
|
+
"query": assembly.query,
|
|
768
|
+
"include_annotation_type": "GENOME_FASTA",
|
|
769
|
+
"hydrated": "FULLY_HYDRATED",
|
|
770
|
+
},
|
|
771
|
+
),
|
|
772
|
+
),
|
|
773
|
+
)
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
__all__ = [
|
|
777
|
+
"DEFAULT_CHUNK_SIZE",
|
|
778
|
+
"DEFAULT_MAX_DOWNLOAD_BYTES",
|
|
779
|
+
"DEFAULT_NCBI_API_BASE_URL",
|
|
780
|
+
"DEFAULT_TIMEOUT",
|
|
781
|
+
"ProgressCallback",
|
|
782
|
+
"download_genome",
|
|
783
|
+
"resolve_genome_assembly",
|
|
784
|
+
"supported_genome_aliases",
|
|
785
|
+
]
|