dnakit 0.1.0.dev0__py3-none-any.whl

dnakit/download/catalogs.py

@@ -0,0 +1,166 @@
+"""Curated public ClinVar and GEO download locations."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from pathlib import Path
+
+from dnakit.exceptions import ConfigurationError
+
+from .files import dataset
+from .models import DatasetDownloadResult, DownloadConfig, DownloadProgress, RemoteFile
+
+_GEO_ACCESSION = re.compile(r"^(GSE|GDS|GPL|GSM)(\d+)$", flags=re.IGNORECASE)
+
+
+def variants(
+    output_dir: str | Path,
+    *,
+    source: str = "clinvar",
+    format: str = "vcf",
+    assembly: str = "GRCh38",
+    include_index: bool = True,
+    include_checksum_files: bool = True,
+    config: DownloadConfig | None = None,
+    progress: Callable[[DownloadProgress], None] | None = None,
+) -> DatasetDownloadResult:
+    """Download current public ClinVar VCF, XML, or variant-summary TSV."""
+
+    source_value = source.casefold()
+    if source_value not in {"clinvar", "dbsnp"}:
+        raise ConfigurationError(
+            "variants source must be clinvar or dbsnp; use dataset() for other archives."
+        )
+    kind = format.casefold()
+    resources: list[RemoteFile]
+    if source_value == "dbsnp":
+        if kind != "vcf":
+            raise ConfigurationError("dbSNP downloads currently use format='vcf'.")
+        accessions = {
+            "GRCh37": "GCF_000001405.25",
+            "GRCh38": "GCF_000001405.40",
+        }
+        accession = accessions.get(assembly)
+        if accession is None:
+            raise ConfigurationError("dbSNP VCF assembly must be GRCh37 or GRCh38.")
+        base = "https://ftp.ncbi.nih.gov/snp/latest_release/VCF"
+        resources = [RemoteFile(f"{base}/{accession}.gz", filename=f"dbsnp_{assembly}.vcf.gz")]
+        if include_index:
+            resources.append(
+                RemoteFile(f"{base}/{accession}.gz.tbi", filename=f"dbsnp_{assembly}.vcf.gz.tbi")
+            )
+        if include_checksum_files:
+            resources.append(
+                RemoteFile(f"{base}/{accession}.gz.md5", filename=f"dbsnp_{assembly}.vcf.gz.md5")
+            )
+            if include_index:
+                resources.append(
+                    RemoteFile(
+                        f"{base}/{accession}.gz.tbi.md5",
+                        filename=f"dbsnp_{assembly}.vcf.gz.tbi.md5",
+                    )
+                )
+    elif kind == "vcf":
+        if assembly not in {"GRCh37", "GRCh38"}:
+            raise ConfigurationError("ClinVar VCF assembly must be GRCh37 or GRCh38.")
+        base = f"https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_{assembly}"
+        resources = [RemoteFile(f"{base}/clinvar.vcf.gz", filename=f"clinvar_{assembly}.vcf.gz")]
+        if include_index:
+            resources.append(
+                RemoteFile(f"{base}/clinvar.vcf.gz.tbi", filename=f"clinvar_{assembly}.vcf.gz.tbi")
+            )
+    elif kind == "xml":
+        resources = [
+            RemoteFile(
+                "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarVCVRelease_00-latest.xml.gz",
+                filename="ClinVarVCVRelease_00-latest.xml.gz",
+            )
+        ]
+    elif kind in {"tsv", "summary"}:
+        resources = [
+            RemoteFile(
+                "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz",
+                filename="clinvar_variant_summary.txt.gz",
+            )
+        ]
+    else:
+        raise ConfigurationError("ClinVar format must be vcf, xml, or tsv.")
+    return dataset(
+        resources,
+        output_dir,
+        kind=f"{source_value}_{kind}",
+        source="NCBI ClinVar" if source_value == "clinvar" else "NCBI dbSNP",
+        metadata={
+            "assembly": assembly if kind == "vcf" else None,
+            "format": kind,
+            "provider_checksum_files_included": include_checksum_files
+            if source_value == "dbsnp"
+            else False,
+        },
+        config=config,
+        progress=progress,
+    )
+
+
+def _geo_range(accession: str) -> tuple[str, str, str]:
+    match = _GEO_ACCESSION.fullmatch(accession.strip())
+    if match is None:
+        raise ConfigurationError("GEO accession must start with GSE, GDS, GPL, or GSM.")
+    prefix = match.group(1).upper()
+    digits = match.group(2)
+    range_name = f"{prefix}{digits[:-3]}nnn" if len(digits) > 3 else f"{prefix}nnn"
+    category = {"GSE": "series", "GDS": "datasets", "GPL": "platforms", "GSM": "samples"}[prefix]
+    return prefix + digits, category, range_name
+
+
+def expression(
+    accession: str,
+    output_dir: str | Path,
+    *,
+    format: str = "soft",
+    platform: str | None = None,
+    config: DownloadConfig | None = None,
+    progress: Callable[[DownloadProgress], None] | None = None,
+) -> DatasetDownloadResult:
+    """Download a standard GEO SOFT, MINiML, or common Series RAW archive."""
+
+    value, category, range_name = _geo_range(accession)
+    kind = format.casefold()
+    root = f"https://ftp.ncbi.nlm.nih.gov/geo/{category}/{range_name}/{value}"
+    if kind == "soft":
+        url = f"{root}/soft/{value}_family.soft.gz"
+        filename = f"{value}_family.soft.gz"
+    elif kind == "miniml":
+        url = f"{root}/miniml/{value}_family.xml.tgz"
+        filename = f"{value}_family.xml.tgz"
+    elif kind in {"raw", "supplementary"}:
+        if not value.startswith("GSE"):
+            raise ConfigurationError("The common RAW archive convention applies to GSE accessions.")
+        url = f"{root}/suppl/{value}_RAW.tar"
+        filename = f"{value}_RAW.tar"
+    elif kind in {"matrix", "series_matrix"}:
+        if not value.startswith("GSE"):
+            raise ConfigurationError("Series Matrix downloads require a GSE accession.")
+        platform_value: str | None = None
+        if platform is not None:
+            platform_value, _, _ = _geo_range(platform)
+            if not platform_value.startswith("GPL"):
+                raise ConfigurationError("platform must be a GPL accession.")
+        stem = value if platform_value is None else f"{value}-{platform_value}"
+        filename = f"{stem}_series_matrix.txt.gz"
+        url = f"{root}/matrix/{filename}"
+    else:
+        raise ConfigurationError("GEO format must be soft, miniml, raw, or matrix.")
+    return dataset(
+        (RemoteFile(url, filename=filename),),
+        output_dir,
+        kind=f"geo_{kind}",
+        source="NCBI GEO",
+        metadata={"accession": value, "format": kind, "platform": platform},
+        config=config,
+        progress=progress,
+    )
+
+
+__all__ = ["expression", "variants"]

dnakit/download/ena.py

@@ -0,0 +1,131 @@
+"""Download public read files discovered through ENA metadata."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Mapping
+from pathlib import Path, PurePosixPath
+from urllib.parse import urlsplit
+
+from dnakit.exceptions import ConfigurationError, DownloadError
+from dnakit.search import QueryResult, SearchConfig
+from dnakit.search.ena import DEFAULT_ENA_PORTAL_URL, DEFAULT_READ_FIELDS
+from dnakit.search.ena import reads as search_reads
+
+from .files import dataset, resolved_config
+from .models import DatasetDownloadResult, DownloadConfig, DownloadProgress, RemoteFile
+
+_RUN_ACCESSION = re.compile(r"^[SED]RR\d+$", flags=re.IGNORECASE)
+
+
+def _ena_https(value: str) -> str:
+    item = value.strip()
+    if item.startswith("ftp://"):
+        return "https://" + item[len("ftp://") :]
+    if item.startswith("https://"):
+        return item
+    if "://" not in item:
+        return "https://" + item.lstrip("/")
+    raise ConfigurationError(
+        "ENA file URL uses an unsupported scheme.", code="INVALID_DOWNLOAD_URL"
+    )
+
+
+def _parts(record: object, key: str) -> tuple[str, ...]:
+    if not isinstance(record, Mapping):
+        return ()
+    raw = record.get(key)
+    if not isinstance(raw, str) or not raw.strip():
+        return ()
+    return tuple(item.strip() for item in raw.split(";") if item.strip())
+
+
+def reads(
+    query: str | QueryResult,
+    output_dir: str | Path,
+    *,
+    file_kind: str = "fastq",
+    config: DownloadConfig | None = None,
+    progress: Callable[[DownloadProgress], None] | None = None,
+    ena_api_base_url: str = DEFAULT_ENA_PORTAL_URL,
+) -> DatasetDownloadResult:
+    """Download public ENA FASTQ, submitted, or SRA files with advertised MD5 checks."""
+
+    resolved = resolved_config(config)
+    if file_kind not in {"fastq", "submitted", "sra"}:
+        raise ConfigurationError("file_kind must be fastq, submitted, or sra.")
+    if isinstance(query, QueryResult):
+        result = query
+        if result.provider != "ENA" or result.query_type != "reads":
+            raise ConfigurationError("QueryResult must come from dnakit.search.reads().")
+    elif isinstance(query, str):
+        expression = (
+            f'run_accession="{query}"' if _RUN_ACCESSION.fullmatch(query.strip()) else query
+        )
+        fields = tuple(DEFAULT_READ_FIELDS)
+        if file_kind == "sra":
+            fields += ("sra_ftp", "sra_md5", "sra_bytes")
+        result = search_reads(
+            expression,
+            fields=fields,
+            limit=resolved.max_files,
+            config=SearchConfig(
+                timeout=min(float(resolved.timeout), 300.0),
+                max_response_bytes=min(resolved.max_file_bytes, 100_000_000),
+                max_records=resolved.max_files,
+            ),
+            api_base_url=ena_api_base_url,
+        )
+    else:
+        raise TypeError("query must be ENA query text, run accession, or QueryResult.")
+    url_key = f"{file_kind}_ftp"
+    md5_key = f"{file_kind}_md5"
+    bytes_key = f"{file_kind}_bytes"
+    resources: list[RemoteFile] = []
+    expected_sizes: dict[str, int | None] = {}
+    for record in result.records:
+        urls = _parts(record, url_key)
+        md5s = _parts(record, md5_key)
+        sizes = _parts(record, bytes_key)
+        if md5s and len(md5s) != len(urls):
+            raise DownloadError(
+                "ENA returned mismatched file and MD5 lists.", code="ENA_METADATA_MISMATCH"
+            )
+        if sizes and len(sizes) != len(urls):
+            raise DownloadError(
+                "ENA returned mismatched file and size lists.", code="ENA_METADATA_MISMATCH"
+            )
+        for index, raw_url in enumerate(urls):
+            url = _ena_https(raw_url)
+            filename = PurePosixPath(urlsplit(url).path).name
+            checksum = md5s[index] if md5s else None
+            resources.append(RemoteFile(url, filename=filename, expected_md5=checksum))
+            try:
+                expected_sizes[filename] = int(sizes[index]) if sizes else None
+            except ValueError as exc:
+                raise DownloadError(
+                    "ENA returned an invalid file size.", code="ENA_METADATA_MISMATCH"
+                ) from exc
+    if not resources:
+        raise DownloadError(
+            f"ENA query returned no public {file_kind} file URLs.",
+            code="ENA_FILES_NOT_FOUND",
+        )
+    if len(resources) > resolved.max_files:
+        raise DownloadError("ENA files exceed max_files.", code="DOWNLOAD_FILE_LIMIT")
+    return dataset(
+        resources,
+        output_dir,
+        kind=f"ena_{file_kind}",
+        source="European Nucleotide Archive",
+        metadata={
+            "query_url": result.request_url,
+            "file_kind": file_kind,
+            "expected_sizes": expected_sizes,
+        },
+        config=resolved,
+        progress=progress,
+    )
+
+
+__all__ = ["reads"]

dnakit/download/encode.py

@@ -0,0 +1,103 @@
+"""Download public files discovered through ENCODE Portal search results."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from pathlib import Path, PurePosixPath
+from urllib.parse import quote, urlsplit
+
+from dnakit.exceptions import ConfigurationError, DownloadError
+from dnakit.search import QueryResult
+from dnakit.search._http import require_https_base
+
+from .files import dataset, resolved_config
+from .models import DatasetDownloadResult, DownloadConfig, DownloadProgress, RemoteFile
+
+DEFAULT_ENCODE_DOWNLOAD_URL = "https://www.encodeproject.org"
+_MD5 = re.compile(r"^[0-9a-fA-F]{32}$")
+
+
+def _url(value: str, base_url: str) -> str:
+    raw = value.strip()
+    parsed = urlsplit(raw)
+    if parsed.scheme:
+        if parsed.scheme != "https" or not parsed.netloc or parsed.fragment:
+            raise ConfigurationError("ENCODE file URL must use HTTPS.")
+        return raw
+    relative = raw.lstrip("/")
+    if not relative or ".." in PurePosixPath(relative).parts:
+        raise ConfigurationError("ENCODE file path is unsafe.")
+    return f"{require_https_base(base_url)}/{quote(relative, safe='/._-@')}"
+
+
+def files(
+    query: QueryResult,
+    output_dir: str | Path,
+    *,
+    config: DownloadConfig | None = None,
+    progress: Callable[[DownloadProgress], None] | None = None,
+    download_base_url: str = DEFAULT_ENCODE_DOWNLOAD_URL,
+) -> DatasetDownloadResult:
+    """Download public ENCODE file records, verifying advertised MD5 values when present."""
+
+    if not isinstance(query, QueryResult) or query.provider != "ENCODE":
+        raise ConfigurationError("query must be a dnakit.search.encode_search() result.")
+    resolved = resolved_config(config)
+    resources: list[RemoteFile] = []
+    expected_total = 0
+    selected: list[dict[str, object]] = []
+    for record in query.records:
+        href = record.get("href")
+        if not isinstance(href, str) or not href.strip():
+            continue
+        url = _url(href, download_base_url)
+        filename = PurePosixPath(urlsplit(url).path).name
+        if not filename:
+            raise DownloadError("ENCODE file URL has no filename.", code="QUERY_RESPONSE_ERROR")
+        raw_md5 = record.get("md5sum")
+        expected_md5 = raw_md5 if isinstance(raw_md5, str) and _MD5.fullmatch(raw_md5) else None
+        raw_size = record.get("file_size")
+        if isinstance(raw_size, int) and not isinstance(raw_size, bool) and raw_size >= 0:
+            if raw_size > resolved.max_file_bytes:
+                raise DownloadError(
+                    "ENCODE file exceeds max_file_bytes.", code="DOWNLOAD_SIZE_LIMIT"
+                )
+            expected_total += raw_size
+        resources.append(RemoteFile(url, filename=filename, expected_md5=expected_md5))
+        selected.append(
+            {
+                "accession": record.get("accession"),
+                "file_format": record.get("file_format"),
+                "output_type": record.get("output_type"),
+                "href": href,
+                "provider_md5_available": expected_md5 is not None,
+            }
+        )
+    if not resources:
+        raise DownloadError(
+            "ENCODE query contains no downloadable file records; search object_type='File'.",
+            code="ENCODE_FILES_NOT_FOUND",
+        )
+    if len(resources) > resolved.max_files:
+        raise DownloadError("ENCODE files exceed max_files.", code="DOWNLOAD_FILE_LIMIT")
+    if expected_total > resolved.max_total_bytes:
+        raise DownloadError(
+            "ENCODE files exceed max_total_bytes.", code="DOWNLOAD_TOTAL_SIZE_LIMIT"
+        )
+    return dataset(
+        resources,
+        output_dir,
+        kind="encode_files",
+        source="ENCODE Portal",
+        metadata={
+            "query_url": query.request_url,
+            "expected_total_bytes": expected_total,
+            "selected_files": selected,
+        },
+        config=resolved,
+        progress=progress,
+    )
+
+
+__all__ = ["DEFAULT_ENCODE_DOWNLOAD_URL", "files"]