dnakit 0.1.0.dev0__py3-none-any.whl

dnakit/download/metadata.py

@@ -0,0 +1,164 @@
+"""Atomically export query metadata to common interchange formats."""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import io
+import json
+import os
+import tempfile
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Literal, cast
+
+from dnakit.core._json import to_json_compatible
+from dnakit.exceptions import ConfigurationError, DownloadError
+from dnakit.search import QueryResult
+
+from .files import _install_staged_files, _write_manifest, resolved_config
+from .models import DatasetDownloadResult, DownloadConfig, DownloadedFile
+
+MetadataFormat = Literal["json", "jsonl", "csv", "tsv", "xml"]
+
+
+def _format(value: str | None, path: Path) -> MetadataFormat:
+    selected = path.suffix.lstrip(".").casefold() if value is None else value.casefold()
+    if selected not in {"json", "jsonl", "csv", "tsv", "xml"}:
+        raise ConfigurationError("Metadata format must be json, jsonl, csv, tsv, or xml.")
+    return cast(MetadataFormat, selected)
+
+
+def _cell(value: object) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (str, int, float)):
+        return str(value)
+    return json.dumps(to_json_compatible(value), ensure_ascii=False, sort_keys=True)
+
+
+def _tabular(result: QueryResult, *, delimiter: str) -> bytes:
+    fields = sorted({str(key) for record in result.records for key in record})
+    stream = io.StringIO(newline="")
+    writer = csv.writer(stream, delimiter=delimiter, lineterminator="\n")
+    if fields:
+        writer.writerow(fields)
+        for record in result.records:
+            writer.writerow(_cell(record.get(field)) for field in fields)
+    return stream.getvalue().encode("utf-8")
+
+
+def _xml(result: QueryResult) -> bytes:
+    root = ET.Element(
+        "dnakit-query",
+        {
+            "provider": result.provider,
+            "query-type": result.query_type,
+            "request-url": result.request_url,
+        },
+    )
+    for index, record in enumerate(result.records):
+        node = ET.SubElement(root, "record", {"index": str(index)})
+        for key in sorted(record):
+            field = ET.SubElement(node, "field", {"name": str(key)})
+            field.text = _cell(record[key])
+    return cast(bytes, ET.tostring(root, encoding="utf-8", xml_declaration=True)) + b"\n"
+
+
+def _payload(result: QueryResult, output_format: MetadataFormat) -> bytes:
+    if output_format == "json":
+        return (
+            json.dumps(result.to_dict(), ensure_ascii=False, sort_keys=True, indent=2) + "\n"
+        ).encode("utf-8")
+    if output_format == "jsonl":
+        return (
+            "".join(
+                json.dumps(to_json_compatible(record), ensure_ascii=False, sort_keys=True) + "\n"
+                for record in result.records
+            )
+        ).encode("utf-8")
+    if output_format in {"csv", "tsv"}:
+        return _tabular(result, delimiter="," if output_format == "csv" else "\t")
+    return _xml(result)
+
+
+def metadata(
+    query: QueryResult,
+    output_path: str | os.PathLike[str],
+    *,
+    format: MetadataFormat | None = None,
+    config: DownloadConfig | None = None,
+) -> DatasetDownloadResult:
+    """Export a ``QueryResult`` as JSON/JSONL/CSV/TSV/XML plus an audit manifest."""
+
+    if not isinstance(query, QueryResult):
+        raise TypeError("query must be QueryResult.")
+    resolved = resolved_config(config)
+    target = Path(output_path).expanduser().resolve()
+    target.parent.mkdir(parents=True, exist_ok=True)
+    if target.exists() and not target.is_file():
+        raise IsADirectoryError(f"Metadata output must be a file: {target}")
+    manifest = target.with_name(f"{target.name}.manifest.json")
+    if manifest.exists() and not manifest.is_file():
+        raise IsADirectoryError(f"Metadata manifest must be a file: {manifest}")
+    if not resolved.overwrite and (target.exists() or manifest.exists()):
+        raise FileExistsError("Refusing to overwrite existing metadata output or manifest.")
+    output_format = _format(format, target)
+    payload = _payload(query, output_format)
+    if len(payload) > resolved.max_file_bytes:
+        raise DownloadError("Metadata output exceeds max_file_bytes.", code="DOWNLOAD_SIZE_LIMIT")
+    md5 = hashlib.md5(payload, usedforsecurity=False).hexdigest()
+    sha256 = hashlib.sha256(payload).hexdigest()
+    item = DownloadedFile(
+        query.request_url,
+        str(target),
+        len(payload),
+        md5,
+        sha256,
+        False,
+    )
+    with tempfile.TemporaryDirectory(prefix=".dnakit-metadata-", dir=str(target.parent)) as raw:
+        stage = Path(raw)
+        staged_target = stage / target.name
+        staged_target.write_bytes(payload)
+        staged_manifest = stage / manifest.name
+        _write_manifest(
+            {
+                "kind": "query_metadata",
+                "source": query.provider,
+                "query_type": query.query_type,
+                "request_url": query.request_url,
+                "returned_records": len(query.records),
+                "reported_total_count": query.total_count,
+                "format": output_format,
+                "file": {
+                    "path": str(target),
+                    "byte_size": len(payload),
+                    "md5": md5,
+                    "sha256": sha256,
+                },
+                "query_metadata": to_json_compatible(query.metadata),
+                "provenance": query.provenance.to_dict(),
+            },
+            staged_manifest,
+            overwrite=False,
+        )
+        _install_staged_files(
+            (staged_target, staged_manifest),
+            (target, manifest),
+            overwrite=resolved.overwrite,
+        )
+    return DatasetDownloadResult(
+        "query_metadata",
+        query.provider,
+        str(target.parent),
+        (item,),
+        str(manifest),
+        query.provenance,
+        metadata={"format": output_format, "query_type": query.query_type},
+    )
+
+
+__all__ = ["MetadataFormat", "metadata"]

dnakit/download/models.py

@@ -0,0 +1,227 @@
+"""Immutable configuration and manifests for public data downloads."""
+
+from __future__ import annotations
+
+import math
+import re
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from typing import Any, cast
+
+from dnakit.core import Provenance
+from dnakit.core._json import FrozenDict, freeze_mapping, to_json_compatible
+from dnakit.exceptions import ConfigurationError
+
+_MD5 = re.compile(r"^[0-9a-fA-F]{32}$")
+_SHA256 = re.compile(r"^[0-9a-fA-F]{64}$")
+
+
+@dataclass(frozen=True, slots=True)
+class DownloadConfig:
+    """Resource limits and credentials for bounded public downloads."""
+
+    timeout: float = 60.0
+    chunk_size: int = 1_048_576
+    max_file_bytes: int = 20_000_000_000
+    max_total_bytes: int = 50_000_000_000
+    max_files: int = 100
+    overwrite: bool = False
+    api_key: str | None = None
+
+    def __post_init__(self) -> None:
+        if (
+            isinstance(self.timeout, bool)
+            or not isinstance(self.timeout, (int, float))
+            or not math.isfinite(self.timeout)
+            or not 0 < self.timeout <= 3_600
+        ):
+            raise ConfigurationError(
+                "DownloadConfig timeout must be in (0, 3600].",
+                code="INVALID_DOWNLOAD_CONFIG",
+            )
+        for name, value, maximum in (
+            ("chunk_size", self.chunk_size, 64 * 1024 * 1024),
+            ("max_file_bytes", self.max_file_bytes, 10**12),
+            ("max_total_bytes", self.max_total_bytes, 10**12),
+            ("max_files", self.max_files, 10_000),
+        ):
+            if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= maximum:
+                raise ConfigurationError(
+                    f"DownloadConfig {name} must be in [1, {maximum}].",
+                    code="INVALID_DOWNLOAD_CONFIG",
+                )
+        if self.max_total_bytes < self.max_file_bytes:
+            raise ConfigurationError(
+                "DownloadConfig max_total_bytes cannot be smaller than max_file_bytes.",
+                code="INVALID_DOWNLOAD_CONFIG",
+            )
+        if not isinstance(self.overwrite, bool):
+            raise ConfigurationError(
+                "DownloadConfig overwrite must be boolean.", code="INVALID_DOWNLOAD_CONFIG"
+            )
+        if self.api_key is not None and (
+            not isinstance(self.api_key, str) or not self.api_key.strip()
+        ):
+            raise ConfigurationError(
+                "DownloadConfig api_key must be None or non-empty text.",
+                code="INVALID_DOWNLOAD_CONFIG",
+            )
+
+
+@dataclass(frozen=True, slots=True)
+class DownloadProgress:
+    """One progress event for a streamed public file download."""
+
+    url: str
+    target: str
+    file_index: int
+    file_count: int
+    bytes_completed: int
+    total_bytes: int | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class RemoteFile:
+    """One explicit HTTPS resource and optional integrity expectations."""
+
+    url: str
+    filename: str | None = None
+    expected_md5: str | None = None
+    expected_sha256: str | None = None
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.url, str) or not self.url.strip():
+            raise ConfigurationError("RemoteFile url must be non-empty text.")
+        if self.filename is not None and (
+            not isinstance(self.filename, str) or not self.filename.strip()
+        ):
+            raise ConfigurationError("RemoteFile filename must be None or non-empty text.")
+        if self.expected_md5 is not None and _MD5.fullmatch(self.expected_md5) is None:
+            raise ConfigurationError("RemoteFile expected_md5 must contain 32 hexadecimal digits.")
+        if self.expected_sha256 is not None and _SHA256.fullmatch(self.expected_sha256) is None:
+            raise ConfigurationError(
+                "RemoteFile expected_sha256 must contain 64 hexadecimal digits."
+            )
+
+
+@dataclass(frozen=True, slots=True)
+class DownloadedFile:
+    """Integrity metadata for one atomically installed file."""
+
+    url: str
+    path: str
+    byte_size: int
+    md5: str
+    sha256: str
+    checksum_verified: bool
+
+    def __post_init__(self) -> None:
+        if (
+            isinstance(self.byte_size, bool)
+            or not isinstance(self.byte_size, int)
+            or self.byte_size < 0
+        ):
+            raise ConfigurationError("DownloadedFile byte_size is invalid.")
+        if _MD5.fullmatch(self.md5) is None or _SHA256.fullmatch(self.sha256) is None:
+            raise ConfigurationError("DownloadedFile checksums are invalid.")
+
+
+@dataclass(frozen=True, init=False)
+class DatasetDownloadResult:
+    """Auditable files and manifest produced by one logical download."""
+
+    kind: str
+    source: str
+    output_directory: str
+    files: tuple[DownloadedFile, ...]
+    manifest_path: str
+    metadata: FrozenDict
+    provenance: Provenance
+
+    def __init__(
+        self,
+        kind: str,
+        source: str,
+        output_directory: str,
+        files: Iterable[DownloadedFile],
+        manifest_path: str,
+        provenance: Provenance,
+        *,
+        metadata: Mapping[str, object] | None = None,
+    ) -> None:
+        for name, value in (
+            ("kind", kind),
+            ("source", source),
+            ("output_directory", output_directory),
+            ("manifest_path", manifest_path),
+        ):
+            if not isinstance(value, str) or not value.strip():
+                raise ConfigurationError(f"DatasetDownloadResult {name} must be non-empty text.")
+        resolved_files = tuple(files)
+        if any(not isinstance(item, DownloadedFile) for item in resolved_files):
+            raise ConfigurationError("DatasetDownloadResult files must be DownloadedFile objects.")
+        if not isinstance(provenance, Provenance):
+            raise ConfigurationError("DatasetDownloadResult provenance must be Provenance.")
+        object.__setattr__(self, "kind", kind)
+        object.__setattr__(self, "source", source)
+        object.__setattr__(self, "output_directory", output_directory)
+        object.__setattr__(self, "files", resolved_files)
+        object.__setattr__(self, "manifest_path", manifest_path)
+        object.__setattr__(self, "metadata", freeze_mapping(metadata))
+        object.__setattr__(self, "provenance", provenance)
+
+    @property
+    def downloaded_bytes(self) -> int:
+        return sum(item.byte_size for item in self.files)
+
+    def to_dict(self) -> dict[str, Any]:
+        return cast(dict[str, Any], to_json_compatible(self))
+
+
+@dataclass(frozen=True, slots=True)
+class IndexArtifact:
+    """One locally generated sequence-index artifact."""
+
+    path: str
+    byte_size: int
+    sha256: str
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.path, str) or not self.path.strip():
+            raise ConfigurationError("IndexArtifact path must be non-empty text.")
+        if (
+            isinstance(self.byte_size, bool)
+            or not isinstance(self.byte_size, int)
+            or self.byte_size < 0
+        ):
+            raise ConfigurationError("IndexArtifact byte_size is invalid.")
+        if _SHA256.fullmatch(self.sha256) is None:
+            raise ConfigurationError("IndexArtifact sha256 is invalid.")
+
+
+@dataclass(frozen=True, slots=True)
+class IndexBuildResult:
+    """Auditable artifacts produced by an explicit external index tool."""
+
+    tool: str
+    fasta_path: str
+    output_prefix: str
+    artifacts: tuple[IndexArtifact, ...]
+    command_output: str
+    elapsed_seconds: float
+    manifest_path: str
+    provenance: Provenance
+
+    def to_dict(self) -> dict[str, Any]:
+        return cast(dict[str, Any], to_json_compatible(self))
+
+
+__all__ = [
+    "DatasetDownloadResult",
+    "DownloadConfig",
+    "DownloadProgress",
+    "DownloadedFile",
+    "IndexArtifact",
+    "IndexBuildResult",
+    "RemoteFile",
+]