dnakit 0.1.0.dev0__py3-none-any.whl

dnakit/fingerprints/sketch.py

@@ -0,0 +1,246 @@
+"""Deterministic bottom-k MinHash and FracMinHash sketches."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Iterable
+
+from dnakit.core import Gap, Topology
+from dnakit.core._json import FrozenDict
+from dnakit.exceptions import ConfigurationError, UnsupportedGapOperationError
+
+from ._shared import SequenceInput, sequence_and_id, validate_bool, validate_positive_int
+from .results import SketchResult
+
+DEFAULT_MAX_SKETCH_HASHES = 1_000_000
+DEFAULT_MAX_UNIQUE_HASHES = 1_000_000
+_UINT64_SPACE = 1 << 64
+_COMPLEMENT = str.maketrans("ACGT", "TGCA")
+
+
+def _hash64(word: str, seed: int) -> int:
+    payload = seed.to_bytes(8, "little", signed=False) + word.encode("ascii")
+    return int.from_bytes(hashlib.sha256(payload).digest()[:8], "big")
+
+
+def _canonical(word: str) -> str:
+    reverse = word.translate(_COMPLEMENT)[::-1]
+    return min(word, reverse)
+
+
+def _validate_common(
+    *,
+    k: int,
+    canonical: bool,
+    seed: int,
+    max_hashes: int,
+    max_unique_hashes: int,
+) -> None:
+    validate_positive_int(k, "k")
+    validate_bool(canonical, "canonical")
+    if isinstance(seed, bool) or not isinstance(seed, int) or not 0 <= seed < _UINT64_SPACE:
+        raise ConfigurationError(
+            "seed must be an integer in [0, 2**64).",
+            code="INVALID_SKETCH_SEED",
+        )
+    validate_positive_int(max_hashes, "max_hashes")
+    if max_hashes > DEFAULT_MAX_SKETCH_HASHES:
+        raise ConfigurationError(
+            "max_hashes exceeds the hard safety ceiling.",
+            code="SKETCH_HASH_LIMIT_INVALID",
+            context={"max_hashes": max_hashes, "hard_limit": DEFAULT_MAX_SKETCH_HASHES},
+        )
+    validate_positive_int(max_unique_hashes, "max_unique_hashes")
+    if max_unique_hashes > DEFAULT_MAX_UNIQUE_HASHES:
+        raise ConfigurationError(
+            "max_unique_hashes exceeds the hard safety ceiling.",
+            code="SKETCH_UNIQUE_HASH_LIMIT_INVALID",
+            context={
+                "max_unique_hashes": max_unique_hashes,
+                "hard_limit": DEFAULT_MAX_UNIQUE_HASHES,
+            },
+        )
+
+
+def _unique_hashes(
+    value: SequenceInput,
+    *,
+    k: int,
+    canonical: bool,
+    seed: int,
+    max_unique_hashes: int,
+) -> tuple[str | None, int, int, tuple[int, ...]]:
+    sequence, sequence_id = sequence_and_id(value)
+    if sequence.topology is Topology.CIRCULAR:
+        raise ConfigurationError(
+            "Native sketches currently require a linear sequence.",
+            code="SKETCH_CIRCULAR_UNSUPPORTED",
+            hint="Rotate a circular sequence to an explicit origin before sketching.",
+        )
+    gaps = tuple(part for part in sequence.parts if isinstance(part, Gap))
+    if gaps:
+        raise UnsupportedGapOperationError(
+            "Sketches do not silently omit or cross explicit Gap objects.",
+            code="SKETCH_GAP_NOT_ALLOWED",
+            context={"gap_count": len(gaps)},
+        )
+    if any(symbol not in "ACGT" for symbol in sequence.symbols):
+        raise ConfigurationError(
+            "Native sketches require canonical A/C/G/T symbols.",
+            code="SKETCH_AMBIGUITY_NOT_ALLOWED",
+        )
+    words: Iterable[str] = (
+        sequence.symbols[index : index + k]
+        for index in range(max(0, sequence.symbol_length - k + 1))
+    )
+    if canonical:
+        words = (_canonical(word) for word in words)
+    unique_hashes: set[int] = set()
+    for word in words:
+        unique_hashes.add(_hash64(word, seed))
+        if len(unique_hashes) > max_unique_hashes:
+            raise ConfigurationError(
+                "Sketch calculation exceeds max_unique_hashes.",
+                code="SKETCH_UNIQUE_HASH_LIMIT_EXCEEDED",
+                context={"unique_hash_count_lower_bound": max_unique_hashes + 1},
+            )
+    hashes = tuple(sorted(unique_hashes))
+    observation_count = max(0, sequence.symbol_length - k + 1)
+    return sequence_id, sequence.symbol_length, observation_count, hashes
+
+
+def minhash(
+    value: SequenceInput,
+    *,
+    k: int,
+    num_hashes: int = 1_000,
+    canonical: bool = True,
+    seed: int = 0,
+    max_hashes: int = DEFAULT_MAX_SKETCH_HASHES,
+    max_unique_hashes: int = DEFAULT_MAX_UNIQUE_HASHES,
+) -> SketchResult:
+    """Return the smallest distinct 64-bit SHA-256-derived k-mer hashes."""
+
+    _validate_common(
+        k=k,
+        canonical=canonical,
+        seed=seed,
+        max_hashes=max_hashes,
+        max_unique_hashes=max_unique_hashes,
+    )
+    validate_positive_int(num_hashes, "num_hashes")
+    if num_hashes > max_hashes:
+        raise ConfigurationError(
+            "num_hashes cannot exceed max_hashes.",
+            code="SKETCH_HASH_LIMIT_EXCEEDED",
+        )
+    sequence_id, symbol_length, observation_count, hashes = _unique_hashes(
+        value,
+        k=k,
+        canonical=canonical,
+        seed=seed,
+        max_unique_hashes=max_unique_hashes,
+    )
+    return SketchResult(
+        name="minhash",
+        method="bottom-k-sha256-64",
+        schema_version="dnakit.sketch.minhash.v1",
+        sequence_id=sequence_id,
+        symbol_length=symbol_length,
+        k=k,
+        canonical=canonical,
+        seed=seed,
+        selection="bottom_k",
+        hashes=hashes[:num_hashes],
+        num_hashes=num_hashes,
+        scaled=None,
+        threshold=None,
+        observation_count=observation_count,
+        unique_hash_count=len(hashes),
+        max_hashes=max_hashes,
+        parameters=FrozenDict(
+            {
+                "hash": "sha256-first-64-big-endian",
+                "deduplicate_kmers": True,
+                "max_unique_hashes": max_unique_hashes,
+                "topology": "linear",
+            }
+        ),
+    )
+
+
+def fracminhash(
+    value: SequenceInput,
+    *,
+    k: int,
+    scaled: int = 1_000,
+    canonical: bool = True,
+    seed: int = 0,
+    max_hashes: int = DEFAULT_MAX_SKETCH_HASHES,
+    max_unique_hashes: int = DEFAULT_MAX_UNIQUE_HASHES,
+) -> SketchResult:
+    """Retain hashes below ``floor(2**64 / scaled)`` with a hard output limit."""
+
+    _validate_common(
+        k=k,
+        canonical=canonical,
+        seed=seed,
+        max_hashes=max_hashes,
+        max_unique_hashes=max_unique_hashes,
+    )
+    validate_positive_int(scaled, "scaled")
+    if scaled > _UINT64_SPACE:
+        raise ConfigurationError(
+            "scaled must not exceed 2**64.",
+            code="INVALID_FRACMINHASH_SCALED",
+        )
+    sequence_id, symbol_length, observation_count, hashes = _unique_hashes(
+        value,
+        k=k,
+        canonical=canonical,
+        seed=seed,
+        max_unique_hashes=max_unique_hashes,
+    )
+    threshold = _UINT64_SPACE // scaled
+    retained = tuple(item for item in hashes if item < threshold)
+    if len(retained) > max_hashes:
+        raise ConfigurationError(
+            "FracMinHash output exceeds max_hashes.",
+            code="SKETCH_HASH_LIMIT_EXCEEDED",
+            context={"retained_hashes": len(retained), "max_hashes": max_hashes},
+            hint="Increase scaled or explicitly raise max_hashes within the hard ceiling.",
+        )
+    return SketchResult(
+        name="fracminhash",
+        method="threshold-sha256-64",
+        schema_version="dnakit.sketch.fracminhash.v1",
+        sequence_id=sequence_id,
+        symbol_length=symbol_length,
+        k=k,
+        canonical=canonical,
+        seed=seed,
+        selection="scaled",
+        hashes=retained,
+        num_hashes=None,
+        scaled=scaled,
+        threshold=threshold,
+        observation_count=observation_count,
+        unique_hash_count=len(hashes),
+        max_hashes=max_hashes,
+        parameters=FrozenDict(
+            {
+                "hash": "sha256-first-64-big-endian",
+                "deduplicate_kmers": True,
+                "max_unique_hashes": max_unique_hashes,
+                "topology": "linear",
+            }
+        ),
+    )
+
+
+__all__ = [
+    "DEFAULT_MAX_SKETCH_HASHES",
+    "DEFAULT_MAX_UNIQUE_HASHES",
+    "fracminhash",
+    "minhash",
+]

dnakit/io/__init__.py

@@ -0,0 +1,112 @@
+"""Streaming DNA record input and output."""
+
+from dnakit.io.annotations import (
+    AGPComponent,
+    AGPDocument,
+    AGPEntry,
+    AGPGap,
+    AnnotationDocument,
+    AnnotationEntry,
+    AnnotationSource,
+    AnnotationTarget,
+    read_agp,
+    read_bed,
+    read_gff3,
+    write_agp,
+    write_bed,
+    write_gff3,
+)
+from dnakit.io.api import ReadMode, read, read_one, read_set, write
+from dnakit.io.config import FeatureWritePolicy, ReadConfig, WriteConfig
+from dnakit.io.indexed import (
+    FastaIndex,
+    FastaIndexEntry,
+    FastqIndex,
+    FastqIndexEntry,
+    build_fasta_index,
+    build_fastq_index,
+    iter_chunks,
+    load_fasta_index,
+    load_fastq_index,
+)
+from dnakit.io.metadata import (
+    MetadataValidationIssue,
+    MetadataValidationReport,
+    filter_by_metadata,
+    merge_metadata,
+    select_metadata,
+    validate_metadata,
+    with_metadata,
+)
+from dnakit.io.parquet import (
+    ParquetBackendStatus,
+    parquet_backend_status,
+    require_parquet_backend,
+)
+from dnakit.io.results import GeneratedID, WriteResult
+from dnakit.io.source import RecordSource
+from dnakit.io.tables import (
+    ParquetCompression,
+    TableColumnType,
+    TableExportResult,
+    TableReadResult,
+    TableSchema,
+    export_result,
+    export_table,
+    read_table,
+)
+
+__all__ = [
+    "AGPComponent",
+    "AGPDocument",
+    "AGPEntry",
+    "AGPGap",
+    "AnnotationDocument",
+    "AnnotationEntry",
+    "AnnotationSource",
+    "AnnotationTarget",
+    "FastaIndex",
+    "FastaIndexEntry",
+    "FastqIndex",
+    "FastqIndexEntry",
+    "FeatureWritePolicy",
+    "GeneratedID",
+    "MetadataValidationIssue",
+    "MetadataValidationReport",
+    "ParquetBackendStatus",
+    "ParquetCompression",
+    "ReadConfig",
+    "ReadMode",
+    "RecordSource",
+    "TableColumnType",
+    "TableExportResult",
+    "TableReadResult",
+    "TableSchema",
+    "WriteConfig",
+    "WriteResult",
+    "build_fasta_index",
+    "build_fastq_index",
+    "export_result",
+    "export_table",
+    "filter_by_metadata",
+    "iter_chunks",
+    "load_fasta_index",
+    "load_fastq_index",
+    "merge_metadata",
+    "parquet_backend_status",
+    "read",
+    "read_agp",
+    "read_bed",
+    "read_gff3",
+    "read_one",
+    "read_set",
+    "read_table",
+    "require_parquet_backend",
+    "select_metadata",
+    "validate_metadata",
+    "with_metadata",
+    "write",
+    "write_agp",
+    "write_bed",
+    "write_gff3",
+]

dnakit/io/_advanced_common.py

@@ -0,0 +1,105 @@
+"""Shared bounded text helpers for advanced codecs."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import TextIO
+
+from dnakit.exceptions import InputFormatError
+
+
+@contextmanager
+def open_text_source(source: str | os.PathLike[str] | TextIO) -> Iterator[TextIO]:
+    """Borrow streams and own path handles."""
+
+    if isinstance(source, (str, os.PathLike)):
+        path = Path(source)
+        try:
+            with path.open("r", encoding="utf-8", newline="") as handle:
+                yield handle
+        except OSError as exc:
+            raise InputFormatError(
+                "Could not open the advanced text input.",
+                code="INPUT_OPEN_FAILED",
+                context={"source": str(path), "reason": str(exc)},
+            ) from exc
+        return
+    if not hasattr(source, "read"):
+        raise TypeError("source must be a path or readable text stream.")
+    probe = source.read(0)
+    if not isinstance(probe, str):
+        raise TypeError("advanced annotation codecs require a decoded text stream.")
+    yield source
+
+
+def checked_lines(handle: TextIO, *, max_line_length: int) -> Iterator[tuple[int, str]]:
+    """Yield newline-free lines while enforcing a hard character limit."""
+
+    if isinstance(max_line_length, bool) or not isinstance(max_line_length, int):
+        raise TypeError("max_line_length must be an integer.")
+    if max_line_length < 1:
+        raise ValueError("max_line_length must be positive.")
+    for line_number, raw in enumerate(handle, start=1):
+        line = raw.rstrip("\r\n")
+        if len(line) > max_line_length:
+            raise InputFormatError(
+                "An input line exceeds the configured resource limit.",
+                code="LINE_TOO_LONG",
+                context={
+                    "line_number": line_number,
+                    "line_length": len(line),
+                    "max_line_length": max_line_length,
+                },
+            )
+        yield line_number, line
+
+
+def write_text_path(
+    target: str | os.PathLike[str] | TextIO,
+    writer: Callable[[TextIO], int],
+    *,
+    overwrite: bool,
+    create_parents: bool,
+) -> int:
+    """Write text atomically for paths and directly to borrowed streams."""
+
+    if not isinstance(overwrite, bool) or not isinstance(create_parents, bool):
+        raise TypeError("overwrite and create_parents must be booleans.")
+    if not isinstance(target, (str, os.PathLike)):
+        if not hasattr(target, "write"):
+            raise TypeError("target must be a path or writable text stream.")
+        count = writer(target)
+        target.flush()
+        return count
+
+    path = Path(target)
+    if path.exists() and not overwrite:
+        raise FileExistsError(f"Refusing to overwrite existing output: {path}")
+    if not path.parent.exists():
+        if not create_parents:
+            raise FileNotFoundError(f"Output parent directory does not exist: {path.parent}")
+        path.parent.mkdir(parents=True, exist_ok=True)
+    descriptor, name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
+    temporary = Path(name)
+    os.close(descriptor)
+    try:
+        with temporary.open("w", encoding="utf-8", newline="") as handle:
+            count = writer(handle)
+            handle.flush()
+            os.fsync(handle.fileno())
+        if overwrite:
+            os.replace(temporary, path)
+        else:
+            os.link(temporary, path)
+            temporary.unlink()
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise
+    return count
+
+
+__all__ = ["checked_lines", "open_text_source", "write_text_path"]