dnakit 0.1.0.dev0__py3-none-any.whl

dnakit/datasets/__init__.py

@@ -0,0 +1,100 @@
+"""Dataset deduplication, clustering, splitting, leakage, and quality evaluation."""
+
+from dnakit.datasets.advanced_deduplicate import deduplicate_approximate, deduplicate_iupac
+from dnakit.datasets.advanced_split import joint_split, temporal_split
+from dnakit.datasets.clustering import (
+    cluster_sequences,
+    hierarchical_cluster,
+    select_representatives,
+)
+from dnakit.datasets.config import (
+    ClusterConfig,
+    DeduplicationConfig,
+    HierarchicalClusteringConfig,
+    IUPACDeduplicationConfig,
+    JointSplitConfig,
+    LeakageConfig,
+    NeuralClusteringConfig,
+    SplitConfig,
+    TemporalSplitConfig,
+)
+from dnakit.datasets.deduplicate import deduplicate
+from dnakit.datasets.evaluation import detect_leakage, evaluate_split_quality
+from dnakit.datasets.filter import (
+    MetadataExclusionField,
+    MetadataValues,
+    exclude_by_metadata,
+    exclude_chromosomes,
+    exclude_species,
+)
+from dnakit.datasets.neural_clustering import neural_cluster_sequences
+from dnakit.datasets.results import (
+    Cluster,
+    ClusteringResult,
+    DedupGroup,
+    DeduplicationResult,
+    HierarchicalClusteringResult,
+    IUPACDeduplicationResult,
+    IUPACDuplicateGroup,
+    IUPACPairRelation,
+    JointSplitResult,
+    LeakageEvent,
+    LeakageReport,
+    LinkageStep,
+    NeuralClusteringResult,
+    RepresentativeSelectionResult,
+    SplitAssignment,
+    SplitQualityResult,
+    SplitResult,
+    SplitSubset,
+    TemporalSplitResult,
+)
+from dnakit.datasets.split import split
+
+__all__ = [
+    "Cluster",
+    "ClusterConfig",
+    "ClusteringResult",
+    "DedupGroup",
+    "DeduplicationConfig",
+    "DeduplicationResult",
+    "HierarchicalClusteringConfig",
+    "HierarchicalClusteringResult",
+    "IUPACDeduplicationConfig",
+    "IUPACDeduplicationResult",
+    "IUPACDuplicateGroup",
+    "IUPACPairRelation",
+    "JointSplitConfig",
+    "JointSplitResult",
+    "LeakageConfig",
+    "LeakageEvent",
+    "LeakageReport",
+    "LinkageStep",
+    "MetadataExclusionField",
+    "MetadataValues",
+    "NeuralClusteringConfig",
+    "NeuralClusteringResult",
+    "RepresentativeSelectionResult",
+    "SplitAssignment",
+    "SplitConfig",
+    "SplitQualityResult",
+    "SplitResult",
+    "SplitSubset",
+    "TemporalSplitConfig",
+    "TemporalSplitResult",
+    "cluster_sequences",
+    "deduplicate",
+    "deduplicate_approximate",
+    "deduplicate_iupac",
+    "detect_leakage",
+    "evaluate_split_quality",
+    "exclude_by_metadata",
+    "exclude_chromosomes",
+    "exclude_species",
+    "hierarchical_cluster",
+    "joint_split",
+    "neural_cluster_sequences",
+    "select_representatives",
+    "split",
+    "temporal_split",
+]

dnakit/datasets/_advanced_shared.py

@@ -0,0 +1,198 @@
+"""Shared bounded pairwise primitives for advanced dataset algorithms."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable
+
+from dnakit.alignment import AlignmentConfig, align_pairwise
+from dnakit.core import DNARecord, Gap, Topology
+from dnakit.exceptions import ConfigurationError, UnsupportedGapOperationError
+from dnakit.fingerprints import kmer_fingerprint
+from dnakit.similarity import edit_distance, fingerprint_similarity, kmer_similarity
+
+from .config import ClusterMethod
+
+
+class UnionFind:
+    def __init__(self, size: int) -> None:
+        self.parent = list(range(size))
+
+    def find(self, value: int) -> int:
+        root = value
+        while self.parent[root] != root:
+            root = self.parent[root]
+        while self.parent[value] != value:
+            parent = self.parent[value]
+            self.parent[value] = root
+            value = parent
+        return root
+
+    def union(self, left: int, right: int) -> None:
+        left_root, right_root = self.find(left), self.find(right)
+        if left_root != right_root:
+            self.parent[right_root] = left_root
+
+    def groups(self) -> tuple[tuple[int, ...], ...]:
+        grouped: dict[int, list[int]] = {}
+        for index in range(len(self.parent)):
+            grouped.setdefault(self.find(index), []).append(index)
+        return tuple(tuple(values) for values in grouped.values())
+
+
+def validate_max_records(value: int, maximum: int) -> None:
+    if value > maximum:
+        raise ConfigurationError(
+            "Advanced dataset input exceeds max_records.",
+            code="ADVANCED_DATASET_SIZE_LIMIT",
+            context={"record_count": value, "max_records": maximum},
+        )
+
+
+def materialize_limited(
+    records: Iterable[DNARecord],
+    *,
+    max_records: int,
+) -> tuple[DNARecord, ...]:
+    try:
+        iterator = iter(records)
+    except TypeError as exc:
+        raise ConfigurationError(
+            "records must be iterable.", code="INVALID_DATASET_RECORDS"
+        ) from exc
+    values: list[DNARecord] = []
+    for index, record in enumerate(iterator):
+        if not isinstance(record, DNARecord):
+            raise ConfigurationError(
+                "Every dataset item must be a DNARecord.",
+                code="INVALID_DATASET_RECORD",
+                context={"input_index": index},
+            )
+        values.append(record)
+        if len(values) > max_records:
+            validate_max_records(len(values), max_records)
+    return tuple(values)
+
+
+def validate_pairwise_sequences(records: tuple[DNARecord, ...], *, operation: str) -> None:
+    for index, record in enumerate(records):
+        if record.sequence.topology is Topology.CIRCULAR:
+            raise ConfigurationError(
+                f"{operation} requires linear sequences.",
+                code="CIRCULAR_ADVANCED_DATASET_UNSUPPORTED",
+                context={"input_index": index, "record_id": record.id},
+            )
+        if any(isinstance(part, Gap) for part in record.sequence.parts):
+            raise UnsupportedGapOperationError(
+                f"{operation} does not silently omit or bridge Gap objects.",
+                code="ADVANCED_DATASET_GAP_UNSUPPORTED",
+                context={"input_index": index, "record_id": record.id},
+            )
+
+
+def pair_count(size: int) -> int:
+    return size * (size - 1) // 2
+
+
+def ensure_pair_limit(size: int, limit: int) -> int:
+    count = pair_count(size)
+    if count > limit:
+        raise ConfigurationError(
+            "Pairwise workload exceeds max_pairwise_comparisons.",
+            code="ADVANCED_PAIRWISE_LIMIT",
+            context={"record_count": size, "pairwise_comparisons": count, "limit": limit},
+        )
+    return count
+
+
+def quality_score(record: DNARecord) -> float:
+    values = record.letter_annotations.get("phred_quality")
+    return float("-inf") if not values else math.fsum(values) / len(values)
+
+
+def pair_similarity(
+    left: DNARecord,
+    right: DNARecord,
+    *,
+    method: ClusterMethod,
+    k: int,
+    canonical: bool,
+    max_alignment_cells: int,
+) -> float:
+    if method in {"kmer", "fingerprint"} and (
+        left.sequence.symbol_length < k or right.sequence.symbol_length < k
+    ):
+        raise ConfigurationError(
+            "k exceeds at least one sequence length in a pairwise dataset comparison.",
+            code="DATASET_K_EXCEEDS_SEQUENCE_LENGTH",
+            context={
+                "k": k,
+                "left_length": left.sequence.symbol_length,
+                "right_length": right.sequence.symbol_length,
+            },
+        )
+    if method == "identity":
+        alignment_result = align_pairwise(
+            left,
+            right,
+            config=AlignmentConfig(mode="global", max_cells=max_alignment_cells),
+        )
+        return 1.0 if alignment_result.identity is None else alignment_result.identity
+    if method == "edit":
+        distance_result = edit_distance(
+            left,
+            right,
+            max_cells=max_alignment_cells,
+        )
+        denominator = max(left.sequence.symbol_length, right.sequence.symbol_length)
+        return 1.0 if denominator == 0 else 1.0 - distance_result.distance / denominator
+    if method == "kmer":
+        return kmer_similarity(
+            left,
+            right,
+            k=k,
+            metric="jaccard",
+            canonical=canonical,
+        ).value
+    left_fingerprint = kmer_fingerprint(left, k=k, canonical=canonical, mode="binary")
+    right_fingerprint = kmer_fingerprint(right, k=k, canonical=canonical, mode="binary")
+    return fingerprint_similarity(left_fingerprint, right_fingerprint, metric="tanimoto").value
+
+
+def similarity_matrix(
+    records: tuple[DNARecord, ...],
+    *,
+    method: ClusterMethod,
+    k: int,
+    canonical: bool,
+    max_alignment_cells: int,
+) -> tuple[tuple[float, ...], ...]:
+    rows = [
+        [1.0 if left == right else 0.0 for right in range(len(records))]
+        for left in range(len(records))
+    ]
+    for left in range(len(records)):
+        for right in range(left + 1, len(records)):
+            value = pair_similarity(
+                records[left],
+                records[right],
+                method=method,
+                k=k,
+                canonical=canonical,
+                max_alignment_cells=max_alignment_cells,
+            )
+            rows[left][right] = rows[right][left] = value
+    return tuple(tuple(row) for row in rows)
+
+
+__all__ = [
+    "UnionFind",
+    "ensure_pair_limit",
+    "materialize_limited",
+    "pair_count",
+    "pair_similarity",
+    "quality_score",
+    "similarity_matrix",
+    "validate_max_records",
+    "validate_pairwise_sequences",
+]

dnakit/datasets/_metadata.py

@@ -0,0 +1,33 @@
+"""Type-preserving keys for immutable JSON metadata values."""
+
+from __future__ import annotations
+
+from collections.abc import Hashable
+
+from dnakit.core._json import FrozenDict, JSONValue
+
+
+def metadata_value_key(value: JSONValue) -> Hashable:
+    """Return a recursive key that does not apply Python numeric coercion rules."""
+
+    if value is None:
+        return ("null",)
+    if isinstance(value, bool):
+        return ("bool", value)
+    if isinstance(value, int):
+        return ("int", value)
+    if isinstance(value, float):
+        return ("float", value)
+    if isinstance(value, str):
+        return ("string", value)
+    if isinstance(value, tuple):
+        return ("array", tuple(metadata_value_key(item) for item in value))
+    if isinstance(value, FrozenDict):
+        return (
+            "object",
+            tuple((key, metadata_value_key(item)) for key, item in sorted(value.items())),
+        )
+    raise TypeError(f"Unsupported frozen JSON metadata type: {type(value).__name__}")
+
+
+__all__ = ["metadata_value_key"]

dnakit/datasets/advanced_deduplicate.py

@@ -0,0 +1,183 @@
+"""IUPAC-compatible and approximate deduplication."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Literal
+
+from dnakit.core import DNARecord, DNASet, Gap, Topology
+from dnakit.exceptions import ConfigurationError, UnsupportedGapOperationError
+
+from ._advanced_shared import ensure_pair_limit, materialize_limited, quality_score
+from .clustering import cluster_sequences
+from .config import ClusterConfig, IUPACDeduplicationConfig
+from .results import (
+    ClusteringResult,
+    IUPACDeduplicationResult,
+    IUPACDuplicateGroup,
+    IUPACPairRelation,
+)
+
+_IUPAC = {
+    "A": frozenset("A"),
+    "C": frozenset("C"),
+    "G": frozenset("G"),
+    "T": frozenset("T"),
+    "R": frozenset("AG"),
+    "Y": frozenset("CT"),
+    "S": frozenset("GC"),
+    "W": frozenset("AT"),
+    "K": frozenset("GT"),
+    "M": frozenset("AC"),
+    "B": frozenset("CGT"),
+    "D": frozenset("AGT"),
+    "H": frozenset("ACT"),
+    "V": frozenset("ACG"),
+    "N": frozenset("ACGT"),
+}
+
+
+def _compatible(left: str, right: str) -> bool:
+    return len(left) == len(right) and all(
+        bool(_IUPAC[left_symbol] & _IUPAC[right_symbol])
+        for left_symbol, right_symbol in zip(left, right, strict=True)
+    )
+
+
+def _representative(
+    members: tuple[int, ...],
+    records: tuple[DNARecord, ...],
+    policy: str,
+) -> int:
+    if policy == "first":
+        return members[0]
+    if policy == "shortest":
+        return min(members, key=lambda index: (records[index].sequence.symbol_length, index))
+    if policy == "longest":
+        return max(members, key=lambda index: (records[index].sequence.symbol_length, -index))
+    return max(members, key=lambda index: (quality_score(records[index]), -index))
+
+
+def deduplicate_iupac(
+    records: Iterable[DNARecord],
+    *,
+    config: IUPACDeduplicationConfig | None = None,
+) -> IUPACDeduplicationResult:
+    """Greedily build stable complete-link groups of pairwise-compatible IUPAC strings."""
+
+    resolved = IUPACDeduplicationConfig() if config is None else config
+    if not isinstance(resolved, IUPACDeduplicationConfig):
+        raise ConfigurationError(
+            "config must be IUPACDeduplicationConfig or None.",
+            code="INVALID_IUPAC_DEDUPLICATION_CONFIG",
+        )
+    materialized = materialize_limited(records, max_records=resolved.max_records)
+    comparisons = ensure_pair_limit(len(materialized), resolved.max_pairwise_comparisons)
+    for index, record in enumerate(materialized):
+        if record.sequence.topology is not Topology.LINEAR:
+            raise ConfigurationError(
+                "IUPAC-aware deduplication currently requires linear sequences.",
+                code="IUPAC_DEDUPLICATION_LINEAR_REQUIRED",
+                context={"input_index": index},
+            )
+        if any(isinstance(part, Gap) for part in record.sequence.parts):
+            raise UnsupportedGapOperationError(
+                "IUPAC-aware deduplication does not bridge Gap objects.",
+                code="IUPAC_DEDUPLICATION_GAP_UNSUPPORTED",
+            )
+
+    compatibility = [bytearray(len(materialized) - left - 1) for left in range(len(materialized))]
+    identical_pairs = 0
+    compatible_pairs = 0
+    conflict_pairs = 0
+    pair_relations: list[IUPACPairRelation] = []
+    for left in range(len(materialized)):
+        left_symbols = materialized[left].sequence.symbols
+        for right in range(left + 1, len(materialized)):
+            right_symbols = materialized[right].sequence.symbols
+            is_compatible = _compatible(left_symbols, right_symbols)
+            compatibility[left][right - left - 1] = is_compatible
+            relation: Literal["identical", "compatible", "conflict"]
+            if left_symbols == right_symbols:
+                identical_pairs += 1
+                relation = "identical"
+            elif is_compatible:
+                compatible_pairs += 1
+                relation = "compatible"
+            else:
+                conflict_pairs += 1
+                relation = "conflict"
+            pair_relations.append(
+                IUPACPairRelation(
+                    left,
+                    right,
+                    materialized[left].id,
+                    materialized[right].id,
+                    relation,
+                )
+            )
+
+    def pair_is_compatible(left: int, right: int) -> bool:
+        if left > right:
+            left, right = right, left
+        return bool(compatibility[left][right - left - 1])
+
+    groups: list[list[int]] = []
+    for index in range(len(materialized)):
+        for group in groups:
+            if all(pair_is_compatible(index, member) for member in group):
+                group.append(index)
+                break
+        else:
+            groups.append([index])
+
+    audit: list[IUPACDuplicateGroup] = []
+    selected: list[DNARecord] = []
+    for group_index, raw_members in enumerate(groups):
+        members = tuple(raw_members)
+        representative = _representative(members, materialized, resolved.representative_policy)
+        selected.append(materialized[representative])
+        symbols = {materialized[index].sequence.symbols for index in members}
+        audit.append(
+            IUPACDuplicateGroup(
+                group_index,
+                materialized[representative].id,
+                tuple(materialized[index].id for index in members),
+                (
+                    "singleton"
+                    if len(members) == 1
+                    else "identical"
+                    if len(symbols) == 1
+                    else "compatible"
+                ),
+            )
+        )
+    return IUPACDeduplicationResult(
+        DNASet(selected),
+        tuple(audit),
+        tuple(pair_relations),
+        resolved.representative_policy,
+        "position-compatible iff IUPAC base-set intersection is non-empty",
+        "stable-greedy-complete-link",
+        len(materialized),
+        len(selected),
+        comparisons,
+        identical_pairs,
+        compatible_pairs,
+        conflict_pairs,
+        resolved.max_records,
+        resolved.max_pairwise_comparisons,
+    )
+
+
+def deduplicate_approximate(
+    records: Iterable[DNARecord],
+    *,
+    config: ClusterConfig,
+) -> ClusteringResult:
+    """Return threshold-graph components and representatives as near-duplicate groups."""
+
+    return cluster_sequences(records, config=config)
+
+
+__all__ = ["deduplicate_approximate", "deduplicate_iupac"]