oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

oscura/{exploratory → jupyter/exploratory}/fuzzy_advanced.py

@@ -6,7 +6,7 @@ binary sequence alignment.
 
 
 Example:
-    >>> from oscura.exploratory.fuzzy_advanced import (
+    >>> from oscura.jupyter.exploratory.fuzzy_advanced import (
     ...     characterize_variants,
     ...     align_sequences,
     ... )
@@ -20,7 +20,7 @@ from __future__ import annotations
 import logging
 from dataclasses import dataclass
 from enum import Enum
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import numpy as np
 
@@ -211,54 +211,79 @@ def characterize_variants(
         VariantCharacterization with analysis results
 
     Example:
-        >>> patterns = [b'\\x12\\x34\\x56', b'\\x12\\x35\\x56', b'\\x12\\x34\\x57']
+        >>> patterns = [b'\\x12\\x34\\x56', b'\\x12\\x35\\x56']
         >>> result = characterize_variants(patterns)
-        >>> print(f"Consensus: {result.consensus.hex()}")
-        >>> print(f"Variable positions: {result.variable_positions}")
 
     References:
         FUZZY-004: Binary Pattern Variant Characterization and Consensus
     """
     if not patterns:
-        return VariantCharacterization(
-            consensus=b"",
-            positions=[],
-            constant_positions=[],
-            variable_positions=[],
-            suggested_boundaries=[],
-            pattern_count=0,
-            min_length=0,
-        )
+        return _create_empty_characterization()
 
     pattern_count = len(patterns)
     min_length = min(len(p) for p in patterns)
 
+    positions, consensus_bytes, constant_positions, variable_positions = _analyze_positions(
+        patterns, min_length
+    )
+
+    boundaries = _suggest_field_boundaries(positions)
+
+    return VariantCharacterization(
+        consensus=bytes(consensus_bytes),
+        positions=positions,
+        constant_positions=constant_positions,
+        variable_positions=variable_positions,
+        suggested_boundaries=boundaries,
+        pattern_count=pattern_count,
+        min_length=min_length,
+    )
+
+
+def _create_empty_characterization() -> VariantCharacterization:
+    """Create empty characterization result.
+
+    Returns:
+        Empty VariantCharacterization.
+    """
+    return VariantCharacterization(
+        consensus=b"",
+        positions=[],
+        constant_positions=[],
+        variable_positions=[],
+        suggested_boundaries=[],
+        pattern_count=0,
+        min_length=0,
+    )
+
+
+def _analyze_positions(
+    patterns: Sequence[bytes | bytearray], min_length: int
+) -> tuple[list[PositionAnalysis], list[int], list[int], list[int]]:
+    """Analyze each position across all patterns.
+
+    Args:
+        patterns: Collection of patterns.
+        min_length: Minimum pattern length.
+
+    Returns:
+        Tuple of (positions, consensus_bytes, constant_positions, variable_positions).
+    """
     positions: list[PositionAnalysis] = []
     consensus_bytes: list[int] = []
     constant_positions: list[int] = []
     variable_positions: list[int] = []
 
     for pos in range(min_length):
-        # Collect values at this position
         values = [p[pos] for p in patterns if pos < len(p)]
+        distribution = _build_distribution(values)
 
-        # Count distribution
-        distribution: dict[int, int] = {}
-        for v in values:
-            distribution[v] = distribution.get(v, 0) + 1
-
-        # Find consensus (mode)
         consensus_byte = max(distribution, key=distribution.get)  # type: ignore[arg-type]
         consensus_count = distribution[consensus_byte]
         consensus_confidence = consensus_count / len(values)
 
-        # Compute entropy
         entropy = _compute_entropy(values)
-
-        # Classify variation
         variation_type = _classify_variation(entropy, consensus_confidence)
-
-        # Detect error vs intentional variation
         is_error = _detect_error_variation(values, consensus_byte, consensus_confidence)
 
         analysis = PositionAnalysis(
@@ -279,7 +304,33 @@ def characterize_variants(
         else:
             variable_positions.append(pos)
 
-    # Suggest field boundaries (transitions between constant/variable)
+    return positions, consensus_bytes, constant_positions, variable_positions
+
+
+def _build_distribution(values: list[int]) -> dict[int, int]:
+    """Build frequency distribution of values.
+
+    Args:
+        values: List of byte values.
+
+    Returns:
+        Dictionary mapping value to count.
+    """
+    distribution: dict[int, int] = {}
+    for v in values:
+        distribution[v] = distribution.get(v, 0) + 1
+    return distribution
+
+
+def _suggest_field_boundaries(positions: list[PositionAnalysis]) -> list[int]:
+    """Suggest field boundaries from position analysis.
+
+    Args:
+        positions: List of position analyses.
+
+    Returns:
+        List of boundary positions.
+    """
     boundaries: list[int] = []
     prev_is_constant = None
 
@@ -289,15 +340,7 @@ def characterize_variants(
             boundaries.append(pos)
         prev_is_constant = is_constant
 
-    return VariantCharacterization(
-        consensus=bytes(consensus_bytes),
-        positions=positions,
-        constant_positions=constant_positions,
-        variable_positions=variable_positions,
-        suggested_boundaries=boundaries,
-        pattern_count=pattern_count,
-        min_length=min_length,
-    )
+    return boundaries
 
 
 # =============================================================================
@@ -371,16 +414,47 @@ def _needleman_wunsch(
     Returns:
         (aligned_seq1, aligned_seq2, score)
     """
-    m, n = len(seq1), len(seq2)
+    # Initialize matrices
+    DIAG, UP, LEFT = 0, 1, 2
+    score, traceback = _initialize_alignment_matrices(seq1, seq2, gap_open, gap_extend, UP, LEFT)
+
+    # Fill scoring matrix
+    _fill_alignment_matrix(
+        seq1,
+        seq2,
+        score,
+        traceback,
+        match_bonus,
+        mismatch_penalty,
+        gap_open,
+        gap_extend,
+        DIAG,
+        UP,
+        LEFT,
+    )
 
-    # Initialize score matrix
+    # Traceback to build alignment
+    aligned1, aligned2 = _traceback_alignment(
+        seq1, seq2, traceback, len(seq1), len(seq2), DIAG, UP, LEFT
+    )
+
+    return bytes(aligned1), bytes(aligned2), float(score[len(seq1), len(seq2)])
+
+
+def _initialize_alignment_matrices(
+    seq1: bytes,
+    seq2: bytes,
+    gap_open: int,
+    gap_extend: int,
+    UP: int,
+    LEFT: int,
+) -> tuple[np.ndarray[Any, Any], np.ndarray[Any, Any]]:
+    """Initialize score and traceback matrices."""
+    m, n = len(seq1), len(seq2)
     score = np.zeros((m + 1, n + 1), dtype=np.int32)
     traceback = np.zeros((m + 1, n + 1), dtype=np.int8)
 
-    # Direction constants
-    DIAG, UP, LEFT = 0, 1, 2
-
-    # Initialize first row and column
+    # Initialize first row and column with gap penalties
     for i in range(1, m + 1):
         score[i, 0] = gap_open + (i - 1) * gap_extend
         traceback[i, 0] = UP
@@ -389,28 +463,33 @@ def _needleman_wunsch(
         score[0, j] = gap_open + (j - 1) * gap_extend
         traceback[0, j] = LEFT
 
-    # Fill matrix
-    for i in range(1, m + 1):
-        for j in range(1, n + 1):
-            # Match/mismatch
-            if seq1[i - 1] == seq2[j - 1]:
-                diag_score = score[i - 1, j - 1] + match_bonus
-            else:
-                diag_score = score[i - 1, j - 1] + mismatch_penalty
+    return score, traceback
 
-            # Gap in seq2 (moving down)
-            if traceback[i - 1, j] == UP:
-                up_score = score[i - 1, j] + gap_extend
-            else:
-                up_score = score[i - 1, j] + gap_open
 
-            # Gap in seq1 (moving right)
-            if traceback[i, j - 1] == LEFT:
-                left_score = score[i, j - 1] + gap_extend
-            else:
-                left_score = score[i, j - 1] + gap_open
+def _fill_alignment_matrix(
+    seq1: bytes,
+    seq2: bytes,
+    score: np.ndarray[Any, Any],
+    traceback: np.ndarray[Any, Any],
+    match_bonus: int,
+    mismatch_penalty: int,
+    gap_open: int,
+    gap_extend: int,
+    DIAG: int,
+    UP: int,
+    LEFT: int,
+) -> None:
+    """Fill alignment matrix using dynamic programming."""
+    for i in range(1, len(seq1) + 1):
+        for j in range(1, len(seq2) + 1):
+            # Calculate scores for three possible moves
+            diag_score = score[i - 1, j - 1] + (
+                match_bonus if seq1[i - 1] == seq2[j - 1] else mismatch_penalty
+            )
+            up_score = score[i - 1, j] + (gap_extend if traceback[i - 1, j] == UP else gap_open)
+            left_score = score[i, j - 1] + (gap_extend if traceback[i, j - 1] == LEFT else gap_open)
 
-            # Choose best
+            # Choose best score and direction
             best = max(diag_score, up_score, left_score)
             score[i, j] = best
 
@@ -421,10 +500,20 @@ def _needleman_wunsch(
             else:
                 traceback[i, j] = LEFT
 
-    # Traceback
+
+def _traceback_alignment(
+    seq1: bytes,
+    seq2: bytes,
+    traceback: np.ndarray[Any, Any],
+    i: int,
+    j: int,
+    DIAG: int,
+    UP: int,
+    LEFT: int,
+) -> tuple[list[int], list[int]]:
+    """Traceback through matrix to build final alignment."""
     aligned1: list[int] = []
     aligned2: list[int] = []
-    i, j = m, n
 
     while i > 0 or j > 0:
         if i > 0 and j > 0 and traceback[i, j] == DIAG:
@@ -441,11 +530,9 @@ def _needleman_wunsch(
             aligned2.append(seq2[j - 1])
             j -= 1
 
-    # Reverse (traceback goes backwards)
     aligned1.reverse()
     aligned2.reverse()
-
-    return bytes(aligned1), bytes(aligned2), float(score[m, n])
+    return aligned1, aligned2
 
 
 def _smith_waterman(
@@ -660,14 +747,47 @@ def _progressive_alignment(
     gap_extend: int,
 ) -> AlignmentResult:
     """Progressive multiple sequence alignment."""
-    # Start with first sequence as reference
-    ref = sequences[0]
-    aligned_seqs: list[bytes] = [ref]
+    # Align sequences progressively
+    aligned_seqs, total_score = _align_progressively(sequences, gap_open, gap_extend)
+
+    # Build result sequences
+    result_seqs = _build_result_sequences(sequences, aligned_seqs)
+
+    # Analyze alignment
+    alignment_length = len(aligned_seqs[0]) if aligned_seqs else 0
+    conservation = compute_conservation_scores(aligned_seqs)
+    conserved_regions = _find_conserved_regions(conservation)
+    gap_positions = _find_gap_positions(aligned_seqs, alignment_length)
+
+    return AlignmentResult(
+        sequences=result_seqs,
+        conservation_scores=conservation,
+        conserved_regions=conserved_regions,
+        gap_positions=gap_positions,
+        alignment_score=total_score,
+    )
+
+
+def _align_progressively(
+    sequences: Sequence[bytes],
+    gap_open: int,
+    gap_extend: int,
+) -> tuple[list[bytes], float]:
+    """Align sequences progressively.
+
+    Args:
+        sequences: Input sequences.
+        gap_open: Gap opening penalty.
+        gap_extend: Gap extension penalty.
+
+    Returns:
+        Tuple of (aligned sequences, total score).
+    """
+    aligned_seqs: list[bytes] = [sequences[0]]
     total_score = 0.0
 
-    # Align each sequence to growing profile
     for seq in sequences[1:]:
-        # Align to last aligned sequence (simplified - real MSA uses profiles)
+        # Align to reference
         ref_aligned, seq_aligned, score = _needleman_wunsch(
             aligned_seqs[0],
             seq,
@@ -675,25 +795,52 @@ def _progressive_alignment(
             gap_extend=gap_extend,
         )
 
-        # Update all previous alignments to match new length
-        new_aligned: list[bytes] = []
-        for prev in aligned_seqs:
-            # Insert gaps where ref got new gaps
-            new_prev: list[int] = []
-            prev_idx = 0
-            for byte in ref_aligned:
-                if byte == GAP_BYTE:
-                    new_prev.append(GAP_BYTE)
-                elif prev_idx < len(prev):
-                    new_prev.append(prev[prev_idx])
-                    prev_idx += 1
-            new_aligned.append(bytes(new_prev))
-
+        # Update existing alignments
+        new_aligned = _update_alignments(aligned_seqs, ref_aligned)
         aligned_seqs = new_aligned
         aligned_seqs.append(seq_aligned)
         total_score += score
 
-    # Build result
+    return aligned_seqs, total_score
+
+
+def _update_alignments(aligned_seqs: list[bytes], ref_aligned: bytes) -> list[bytes]:
+    """Update existing alignments to match reference.
+
+    Args:
+        aligned_seqs: Current aligned sequences.
+        ref_aligned: New reference alignment.
+
+    Returns:
+        Updated aligned sequences.
+    """
+    new_aligned: list[bytes] = []
+    for prev in aligned_seqs:
+        new_prev: list[int] = []
+        prev_idx = 0
+        for byte in ref_aligned:
+            if byte == GAP_BYTE:
+                new_prev.append(GAP_BYTE)
+            elif prev_idx < len(prev):
+                new_prev.append(prev[prev_idx])
+                prev_idx += 1
+        new_aligned.append(bytes(new_prev))
+    return new_aligned
+
+
+def _build_result_sequences(
+    sequences: Sequence[bytes],
+    aligned_seqs: list[bytes],
+) -> list[AlignedSequence]:
+    """Build result sequences from alignment.
+
+    Args:
+        sequences: Original sequences.
+        aligned_seqs: Aligned sequences.
+
+    Returns:
+        List of AlignedSequence objects.
+    """
     result_seqs: list[AlignedSequence] = []
     for orig, aligned in zip(sequences, aligned_seqs, strict=False):
         gaps = [i for i, b in enumerate(aligned) if b == GAP_BYTE]
@@ -702,15 +849,21 @@ def _progressive_alignment(
                 original=orig,
                 aligned=aligned,
                 gaps=gaps,
-                score=0.0,  # Individual scores not tracked in progressive
+                score=0.0,
             )
         )
+    return result_seqs
 
-    # Compute conservation scores
-    alignment_length = len(aligned_seqs[0]) if aligned_seqs else 0
-    conservation = compute_conservation_scores(aligned_seqs)
 
-    # Find conserved regions
+def _find_conserved_regions(conservation: list[float]) -> list[tuple[int, int]]:
+    """Find conserved regions in alignment.
+
+    Args:
+        conservation: Conservation scores.
+
+    Returns:
+        List of (start, end) tuples.
+    """
     conserved_regions: list[tuple[int, int]] = []
     in_region = False
     region_start = 0
@@ -726,20 +879,25 @@ def _progressive_alignment(
     if in_region:
         conserved_regions.append((region_start, len(conservation) - 1))
 
-    # Find common gap positions
+    return conserved_regions
+
+
+def _find_gap_positions(aligned_seqs: list[bytes], alignment_length: int) -> list[int]:
+    """Find positions with common gaps.
+
+    Args:
+        aligned_seqs: Aligned sequences.
+        alignment_length: Length of alignment.
+
+    Returns:
+        List of gap positions.
+    """
     gap_positions: list[int] = []
     for pos in range(alignment_length):
         gap_count = sum(1 for seq in aligned_seqs if seq[pos] == GAP_BYTE)
         if gap_count > len(aligned_seqs) // 2:
             gap_positions.append(pos)
-
-    return AlignmentResult(
-        sequences=result_seqs,
-        conservation_scores=conservation,
-        conserved_regions=conserved_regions,
-        gap_positions=gap_positions,
-        alignment_score=total_score,
-    )
+    return gap_positions
 
 
 def compute_conservation_scores(