oscura 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

oscura/analyzers/binary/detection/patterns.py

@@ -0,0 +1,320 @@
+"""Efficient pattern mining for repeating byte sequences in binary files.
+
+This module implements brute-force pattern detection for finding
+repeating byte sequences in binary files of any size. Designed for
+high-performance analysis of multi-GB files with minimal memory footprint.
+
+Example:
+    >>> from oscura.analyzers.binary.detection.patterns import PatternMiner
+    >>> from oscura.analyzers.binary.core.file_access import BinaryFile
+    >>> bf = BinaryFile("large_file.bin")
+    >>> miner = PatternMiner()
+    >>> patterns = miner.find_patterns(bf, min_length=4, max_length=64)
+    >>> for pattern in patterns:
+    ...     print(f"Found {len(pattern.bytes)}-byte pattern {pattern.count} times")
+    ...     print(f"  Role: {pattern.role.value}")
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from oscura.analyzers.binary.core.results import Pattern, PatternRole
+
+if TYPE_CHECKING:
+    from oscura.analyzers.binary.core.file_access import BinaryFile
+
+# File size threshold for small files (read entire file vs sampling strategy)
+SMALL_FILE_THRESHOLD = 1_000_000  # 1MB
+
+# Sample size for pattern detection in large files
+PATTERN_SAMPLE_SIZE = 1_000_000  # 1MB per sample location
+
+# Number of sample locations to use for large files
+PATTERN_N_SAMPLES = 5
+
+# Maximum positions stored per pattern to prevent unbounded memory growth
+MAX_POSITIONS_PER_PATTERN = 10_000
+
+# Regular spacing detection threshold (coefficient of variation)
+REGULAR_SPACING_CV_THRESHOLD = 0.1  # 10% variation allowed
+
+# Minimum spacing for delimiter classification (bytes)
+MIN_DELIMITER_SPACING = 1000
+
+# Minimum message spacing for fixed-length detection (bytes)
+MIN_MESSAGE_SPACING = 100
+
+# File position threshold for footer detection (fraction of file size)
+FOOTER_POSITION_THRESHOLD = 0.95  # Last 5% of file
+
+# Pattern deduplication overlap threshold (fraction)
+PATTERN_OVERLAP_THRESHOLD = 0.9  # 90% position overlap
+
+
+class PatternMiner:
+    """Efficient pattern mining for repeating byte sequences.
+
+    Uses brute-force substring matching for pattern detection in large
+    binary files. Samples multiple file locations to avoid scanning
+    entire multi-GB files while maintaining high detection accuracy.
+
+    Example:
+        >>> miner = PatternMiner()
+        >>> bf = BinaryFile("data.bin")
+        >>> patterns = miner.find_patterns(bf, min_length=2, max_length=64)
+        >>> print(f"Found {len(patterns)} repeating patterns")
+    """
+
+    def __init__(self) -> None:
+        """Initialize pattern miner."""
+        self._hash_base = 257  # Prime number for hashing
+        self._hash_mod = 2**31 - 1  # Large prime for hash modulus
+
+    def find_patterns(
+        self,
+        file: BinaryFile,
+        min_length: int = 2,
+        max_length: int = 64,
+        min_occurrences: int = 3,
+    ) -> list[Pattern]:
+        """Find repeating byte patterns in binary file.
+
+        Scans file for repeating byte sequences using brute-force
+        substring matching. Samples multiple file locations for large
+        files to maintain performance.
+
+        Args:
+            file: Binary file to analyze.
+            min_length: Minimum pattern length in bytes (default: 2).
+            max_length: Maximum pattern length in bytes (default: 64).
+            min_occurrences: Minimum pattern occurrences to report (default: 3).
+
+        Returns:
+            List of Pattern objects sorted by occurrence count (descending).
+
+        Example:
+            >>> bf = BinaryFile("data.bin")
+            >>> miner = PatternMiner()
+            >>> patterns = miner.find_patterns(bf, min_length=4, min_occurrences=5)
+            >>> for p in patterns:
+            ...     print(f"{p.bytes.hex()}: {p.count} occurrences")
+        """
+        if min_length < 1:
+            raise ValueError(f"min_length must be >= 1, got {min_length}")
+        if max_length < min_length:
+            raise ValueError(f"max_length ({max_length}) must be >= min_length ({min_length})")
+        if min_occurrences < 1:
+            raise ValueError(f"min_occurrences must be >= 1, got {min_occurrences}")
+
+        # Handle empty files
+        if file.size == 0:
+            return []
+
+        # Sample multiple file locations for large files
+        sample_data = self._sample_file(file)
+
+        # Find patterns of each length
+        all_patterns: list[Pattern] = []
+        for length in range(min_length, min(max_length + 1, len(sample_data) + 1)):
+            length_patterns = self._find_patterns_of_length(sample_data, length, min_occurrences)
+            all_patterns.extend(length_patterns)
+
+        # Deduplicate patterns (remove subpatterns)
+        deduplicated = self._deduplicate_patterns(all_patterns)
+
+        # Classify pattern roles
+        classified = [self._classify_pattern_with_file_info(p, file) for p in deduplicated]
+
+        # Sort by occurrence count (most frequent first)
+        classified.sort(key=lambda p: p.count, reverse=True)
+
+        return classified
+
+    def _sample_file(self, file: BinaryFile) -> bytes:
+        """Sample file data for pattern analysis.
+
+        For large files, samples multiple locations instead of reading
+        entire file to maintain performance.
+
+        Args:
+            file: Binary file to sample.
+
+        Returns:
+            Concatenated sample data from multiple file locations.
+        """
+        # For small files, read entire file
+        if file.size < SMALL_FILE_THRESHOLD:
+            return file.read_bytes(0, file.size)
+
+        # For large files, sample multiple locations
+        n_samples = PATTERN_N_SAMPLES
+        sample_size = PATTERN_SAMPLE_SIZE
+
+        samples = file.sample_locations(n_samples=n_samples, sample_size=sample_size)
+
+        # Concatenate samples
+        return b"".join(samples)
+
+    def _find_patterns_of_length(
+        self, data: bytes, length: int, min_occurrences: int
+    ) -> list[Pattern]:
+        """Find all patterns of specific length in data.
+
+        Uses brute-force substring matching for pattern detection.
+
+        Args:
+            data: Byte data to search.
+            length: Pattern length in bytes.
+            min_occurrences: Minimum occurrences to report.
+
+        Returns:
+            List of Pattern objects for this length.
+        """
+        if len(data) < length:
+            return []
+
+        # Build hash table of patterns
+        pattern_positions: dict[bytes, list[int]] = defaultdict(list)
+
+        # Scan data for patterns
+        for i in range(len(data) - length + 1):
+            pattern_bytes = data[i : i + length]
+            if len(pattern_positions[pattern_bytes]) < MAX_POSITIONS_PER_PATTERN:
+                pattern_positions[pattern_bytes].append(i)
+
+        # Filter by minimum occurrences
+        patterns: list[Pattern] = []
+        for pattern_bytes, positions in pattern_positions.items():
+            count = len(positions)
+            if count >= min_occurrences:
+                # Calculate spacing statistics
+                spacings = np.diff(positions) if len(positions) > 1 else np.array([])
+                avg_spacing = float(np.mean(spacings)) if len(spacings) > 0 else 0.0
+                spacing_variance = float(np.var(spacings)) if len(spacings) > 0 else 0.0
+
+                # Determine if spacing is regular
+                regular = self._is_regular_spacing(spacings)
+
+                # Create pattern with initial role (will classify later)
+                patterns.append(
+                    Pattern(
+                        bytes=pattern_bytes,
+                        positions=positions,
+                        count=count,
+                        avg_spacing=avg_spacing,
+                        spacing_variance=spacing_variance,
+                        regular=regular,
+                        role=PatternRole.UNKNOWN,
+                    )
+                )
+
+        return patterns
+
+    def _is_regular_spacing(self, spacings: np.ndarray[int, np.dtype[np.int64]]) -> bool:
+        """Determine if pattern spacing is regular.
+
+        Args:
+            spacings: Array of spacing values between pattern occurrences.
+
+        Returns:
+            True if spacing is regular (coefficient of variation < 0.1).
+        """
+        if len(spacings) < 2:
+            return False
+
+        mean_spacing = float(np.mean(spacings))
+        if mean_spacing == 0:
+            return False
+
+        std_spacing = float(np.std(spacings))
+        coefficient_of_variation = std_spacing / mean_spacing
+
+        # Consider regular if CV below threshold
+        return coefficient_of_variation < REGULAR_SPACING_CV_THRESHOLD
+
+    def _classify_pattern_with_file_info(self, pattern: Pattern, file: BinaryFile) -> Pattern:
+        """Classify pattern role using file context.
+
+        Args:
+            pattern: Pattern to classify.
+            file: Binary file for context.
+
+        Returns:
+            Pattern with updated role classification.
+        """
+        if not pattern.positions:
+            return pattern
+
+        first_position = pattern.positions[0]
+
+        # Header: at file start (position 0)
+        if first_position == 0:
+            pattern.role = PatternRole.HEADER
+            return pattern
+
+        # Footer: at file end
+        last_position = pattern.positions[-1]
+        if last_position > FOOTER_POSITION_THRESHOLD * file.size:
+            pattern.role = PatternRole.FOOTER
+            return pattern
+
+        # Delimiter: regular spacing + large gaps
+        if pattern.regular and pattern.avg_spacing > MIN_DELIMITER_SPACING:
+            pattern.role = PatternRole.DELIMITER
+            return pattern
+
+        # Sync marker: high frequency + short pattern
+        if len(pattern.bytes) <= 4 and pattern.count > 10:
+            pattern.role = PatternRole.SYNC_MARKER
+            return pattern
+
+        # Default to unknown
+        pattern.role = PatternRole.UNKNOWN
+        return pattern
+
+    def _deduplicate_patterns(self, patterns: list[Pattern]) -> list[Pattern]:
+        """Remove duplicate and subpatterns from pattern list.
+
+        Keeps longer patterns when a pattern is a substring of another
+        with the same positions.
+
+        Args:
+            patterns: List of patterns to deduplicate.
+
+        Returns:
+            Deduplicated list of patterns.
+        """
+        if not patterns:
+            return []
+
+        # Sort by length (longest first) and count (most frequent first)
+        sorted_patterns = sorted(patterns, key=lambda p: (len(p.bytes), p.count), reverse=True)
+
+        # Keep track of deduplicated patterns
+        deduplicated: list[Pattern] = []
+        seen_position_sets: list[set[int]] = []
+
+        for pattern in sorted_patterns:
+            position_set = set(pattern.positions)
+
+            # Check if this pattern is a duplicate (same positions)
+            is_duplicate = False
+            for seen_set in seen_position_sets:
+                # If position sets are very similar, consider duplicate
+                overlap = len(position_set & seen_set) / max(len(position_set), len(seen_set))
+                if overlap > PATTERN_OVERLAP_THRESHOLD:
+                    is_duplicate = True
+                    break
+
+            if not is_duplicate:
+                deduplicated.append(pattern)
+                seen_position_sets.append(position_set)
+
+        return deduplicated
+
+
+__all__ = ["PatternMiner"]