oscura 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

oscura/analyzers/binary/detection/structure.py

@@ -0,0 +1,630 @@
+"""Structure inference for message and field boundaries in binary files.
+
+This module implements message boundary detection and field inference for
+binary files containing structured data. Uses pattern analysis and entropy-based
+field boundary detection to reverse engineer message formats.
+
+Example:
+    >>> from oscura.analyzers.binary.core.file_access import BinaryFile
+    >>> from oscura.analyzers.binary.detection.patterns import PatternMiner
+    >>> from oscura.analyzers.binary.detection.encoding import EncodingDetector
+    >>> from oscura.analyzers.binary.detection.structure import StructureInferencer
+    >>>
+    >>> bf = BinaryFile("data.bin")
+    >>> pattern_miner = PatternMiner()
+    >>> encoding_detector = EncodingDetector()
+    >>> patterns = pattern_miner.find_patterns(bf)
+    >>> encoding = encoding_detector.detect(bf)
+    >>> inferencer = StructureInferencer()
+    >>> result = inferencer.infer(bf, patterns, encoding)
+    >>> print(f"Found {result.message_count} messages")
+    >>> for field in result.fields:
+    ...     print(f"  {field.name}: {field.field_type.value}")
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import TYPE_CHECKING
+
+import numpy as np
+from numpy.typing import NDArray
+
+from oscura.analyzers.binary.core.results import (
+    EncodingResult,
+    Field,
+    FieldType,
+    Message,
+    Pattern,
+    PatternRole,
+    StructureResult,
+)
+
+if TYPE_CHECKING:
+    from oscura.analyzers.binary.core.file_access import BinaryFile
+
+# Sample size for structure analysis (bytes)
+STRUCTURE_SAMPLE_SIZE = 100_000  # 100KB
+
+# Length field validation range (bytes)
+MIN_MESSAGE_LENGTH = 10
+MAX_MESSAGE_LENGTH = 10_000
+
+# Maximum size for last message extraction (bytes)
+MAX_LAST_MESSAGE_SIZE = 10_000
+
+
+class StructureInferencer:
+    """Infer message and field structure from binary files.
+
+    Detects message boundaries using pattern analysis and infers field
+    boundaries using entropy-based techniques. Designed for reverse
+    engineering unknown binary protocols and file formats.
+
+    Example:
+        >>> inferencer = StructureInferencer()
+        >>> bf = BinaryFile("data.bin")
+        >>> result = inferencer.infer(bf, patterns, encoding)
+        >>> if result.has_messages:
+        ...     print(f"Message length: {result.message_length} bytes")
+        ...     print(f"Fields: {len(result.fields)}")
+    """
+
+    def __init__(
+        self,
+        min_messages: int = 3,
+        max_field_length: int = 1024,
+        entropy_window: int = 8,
+        entropy_threshold: float = 0.3,
+    ):
+        """Initialize structure inferencer.
+
+        Args:
+            min_messages: Minimum messages required to infer structure.
+            max_field_length: Maximum field length in bytes.
+            entropy_window: Window size for entropy analysis.
+            entropy_threshold: Entropy change threshold for field boundaries.
+        """
+        self.min_messages = min_messages
+        self.max_field_length = max_field_length
+        self.entropy_window = entropy_window
+        self.entropy_threshold = entropy_threshold
+
+    def infer(
+        self, file: BinaryFile, patterns: list[Pattern], encoding: EncodingResult
+    ) -> StructureResult:
+        """Infer file structure from patterns and encoding.
+
+        Args:
+            file: Binary file to analyze.
+            patterns: Detected byte patterns from PatternMiner.
+            encoding: Detected encoding from EncodingDetector.
+
+        Returns:
+            StructureResult with message and field information.
+
+        Example:
+            >>> bf = BinaryFile("data.bin")
+            >>> patterns = miner.find_patterns(bf)
+            >>> encoding = detector.detect(bf)
+            >>> inferencer = StructureInferencer()
+            >>> result = inferencer.infer(bf, patterns, encoding)
+            >>> print(f"Confidence: {result.confidence:.1%}")
+        """
+        # Strategy 1: Detect message boundaries
+        boundaries = self._detect_boundaries(file, patterns)
+
+        if len(boundaries) < self.min_messages:
+            # No clear message structure detected
+            return StructureResult(
+                has_messages=False,
+                message_length=None,
+                message_count=0,
+                messages=[],
+                fields=[],
+                confidence=0.0,
+            )
+
+        # Strategy 2: Extract messages
+        messages = self._extract_messages(file, boundaries)
+
+        if len(messages) < self.min_messages:
+            return StructureResult(
+                has_messages=False,
+                message_length=None,
+                message_count=0,
+                messages=[],
+                fields=[],
+                confidence=0.0,
+            )
+
+        # Strategy 3: Infer fields from messages
+        fields = self._infer_fields(messages)
+
+        # Calculate confidence based on consistency
+        confidence = self._calculate_confidence(messages, fields, patterns)
+
+        # Determine message length (use most common if variable)
+        message_lengths = [msg.length for msg in messages]
+        message_length = int(np.median(message_lengths))
+
+        return StructureResult(
+            has_messages=True,
+            message_length=message_length,
+            message_count=len(messages),
+            messages=messages,
+            fields=fields,
+            confidence=confidence,
+        )
+
+    def _detect_boundaries(self, file: BinaryFile, patterns: list[Pattern]) -> list[int]:
+        """Detect message boundaries using pattern analysis.
+
+        Tries multiple strategies in order:
+        1. Use header pattern positions
+        2. Use delimiter pattern with regular spacing
+        3. Fixed-length from regular spacing patterns
+        4. Length field detection
+
+        Args:
+            file: Binary file to analyze.
+            patterns: Detected byte patterns.
+
+        Returns:
+            List of byte offsets where messages start.
+        """
+        if not patterns:
+            return []
+
+        # Strategy 1: Use header patterns
+        header_patterns = [p for p in patterns if p.role == PatternRole.HEADER]
+        if header_patterns:
+            # Use most frequent header pattern
+            header = max(header_patterns, key=lambda p: p.count)
+            if header.count >= self.min_messages:
+                return sorted(header.positions)
+
+        # Strategy 2: Use delimiter patterns with regular spacing
+        delimiter_patterns = [p for p in patterns if p.role == PatternRole.DELIMITER and p.regular]
+        if delimiter_patterns:
+            # Use delimiter with most regular spacing
+            delimiter = max(delimiter_patterns, key=lambda p: p.count)
+            if delimiter.count >= self.min_messages:
+                return sorted(delimiter.positions)
+
+        # Strategy 3: Fixed-length messages from regular spacing
+        regular_patterns = [p for p in patterns if p.regular and p.count >= self.min_messages]
+        if regular_patterns:
+            # Use pattern with largest average spacing (likely message boundaries)
+            best_pattern = max(regular_patterns, key=lambda p: p.avg_spacing)
+            # Import MIN_MESSAGE_SPACING from patterns module
+            from oscura.analyzers.binary.detection.patterns import MIN_MESSAGE_SPACING
+
+            if best_pattern.avg_spacing > MIN_MESSAGE_SPACING:
+                return sorted(best_pattern.positions)
+
+        # Strategy 4: Length field detection
+        length_boundaries = self._detect_length_field_boundaries(file)
+        if len(length_boundaries) >= self.min_messages:
+            return length_boundaries
+
+        # No clear boundaries found
+        return []
+
+    def _detect_length_field_boundaries(self, file: BinaryFile) -> list[int]:
+        """Detect message boundaries using length field analysis.
+
+        Scans file for uint16/uint32 values that might indicate message lengths
+        and validates by checking if they form consistent message structures.
+
+        Args:
+            file: Binary file to analyze.
+
+        Returns:
+            List of detected message boundary positions.
+        """
+        # Sample beginning of file for length field detection
+        sample_size = min(STRUCTURE_SAMPLE_SIZE, file.size)
+        sample_data = file.read_bytes(0, sample_size)
+
+        boundaries: list[int] = []
+
+        # Try uint16 length fields (2 bytes)
+        for offset in range(0, len(sample_data) - 2, 2):
+            # Read potential length field (little-endian)
+            length = int.from_bytes(sample_data[offset : offset + 2], byteorder="little")
+
+            # Validate length is reasonable
+            if MIN_MESSAGE_LENGTH <= length <= MAX_MESSAGE_LENGTH:
+                # Check if this forms a valid message
+                next_offset = offset + 2 + length
+                if next_offset < len(sample_data) - 2:
+                    # Check if next position also has valid length field
+                    next_length = int.from_bytes(
+                        sample_data[next_offset : next_offset + 2], byteorder="little"
+                    )
+                    if MIN_MESSAGE_LENGTH <= next_length <= MAX_MESSAGE_LENGTH:
+                        boundaries.append(offset)
+
+                        # Found potential pattern, continue from next message
+                        if len(boundaries) >= self.min_messages:
+                            break
+
+        return boundaries
+
+    def _extract_messages(self, file: BinaryFile, boundaries: list[int]) -> list[Message]:
+        """Extract messages from file using detected boundaries.
+
+        Args:
+            file: Binary file to read from.
+            boundaries: List of message start positions.
+
+        Returns:
+            List of Message objects.
+        """
+        messages: list[Message] = []
+
+        for i, start in enumerate(boundaries):
+            # Calculate message length
+            if i + 1 < len(boundaries):
+                # Length is distance to next boundary
+                length = boundaries[i + 1] - start
+            else:
+                # Last message: use remaining file data with size limit
+                length = min(MAX_LAST_MESSAGE_SIZE, file.size - start)
+
+            # Validate length is reasonable
+            if length <= 0 or length > MAX_MESSAGE_LENGTH:
+                continue
+
+            # Read message data
+            data = file.read_bytes(start, length)
+
+            if len(data) < length:
+                # End of file reached
+                length = len(data)
+
+            if length > 0:
+                messages.append(Message(offset=start, length=length, data=data, index=i))
+
+        return messages
+
+    def _infer_fields(self, messages: list[Message]) -> list[Field]:
+        """Infer field boundaries and types from messages.
+
+        Uses entropy analysis and value analysis to detect field boundaries.
+
+        Args:
+            messages: List of messages to analyze.
+
+        Returns:
+            List of Field objects with inferred types.
+        """
+        if not messages:
+            return []
+
+        # Find consistent message length (use most common)
+        lengths = [msg.length for msg in messages]
+        length_counts = Counter(lengths)
+        most_common_length = length_counts.most_common(1)[0][0]
+
+        # Filter messages with consistent length
+        consistent_messages = [msg for msg in messages if msg.length == most_common_length]
+
+        if len(consistent_messages) < self.min_messages:
+            # Not enough consistent messages
+            return []
+
+        # Align messages as 2D numpy array (rows=messages, cols=bytes)
+        aligned = self._align_messages(consistent_messages, most_common_length)
+
+        # Detect field boundaries using entropy analysis
+        field_boundaries = self._detect_field_boundaries(aligned)
+
+        # Classify fields
+        fields: list[Field] = []
+        for i, start_offset in enumerate(field_boundaries):
+            # Calculate field length
+            if i + 1 < len(field_boundaries):
+                length = field_boundaries[i + 1] - start_offset
+            else:
+                length = most_common_length - start_offset
+
+            # Ensure reasonable field length
+            if length <= 0 or length > self.max_field_length:
+                continue
+
+            # Classify field type
+            field = self._classify_field(aligned, start_offset, length)
+            fields.append(field)
+
+        # Merge adjacent similar fields
+        fields = self._merge_adjacent_fields(fields)
+
+        return fields
+
+    def _align_messages(self, messages: list[Message], length: int) -> NDArray[np.uint8]:
+        """Align messages as 2D numpy array.
+
+        Args:
+            messages: List of messages with same length.
+            length: Message length in bytes.
+
+        Returns:
+            2D array with shape (n_messages, length).
+        """
+        n_messages = len(messages)
+        aligned = np.zeros((n_messages, length), dtype=np.uint8)
+
+        for i, msg in enumerate(messages):
+            # Copy message bytes to row
+            msg_bytes = np.frombuffer(msg.data[:length], dtype=np.uint8)
+            aligned[i, : len(msg_bytes)] = msg_bytes
+
+        return aligned
+
+    def _detect_field_boundaries(self, aligned: NDArray[np.uint8]) -> list[int]:
+        """Detect field boundaries using sliding window entropy analysis.
+
+        Args:
+            aligned: Aligned message array (n_messages x message_length).
+
+        Returns:
+            List of field boundary offsets.
+        """
+        if aligned.shape[0] < 2 or aligned.shape[1] < self.entropy_window:
+            return [0]
+
+        n_messages, msg_length = aligned.shape
+        boundaries = [0]  # Always start at offset 0
+
+        # Compute entropy for each byte position
+        entropies = np.zeros(msg_length)
+        for pos in range(msg_length):
+            values = aligned[:, pos]
+            entropies[pos] = self._compute_byte_entropy(values)
+
+        # Smooth entropy using sliding window
+        window = self.entropy_window
+        smoothed = np.convolve(entropies, np.ones(window) / window, mode="valid")
+
+        # Detect boundaries where entropy changes significantly
+        entropy_diff = np.abs(np.diff(smoothed))
+        threshold = np.mean(entropy_diff) + self.entropy_threshold * np.std(entropy_diff)
+
+        for i in range(len(entropy_diff)):
+            if entropy_diff[i] > threshold:
+                # Found potential boundary
+                boundary_pos = i + window // 2
+                if boundary_pos > boundaries[-1] + 1:  # Avoid duplicate boundaries
+                    boundaries.append(boundary_pos)
+
+        # If no boundaries detected beyond start, add some default boundaries
+        if len(boundaries) == 1 and msg_length >= 16:
+            # Add boundaries at common field sizes (4, 8, 12, 16 bytes)
+            for offset in [4, 8, 12, 16]:
+                if offset < msg_length and offset not in boundaries:
+                    boundaries.append(offset)
+
+        return sorted(boundaries)
+
+    def _compute_byte_entropy(self, values: NDArray[np.uint8]) -> float:
+        """Compute Shannon entropy for byte values.
+
+        Args:
+            values: Array of byte values.
+
+        Returns:
+            Entropy in bits (0-8).
+        """
+        if len(values) == 0:
+            return 0.0
+
+        # Count value frequencies
+        counts = np.bincount(values, minlength=256)
+        probabilities = counts[counts > 0] / len(values)
+
+        # Shannon entropy
+        entropy = -np.sum(probabilities * np.log2(probabilities))
+        return float(entropy)
+
+    def _classify_field(self, aligned: NDArray[np.uint8], offset: int, length: int) -> Field:
+        """Classify field type based on value analysis.
+
+        Args:
+            aligned: Aligned message array.
+            offset: Field start offset.
+            length: Field length in bytes.
+
+        Returns:
+            Classified Field object.
+        """
+        # Extract field values from all messages
+        field_data = aligned[:, offset : offset + length]
+
+        # Convert to comparable values based on length
+        if length == 1:
+            values_int = field_data[:, 0].astype(int)
+        elif length == 2:
+            # uint16 interpretation
+            values_int = field_data[:, 0].astype(int) + field_data[:, 1].astype(int) * 256
+        elif length == 4:
+            # uint32 interpretation
+            values_int = (
+                field_data[:, 0].astype(int)
+                + field_data[:, 1].astype(int) * 256
+                + field_data[:, 2].astype(int) * 65536
+                + field_data[:, 3].astype(int) * 16777216
+            )
+        else:
+            # For longer fields, use bytes directly
+            values_int = None
+
+        # Check if field is constant
+        unique_count = len(np.unique(field_data, axis=0))
+        is_constant = unique_count == 1
+
+        if is_constant:
+            field_type = FieldType.CONSTANT
+            name = f"const_{offset:04d}"
+            sample_values = [field_data[0].tobytes()]
+        elif values_int is not None:
+            # Analyze numeric field
+            field_type, name = self._classify_numeric_field(values_int, offset, length)
+            sample_values = values_int[:5].tolist()
+        else:
+            # Unknown/payload field
+            field_type = FieldType.UNKNOWN
+            name = f"field_{offset:04d}"
+            sample_values = [row.tobytes() for row in field_data[:5]]
+
+        # Compute statistics
+        statistics: dict[str, float] = {}
+        if values_int is not None:
+            statistics = {
+                "mean": float(np.mean(values_int)),
+                "std": float(np.std(values_int)),
+                "min": float(np.min(values_int)),
+                "max": float(np.max(values_int)),
+                "unique_count": int(len(np.unique(values_int))),
+            }
+
+        return Field(
+            name=name,
+            offset=offset,
+            length=length,
+            field_type=field_type,
+            constant=is_constant,
+            values=sample_values,
+            statistics=statistics,
+            metadata={"entropy": self._compute_byte_entropy(field_data.flatten())},
+        )
+
+    def _classify_numeric_field(
+        self, values: NDArray[np.int64], offset: int, length: int
+    ) -> tuple[FieldType, str]:
+        """Classify numeric field type.
+
+        Args:
+            values: Field values as integers.
+            offset: Field offset in message.
+            length: Field length in bytes.
+
+        Returns:
+            Tuple of (FieldType, field_name).
+        """
+        # Check for sequence (incrementing values)
+        if len(values) > 2:
+            diffs = np.diff(values)
+            if np.all(diffs == 1):
+                return FieldType.SEQUENCE, f"seq_{offset:04d}"
+            if np.all(diffs >= 0) and np.mean(diffs) > 0.8:
+                # Mostly incrementing
+                return FieldType.SEQUENCE, f"counter_{offset:04d}"
+
+        # Check for timestamp (large values, mostly increasing)
+        if length >= 4:
+            if np.min(values) > 1_000_000 and np.all(np.diff(values) >= 0):
+                return FieldType.TIMESTAMP, f"timestamp_{offset:04d}"
+
+        # Check for length field (values match some pattern)
+        unique_count = len(np.unique(values))
+        if unique_count < len(values) * 0.5 and np.max(values) < 10_000:
+            return FieldType.LENGTH, f"length_{offset:04d}"
+
+        # Check for checksum (appears random but not too large)
+        entropy = self._compute_byte_entropy(values.astype(np.uint8))
+        if entropy > 5.0 and length <= 4:
+            return FieldType.CHECKSUM, f"checksum_{offset:04d}"
+
+        # Default to unknown
+        return FieldType.UNKNOWN, f"field_{offset:04d}"
+
+    def _merge_adjacent_fields(self, fields: list[Field]) -> list[Field]:
+        """Merge adjacent fields with similar characteristics.
+
+        Args:
+            fields: List of fields to merge.
+
+        Returns:
+            Merged field list.
+        """
+        if len(fields) <= 1:
+            return fields
+
+        merged: list[Field] = []
+        current = fields[0]
+
+        for next_field in fields[1:]:
+            # Check if fields should be merged
+            if (
+                current.field_type == next_field.field_type
+                and current.offset + current.length == next_field.offset
+                and current.field_type in (FieldType.PAYLOAD, FieldType.UNKNOWN, FieldType.CONSTANT)
+            ):
+                # Merge fields
+                current = Field(
+                    name=current.name,
+                    offset=current.offset,
+                    length=current.length + next_field.length,
+                    field_type=current.field_type,
+                    constant=current.constant and next_field.constant,
+                    values=current.values,
+                    statistics=current.statistics,
+                    metadata=current.metadata,
+                )
+            else:
+                # Save current and start new
+                merged.append(current)
+                current = next_field
+
+        # Add last field
+        merged.append(current)
+
+        return merged
+
+    def _calculate_confidence(
+        self, messages: list[Message], fields: list[Field], patterns: list[Pattern]
+    ) -> float:
+        """Calculate confidence in structure inference.
+
+        Args:
+            messages: Extracted messages.
+            fields: Inferred fields.
+            patterns: Detected patterns.
+
+        Returns:
+            Confidence score (0.0 to 1.0).
+        """
+        if not messages:
+            return 0.0
+
+        confidence = 0.0
+
+        # Factor 1: Message length consistency (up to 0.4)
+        lengths = [msg.length for msg in messages]
+        if len(lengths) > 1:
+            length_std = float(np.std(lengths))
+            length_mean = float(np.mean(lengths))
+            if length_mean > 0:
+                cv = length_std / length_mean  # Coefficient of variation
+                length_confidence = float(max(0.0, 1.0 - cv))  # Lower CV = higher confidence
+                confidence += length_confidence * 0.4
+
+        # Factor 2: Number of fields detected (up to 0.3)
+        if fields:
+            # More fields = better structure understanding
+            field_confidence = min(1.0, len(fields) / 10.0)
+            confidence += field_confidence * 0.3
+
+        # Factor 3: Pattern support (up to 0.3)
+        if patterns:
+            # Having regular patterns increases confidence
+            regular_patterns = sum(1 for p in patterns if p.regular)
+            pattern_confidence = min(1.0, regular_patterns / 3.0)
+            confidence += pattern_confidence * 0.3
+
+        return min(1.0, confidence)
+
+
+__all__ = ["StructureInferencer"]

oscura/analyzers/binary/export/__init__.py

@@ -0,0 +1,9 @@
+"""Export tools for binary analysis."""
+
+from __future__ import annotations
+
+from oscura.analyzers.binary.export.dissector import DissectorGenerator
+
+__all__ = [
+    "DissectorGenerator",
+]