oscura 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

oscura/analyzers/binary/export/dissector.py

@@ -0,0 +1,174 @@
+"""Dissector generation for binary protocols."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from oscura.analyzers.binary.core.results import BinaryAnalysisResult, Field
+
+
+class DissectorGenerator:
+    """Generate protocol dissectors from binary analysis results.
+
+    Supports Wireshark Lua and Kaitai Struct formats.
+
+    Example:
+        >>> generator = DissectorGenerator()
+        >>> generator.generate_wireshark_lua(results, "protocol.lua")
+        >>> generator.generate_kaitai_struct(results, "protocol.ksy")
+    """
+
+    def generate_wireshark_lua(self, result: BinaryAnalysisResult, output_path: str | Path) -> None:
+        """Generate Wireshark Lua dissector.
+
+        Args:
+            result: Binary analysis result.
+            output_path: Path to save Lua dissector.
+
+        Example:
+            >>> generator = DissectorGenerator()
+            >>> generator.generate_wireshark_lua(results, "custom.lua")
+        """
+        if not result.structure or not result.structure.has_messages:
+            # Create minimal dissector
+            lua_code = """-- No structure detected
+-- This is a placeholder dissector
+
+local proto = Proto("custom", "Custom Protocol")
+
+function proto.dissector(buffer, pinfo, tree)
+    pinfo.cols.protocol = "CUSTOM"
+    local subtree = tree:add(proto, buffer(), "Custom Protocol (No structure detected)")
+end
+
+DissectorTable.get("wtap_encap"):add(wtap.USER0, proto)
+"""
+            Path(output_path).write_text(lua_code)
+            return
+
+        # Generate full dissector
+        fields = result.structure.fields
+        message_length = result.structure.message_length
+
+        lua_code = f"""-- Auto-generated Wireshark dissector
+-- Generated by Oscura Binary Analysis
+-- Message length: {message_length} bytes
+
+local proto = Proto("custom", "Custom Binary Protocol")
+
+-- Field definitions
+"""
+
+        # Add field definitions
+        for field in fields:
+            field_name = field.name.replace(" ", "_").lower()
+            lua_code += f'proto.fields.{field_name} = ProtoField.bytes("custom.{field_name}", "{field.name}", base.HEX)\n'
+
+        lua_code += """
+-- Dissector function
+function proto.dissector(buffer, pinfo, tree)
+    local length = buffer:len()
+    if length == 0 then return end
+
+    pinfo.cols.protocol = proto.name
+
+    local subtree = tree:add(proto, buffer(), "Custom Binary Protocol")
+
+"""
+
+        # Add field parsing
+        for field in fields:
+            field_name = field.name.replace(" ", "_").lower()
+            lua_code += f"    -- {field.name} ({field.field_type.value})\n"
+            lua_code += f"    if length >= {field.offset + field.length} then\n"
+            lua_code += f"        subtree:add(proto.fields.{field_name}, buffer({field.offset}, {field.length}))\n"
+            lua_code += "    end\n\n"
+
+        lua_code += """end
+
+-- Register dissector
+DissectorTable.get("wtap_encap"):add(wtap.USER0, proto)
+
+-- Also register for UDP port (example)
+-- local udp_table = DissectorTable.get("udp.port")
+-- udp_table:add(12345, proto)
+"""
+
+        Path(output_path).write_text(lua_code)
+
+    def generate_kaitai_struct(self, result: BinaryAnalysisResult, output_path: str | Path) -> None:
+        """Generate Kaitai Struct YAML definition.
+
+        Args:
+            result: Binary analysis result.
+            output_path: Path to save Kaitai Struct definition.
+
+        Example:
+            >>> generator = DissectorGenerator()
+            >>> generator.generate_kaitai_struct(results, "protocol.ksy")
+        """
+        if not result.structure or not result.structure.has_messages:
+            # Create minimal definition
+            ksy_content = """meta:
+  id: custom_protocol
+  title: Custom Binary Protocol
+  endian: le
+
+doc: |
+  No structure detected. This is a placeholder definition.
+
+seq: []
+"""
+            Path(output_path).write_text(ksy_content)
+            return
+
+        # Generate full definition
+        fields = result.structure.fields
+
+        ksy_content = f"""meta:
+  id: custom_protocol
+  title: Custom Binary Protocol
+  endian: le
+
+doc: |
+  Auto-generated Kaitai Struct definition from Oscura Binary Analysis.
+  Message length: {result.structure.message_length} bytes
+  Messages detected: {result.structure.message_count}
+
+seq:
+"""
+
+        # Add field definitions
+        for field in fields:
+            field_name = field.name.replace(" ", "_").lower()
+            ksy_type = self._map_field_to_kaitai_type(field)
+
+            ksy_content += f"  - id: {field_name}\n"
+            ksy_content += f"    type: {ksy_type}\n"
+            ksy_content += f"    doc: '{field.field_type.value} field at offset {field.offset}'\n"
+
+        Path(output_path).write_text(ksy_content)
+
+    def _map_field_to_kaitai_type(self, field: Field) -> str:
+        """Map field to Kaitai Struct type.
+
+        Args:
+            field: Field to map.
+
+        Returns:
+            Kaitai type string.
+        """
+        # Map by field length
+        if field.length == 1:
+            return "u1"
+        elif field.length == 2:
+            return "u2"
+        elif field.length == 4:
+            return "u4"
+        elif field.length == 8:
+            return "u8"
+        else:
+            # Variable or custom length
+            return f"bytes({field.length})" if field.length < 1024 else "bytes_eos"

oscura/analyzers/binary/inference/__init__.py

@@ -0,0 +1,15 @@
+"""Semantic analysis for binary fields."""
+
+from __future__ import annotations
+
+from oscura.analyzers.binary.inference.checksums import ChecksumAnalyzer
+from oscura.analyzers.binary.inference.fields import SemanticAnalyzer
+from oscura.analyzers.binary.inference.sequences import SequenceAnalyzer
+from oscura.analyzers.binary.inference.timestamps import TimestampAnalyzer
+
+__all__ = [
+    "ChecksumAnalyzer",
+    "SemanticAnalyzer",
+    "SequenceAnalyzer",
+    "TimestampAnalyzer",
+]

oscura/analyzers/binary/inference/checksums.py

@@ -0,0 +1,214 @@
+"""Checksum field analysis."""
+
+from __future__ import annotations
+
+import zlib
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from oscura.analyzers.binary.core.results import Field, Message
+
+
+class ChecksumAnalyzer:
+    """Analyze checksum fields.
+
+    Tests various checksum algorithms and validates against message data.
+
+    Example:
+        >>> analyzer = ChecksumAnalyzer()
+        >>> metadata = analyzer.analyze(field, messages)
+        >>> if metadata["algorithm"] != "unknown":
+        ...     print(f"Algorithm: {metadata['algorithm']}")
+        ...     print(f"Match rate: {metadata['match_rate']:.1%}")
+    """
+
+    def analyze(self, field: Field, messages: list[Message]) -> dict[str, Any]:
+        """Analyze checksum field.
+
+        Args:
+            field: Field to analyze.
+            messages: List of messages containing field.
+
+        Returns:
+            Metadata dict with checksum information.
+
+        Example:
+            >>> result = analyzer.analyze(field, messages)
+            >>> result["algorithm"]  # e.g., "crc32"
+            >>> result["match_rate"]  # e.g., 0.997
+        """
+        if len(messages) < 5:
+            return {
+                "algorithm": "unknown",
+                "reason": "insufficient_data",
+                "match_rate": 0.0,
+            }
+
+        # Test various algorithms
+        algorithms = []
+
+        if field.length == 1:
+            algorithms = ["sum8", "xor8"]
+        elif field.length == 2:
+            algorithms = ["crc16", "fletcher16", "sum16"]
+        elif field.length == 4:
+            algorithms = ["crc32", "adler32", "sum32"]
+        else:
+            return {
+                "algorithm": "unknown",
+                "reason": "unsupported_length",
+                "match_rate": 0.0,
+            }
+
+        best_algorithm = "unknown"
+        best_match_rate = 0.0
+        best_matches = 0
+
+        for algorithm in algorithms:
+            match_rate, matches = self._test_algorithm(algorithm, field, messages)
+
+            if match_rate > best_match_rate:
+                best_match_rate = match_rate
+                best_algorithm = algorithm
+                best_matches = matches
+
+        # Require >95% match rate for confirmation
+        if best_match_rate < 0.95:
+            return {
+                "algorithm": "unknown",
+                "reason": "low_match_rate",
+                "match_rate": best_match_rate,
+                "tested_algorithms": algorithms,
+            }
+
+        return {
+            "algorithm": best_algorithm,
+            "match_rate": best_match_rate,
+            "validated_count": best_matches,
+            "total_messages": len(messages),
+            "confidence": min(1.0, best_match_rate),
+        }
+
+    def _test_algorithm(
+        self, algorithm: str, field: Field, messages: list[Message]
+    ) -> tuple[float, int]:
+        """Test a checksum algorithm against messages.
+
+        Args:
+            algorithm: Algorithm name to test.
+            field: Checksum field.
+            messages: List of messages.
+
+        Returns:
+            (match_rate, match_count) tuple.
+        """
+        matches = 0
+        tested = 0
+
+        for msg in messages[:100]:  # Test first 100 messages
+            if field.offset + field.length > len(msg.data):
+                continue
+
+            # Extract expected checksum from field
+            expected_bytes = msg.data[field.offset : field.offset + field.length]
+            expected = int.from_bytes(expected_bytes, byteorder="little", signed=False)
+
+            # Calculate checksum over payload (excluding checksum field)
+            # Try checksumming data before and after the field
+            payload_before = msg.data[: field.offset]
+            payload_after = msg.data[field.offset + field.length :]
+
+            # Most common: checksum covers everything except itself
+            payload = payload_before + payload_after
+
+            # Calculate checksum
+            calculated = self._calculate_checksum(algorithm, payload)
+
+            if calculated == expected:
+                matches += 1
+
+            tested += 1
+
+        match_rate = matches / tested if tested > 0 else 0.0
+        return match_rate, matches
+
+    def _calculate_checksum(self, algorithm: str, data: bytes) -> int:
+        """Calculate checksum using specified algorithm.
+
+        Args:
+            algorithm: Algorithm name.
+            data: Data to checksum.
+
+        Returns:
+            Checksum value.
+        """
+        if algorithm == "crc32":
+            return zlib.crc32(data) & 0xFFFFFFFF
+
+        elif algorithm == "adler32":
+            return zlib.adler32(data) & 0xFFFFFFFF
+
+        elif algorithm == "crc16":
+            return self._crc16(data)
+
+        elif algorithm == "fletcher16":
+            return self._fletcher16(data)
+
+        elif algorithm == "sum8":
+            return sum(data) & 0xFF
+
+        elif algorithm == "xor8":
+            result = 0
+            for b in data:
+                result ^= b
+            return result & 0xFF
+
+        elif algorithm == "sum16":
+            return sum(data) & 0xFFFF
+
+        elif algorithm == "sum32":
+            return sum(data) & 0xFFFFFFFF
+
+        return 0
+
+    def _crc16(self, data: bytes) -> int:
+        """Calculate CRC-16 (CCITT).
+
+        Args:
+            data: Input data.
+
+        Returns:
+            CRC-16 value.
+        """
+        crc = 0xFFFF
+
+        for byte in data:
+            crc ^= byte << 8
+
+            for _ in range(8):
+                if crc & 0x8000:
+                    crc = (crc << 1) ^ 0x1021
+                else:
+                    crc = crc << 1
+
+                crc &= 0xFFFF
+
+        return crc
+
+    def _fletcher16(self, data: bytes) -> int:
+        """Calculate Fletcher-16 checksum.
+
+        Args:
+            data: Input data.
+
+        Returns:
+            Fletcher-16 value.
+        """
+        sum1 = 0
+        sum2 = 0
+
+        for byte in data:
+            sum1 = (sum1 + byte) % 255
+            sum2 = (sum2 + sum1) % 255
+
+        return (sum2 << 8) | sum1

oscura/analyzers/binary/inference/fields.py

@@ -0,0 +1,150 @@
+"""Semantic field analysis coordinator."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from oscura.analyzers.binary.core.results import FieldType
+from oscura.analyzers.binary.inference.checksums import ChecksumAnalyzer
+from oscura.analyzers.binary.inference.sequences import SequenceAnalyzer
+from oscura.analyzers.binary.inference.timestamps import TimestampAnalyzer
+
+if TYPE_CHECKING:
+    from oscura.analyzers.binary.core.file_access import BinaryFile
+    from oscura.analyzers.binary.core.results import Field, StructureResult
+
+
+class SemanticAnalyzer:
+    """Coordinate semantic analysis of binary fields.
+
+    Enhances field information with semantic metadata including:
+    - Sequence number detection and analysis
+    - Timestamp format identification
+    - Checksum algorithm detection and validation
+    - Payload characteristics
+
+    Example:
+        >>> analyzer = SemanticAnalyzer()
+        >>> enhanced_fields = analyzer.analyze_fields(binary_file, structure)
+        >>> for field in enhanced_fields:
+        ...     if field.field_type == FieldType.CHECKSUM:
+        ...         print(f"Checksum: {field.metadata['algorithm']}")
+    """
+
+    def __init__(self) -> None:
+        """Initialize semantic analyzer."""
+        self.sequence_analyzer = SequenceAnalyzer()
+        self.timestamp_analyzer = TimestampAnalyzer()
+        self.checksum_analyzer = ChecksumAnalyzer()
+
+    def analyze_fields(self, file: BinaryFile, structure: StructureResult) -> list[Field]:
+        """Enhance fields with semantic analysis.
+
+        Args:
+            file: Binary file being analyzed.
+            structure: Structure inference result.
+
+        Returns:
+            List of enhanced fields with metadata.
+
+        Example:
+            >>> analyzer = SemanticAnalyzer()
+            >>> fields = analyzer.analyze_fields(binary_file, structure_result)
+        """
+        if not structure.has_messages or not structure.fields:
+            return structure.fields
+
+        enhanced_fields = []
+
+        for field in structure.fields:
+            # Analyze based on field type
+            if field.field_type == FieldType.SEQUENCE:
+                metadata = self.sequence_analyzer.analyze(field, structure.messages)
+                field.metadata.update(metadata)
+
+            elif field.field_type == FieldType.TIMESTAMP:
+                metadata = self.timestamp_analyzer.analyze(field, structure.messages)
+                field.metadata.update(metadata)
+
+            elif field.field_type == FieldType.CHECKSUM:
+                metadata = self.checksum_analyzer.analyze(field, structure.messages)
+                field.metadata.update(metadata)
+
+            elif field.field_type == FieldType.PAYLOAD:
+                metadata = self._analyze_payload(field, structure.messages)
+                field.metadata.update(metadata)
+
+            enhanced_fields.append(field)
+
+        return enhanced_fields
+
+    def _analyze_payload(self, field: Field, messages: list[Any]) -> dict[str, Any]:
+        """Analyze payload field characteristics.
+
+        Args:
+            field: Payload field.
+            messages: List of messages.
+
+        Returns:
+            Metadata dict with payload characteristics.
+        """
+        if len(messages) == 0:
+            return {"payload_type": "unknown"}
+
+        # Sample first few payloads
+        sample_size = min(10, len(messages))
+        entropies = []
+
+        for msg in messages[:sample_size]:
+            if field.offset + field.length <= len(msg.data):
+                payload = msg.data[field.offset : field.offset + field.length]
+                entropy = self._calculate_entropy(payload)
+                entropies.append(entropy)
+
+        if not entropies:
+            return {"payload_type": "unknown"}
+
+        avg_entropy = sum(entropies) / len(entropies)
+
+        # Classify payload type by entropy
+        if avg_entropy > 7.5:
+            payload_type = "compressed_or_encrypted"
+        elif avg_entropy > 5.0:
+            payload_type = "binary_data"
+        elif avg_entropy > 3.0:
+            payload_type = "mixed_content"
+        else:
+            payload_type = "low_entropy"
+
+        return {
+            "payload_type": payload_type,
+            "average_entropy": avg_entropy,
+            "entropy_bits_per_byte": avg_entropy,
+        }
+
+    def _calculate_entropy(self, data: bytes) -> float:
+        """Calculate Shannon entropy of data.
+
+        Args:
+            data: Input bytes.
+
+        Returns:
+            Entropy in bits per byte.
+        """
+        if len(data) == 0:
+            return 0.0
+
+        from collections import Counter
+
+        counts = Counter(data)
+        total = len(data)
+
+        entropy = 0.0
+        for count in counts.values():
+            p = count / total
+            if p > 0:
+                import math
+
+                entropy -= p * math.log2(p)
+
+        return entropy