oscura 0.10.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

oscura/__init__.py

@@ -53,7 +53,7 @@ try:
     __version__ = version("oscura")
 except Exception:
     # Fallback for development/testing when package not installed
-    __version__ = "0.10.0"
+    __version__ = "0.12.0"
 
 __author__ = "Oscura Contributors"
 

oscura/__main__.py

@@ -106,6 +106,10 @@ def download_file(url: str, dest: Path, checksum: str | None = None) -> bool:
         # Create SSL context that works in most environments
         context = ssl.create_default_context()
 
+        # SEC-003: Validate URL scheme to prevent file:// attacks
+        if not url.startswith(("http://", "https://")):
+            raise ValueError(f"Unsupported URL scheme (only http/https allowed): {url}")
+
         print(f"  Downloading: {url}")
 
         with urllib.request.urlopen(url, context=context, timeout=30) as response:

oscura/analyzers/binary/__init__.py

@@ -0,0 +1,36 @@
+"""Binary file analysis framework.
+
+Provides comprehensive analysis of binary files including:
+- Multi-stage dtype detection with validation
+- Pattern mining for repeating sequences
+- Message and field structure inference
+- Semantic field classification
+- Protocol identification
+- Visualization and dissector generation
+"""
+
+from __future__ import annotations
+
+from oscura.analyzers.binary.core.pipeline import BinaryAnalyzer
+from oscura.analyzers.binary.core.results import (
+    BinaryAnalysisResult,
+    EncodingResult,
+    Field,
+    FieldType,
+    Message,
+    Pattern,
+    PatternRole,
+    StructureResult,
+)
+
+__all__ = [
+    "BinaryAnalysisResult",
+    "BinaryAnalyzer",
+    "EncodingResult",
+    "Field",
+    "FieldType",
+    "Message",
+    "Pattern",
+    "PatternRole",
+    "StructureResult",
+]

oscura/analyzers/binary/core/__init__.py

@@ -0,0 +1,29 @@
+"""Core binary analysis components."""
+
+from __future__ import annotations
+
+from oscura.analyzers.binary.core.file_access import BinaryFile
+from oscura.analyzers.binary.core.pipeline import BinaryAnalyzer
+from oscura.analyzers.binary.core.results import (
+    BinaryAnalysisResult,
+    EncodingResult,
+    Field,
+    FieldType,
+    Message,
+    Pattern,
+    PatternRole,
+    StructureResult,
+)
+
+__all__ = [
+    "BinaryAnalysisResult",
+    "BinaryAnalyzer",
+    "BinaryFile",
+    "EncodingResult",
+    "Field",
+    "FieldType",
+    "Message",
+    "Pattern",
+    "PatternRole",
+    "StructureResult",
+]

oscura/analyzers/binary/core/file_access.py

@@ -0,0 +1,193 @@
+"""Efficient binary file access with memory mapping, caching, and thread safety."""
+
+from __future__ import annotations
+
+import mmap
+import threading
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+if TYPE_CHECKING:
+    from os import PathLike
+
+
+class BinaryFile:
+    """Memory-efficient, thread-safe binary file access layer.
+
+    Provides efficient access to binary files of any size using memory mapping
+    and caching. Designed to handle multi-GB files with minimal memory footprint.
+    All operations are protected by an RLock for thread safety.
+
+    Example:
+        >>> bf = BinaryFile("large_file.bin")
+        >>> data = bf.read_bytes(0, 1024)  # Read first 1KB
+        >>> samples = bf.sample_locations(n_samples=5)  # Sample 5 locations
+        >>> bf.close()
+    """
+
+    def __init__(self, path: str | PathLike[str]):
+        """Initialize binary file access.
+
+        Args:
+            path: Path to binary file.
+        """
+        self.path = Path(path)
+        self.size = self.path.stat().st_size
+        self._file_handle: Any | None = None
+        self._mmap: mmap.mmap | None = None
+        self._lock = threading.RLock()
+
+    def _ensure_open(self) -> None:
+        """Ensure file is open and memory mapped.
+
+        Thread-safe. Cleans up partial state on failure to prevent
+        resource leaks.
+        """
+        with self._lock:
+            if self._mmap is None:
+                try:
+                    self._file_handle = open(self.path, "rb")  # noqa: SIM115
+                    if self.size > 0:
+                        self._mmap = mmap.mmap(
+                            self._file_handle.fileno(), 0, access=mmap.ACCESS_READ
+                        )
+                except Exception:
+                    if self._file_handle is not None:
+                        self._file_handle.close()
+                        self._file_handle = None
+                    raise
+
+    def read_bytes(self, offset: int, length: int) -> bytes:
+        """Read bytes from file at given offset.
+
+        Args:
+            offset: Byte offset to start reading.
+            length: Number of bytes to read.
+
+        Returns:
+            Bytes read from file.
+
+        Example:
+            >>> bf = BinaryFile("data.bin")
+            >>> header = bf.read_bytes(0, 16)
+        """
+        if offset + length > self.size:
+            length = max(0, self.size - offset)
+
+        if length <= 0:
+            return b""
+
+        self._ensure_open()
+
+        with self._lock:
+            if self._mmap is not None:
+                return bytes(self._mmap[offset : offset + length])
+
+        return b""
+
+    def read_regions(self, regions: list[tuple[int, int]]) -> list[bytes]:
+        """Read multiple regions efficiently.
+
+        Args:
+            regions: List of (offset, length) tuples.
+
+        Returns:
+            List of bytes for each region.
+
+        Example:
+            >>> bf = BinaryFile("data.bin")
+            >>> regions = [(0, 16), (1000, 16), (2000, 16)]
+            >>> data = bf.read_regions(regions)
+        """
+        return [self.read_bytes(offset, length) for offset, length in regions]
+
+    def sample_locations(self, n_samples: int = 5, sample_size: int = 8192) -> list[bytes]:
+        """Sample data from multiple file locations.
+
+        Samples evenly distributed throughout the file for robust analysis.
+
+        Args:
+            n_samples: Number of locations to sample.
+            sample_size: Bytes to read at each location.
+
+        Returns:
+            List of byte samples from different file locations.
+
+        Example:
+            >>> bf = BinaryFile("data.bin")
+            >>> samples = bf.sample_locations(n_samples=5, sample_size=4096)
+        """
+        if self.size < sample_size:
+            return [self.read_bytes(0, self.size)]
+
+        # Calculate evenly-spaced locations
+        locations = np.linspace(0, self.size - sample_size, n_samples, dtype=int)
+
+        return [self.read_bytes(int(loc), sample_size) for loc in locations]
+
+    def read_array(
+        self, offset: int, count: int, dtype: str | np.dtype[Any]
+    ) -> np.ndarray[Any, Any]:
+        """Read data as numpy array.
+
+        Args:
+            offset: Byte offset to start reading.
+            count: Number of elements to read (-1 for all).
+            dtype: NumPy dtype for interpretation.
+
+        Returns:
+            NumPy array of data.
+
+        Example:
+            >>> bf = BinaryFile("data.bin")
+            >>> data = bf.read_array(0, 1000, dtype="uint16")
+        """
+        np_dtype = np.dtype(dtype)
+        bytes_per_element = np_dtype.itemsize
+        byte_offset = offset * bytes_per_element
+
+        if count == -1:
+            # Read all available data
+            available_bytes = self.size - byte_offset
+            count = available_bytes // bytes_per_element
+
+        if count <= 0:
+            return np.array([], dtype=np_dtype)
+
+        bytes_to_read = count * bytes_per_element
+
+        if byte_offset + bytes_to_read > self.size:
+            bytes_to_read = self.size - byte_offset
+            count = bytes_to_read // bytes_per_element
+
+        data_bytes = self.read_bytes(byte_offset, bytes_to_read)
+        return np.frombuffer(data_bytes, dtype=np_dtype, count=count)
+
+    def close(self) -> None:
+        """Close file and release resources."""
+        with self._lock:
+            if self._mmap is not None:
+                self._mmap.close()
+                self._mmap = None
+
+            if self._file_handle is not None:
+                self._file_handle.close()
+                self._file_handle = None
+
+    def __enter__(self) -> BinaryFile:
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        """Context manager exit."""
+        self.close()
+
+    def __del__(self) -> None:
+        """Destructor to ensure cleanup."""
+        self.close()
+
+    def __repr__(self) -> str:
+        """String representation."""
+        return f"BinaryFile('{self.path}', size={self.size:,} bytes)"

oscura/analyzers/binary/core/pipeline.py

@@ -0,0 +1,161 @@
+"""Binary analysis pipeline orchestrator."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from oscura.analyzers.binary.core.file_access import BinaryFile
+from oscura.analyzers.binary.core.results import BinaryAnalysisResult
+
+if TYPE_CHECKING:
+    from os import PathLike
+
+
+class BinaryAnalyzer:
+    """Complete binary file analysis pipeline.
+
+    Orchestrates multi-stage analysis of binary files including encoding
+    detection, pattern mining, structure inference, and semantic analysis.
+
+    Example:
+        >>> analyzer = BinaryAnalyzer("unknown_file.bin")
+        >>> results = analyzer.analyze(max_samples=100_000)
+        >>> print(results)
+        >>> results.to_dict()  # Export to dictionary
+    """
+
+    def __init__(self, file_path: str | PathLike[str]):
+        """Initialize binary analyzer.
+
+        Args:
+            file_path: Path to binary file to analyze.
+        """
+        self.file_path = Path(file_path)
+        self.binary_file = BinaryFile(self.file_path)
+        self.results: BinaryAnalysisResult | None = None
+
+    def analyze(
+        self,
+        max_samples: int = 100_000,
+        enable_structure_inference: bool = True,
+        enable_semantic_analysis: bool = True,
+    ) -> BinaryAnalysisResult:
+        """Run complete analysis pipeline.
+
+        Args:
+            max_samples: Maximum samples to load for analysis.
+            enable_structure_inference: Enable message/field structure detection.
+            enable_semantic_analysis: Enable field type classification.
+
+        Returns:
+            Complete analysis results.
+
+        Example:
+            >>> analyzer = BinaryAnalyzer("data.bin")
+            >>> results = analyzer.analyze(max_samples=100_000)
+            >>> print(f"Detected dtype: {results.encoding.dtype}")
+            >>> print(f"Found {len(results.patterns)} patterns")
+        """
+        # Import analysis components (lazy import to avoid circular dependencies)
+        from oscura.analyzers.binary.detection.encoding import EncodingDetector
+        from oscura.analyzers.binary.detection.patterns import PatternMiner
+
+        # Stage 1: Encoding Detection
+        encoding_detector = EncodingDetector()
+        encoding = encoding_detector.detect(
+            self.binary_file, validation=True, max_samples=max_samples
+        )
+
+        # Stage 2: Pattern Mining
+        pattern_miner = PatternMiner()
+        patterns = pattern_miner.find_patterns(
+            self.binary_file, min_length=2, max_length=64, min_occurrences=3
+        )
+
+        # Stage 3: Structure Inference (if enabled and patterns found)
+        structure = None
+        if enable_structure_inference and patterns:
+            from oscura.analyzers.binary.detection.structure import StructureInferencer
+
+            structure_inferencer = StructureInferencer()
+            structure = structure_inferencer.infer(
+                self.binary_file, patterns=patterns, encoding=encoding
+            )
+
+            # Stage 4: Semantic Analysis (if enabled and structure found)
+            if enable_semantic_analysis and structure is not None and structure.has_messages:
+                from oscura.analyzers.binary.inference.fields import SemanticAnalyzer
+
+                semantic_analyzer = SemanticAnalyzer()
+                structure.fields = semantic_analyzer.analyze_fields(
+                    self.binary_file, structure=structure
+                )
+
+        # Calculate overall confidence
+        confidence = self._calculate_confidence(encoding, patterns, structure)
+
+        # Build result
+        self.results = BinaryAnalysisResult(
+            encoding=encoding,
+            patterns=patterns,
+            structure=structure,
+            confidence=confidence,
+        )
+
+        return self.results
+
+    def _calculate_confidence(self, encoding: Any, patterns: list[Any], structure: Any) -> float:
+        """Calculate overall analysis confidence.
+
+        Args:
+            encoding: Encoding detection result.
+            patterns: Pattern mining results.
+            structure: Structure inference result.
+
+        Returns:
+            Overall confidence score (0-1).
+        """
+        scores = [encoding.confidence]
+
+        # Pattern quality score
+        if patterns:
+            regular_patterns = sum(1 for p in patterns if p.regular)
+            pattern_score = min(1.0, regular_patterns / max(1, len(patterns)))
+            scores.append(pattern_score)
+
+        # Structure quality score
+        if structure and structure.has_messages:
+            scores.append(structure.confidence)
+
+        return sum(scores) / len(scores) if scores else 0.0
+
+    def export_results(self, output_path: str | PathLike[str]) -> None:
+        """Export analysis results to JSON.
+
+        Args:
+            output_path: Path to output JSON file.
+
+        Example:
+            >>> analyzer = BinaryAnalyzer("data.bin")
+            >>> results = analyzer.analyze()
+            >>> analyzer.export_results("analysis_results.json")
+        """
+        if self.results is None:
+            msg = "No results to export. Run analyze() first."
+            raise ValueError(msg)
+
+        import json
+
+        output_path = Path(output_path)
+        output_path.write_text(json.dumps(self.results.to_dict(), indent=2))
+
+    def __repr__(self) -> str:
+        """String representation."""
+        return f"BinaryAnalyzer('{self.file_path}')"
+
+    def __str__(self) -> str:
+        """Human-readable string."""
+        if self.results:
+            return str(self.results)
+        return f"BinaryAnalyzer('{self.file_path}', not yet analyzed)"

oscura/analyzers/binary/core/results.py

@@ -0,0 +1,217 @@
+"""Result types for binary analysis."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+class PatternRole(Enum):
+    """Role classification for repeating patterns."""
+
+    HEADER = "header"
+    FOOTER = "footer"
+    DELIMITER = "delimiter"
+    SYNC_MARKER = "sync_marker"
+    UNKNOWN = "unknown"
+
+
+class FieldType(Enum):
+    """Field type classification."""
+
+    CONSTANT = "constant"
+    SEQUENCE = "sequence"
+    TIMESTAMP = "timestamp"
+    CHECKSUM = "checksum"
+    PAYLOAD = "payload"
+    LENGTH = "length"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class Pattern:
+    """Repeating byte pattern in binary file."""
+
+    bytes: bytes
+    positions: list[int]
+    count: int
+    avg_spacing: float
+    spacing_variance: float
+    regular: bool
+    role: PatternRole
+
+    def to_dict(self) -> dict[str, Any]:
+        """Export to dictionary."""
+        return {
+            "hex": self.bytes.hex(" ").upper(),
+            "length": len(self.bytes),
+            "occurrences": self.count,
+            "regular_spacing": self.regular,
+            "avg_spacing": self.avg_spacing,
+            "spacing_variance": self.spacing_variance,
+            "role": self.role.value,
+            "sample_positions": self.positions[:10],
+        }
+
+
+@dataclass
+class Message:
+    """Extracted message from binary file."""
+
+    offset: int
+    length: int
+    data: bytes
+    index: int
+
+    def to_dict(self) -> dict[str, Any]:
+        """Export to dictionary."""
+        return {
+            "index": self.index,
+            "offset": self.offset,
+            "length": self.length,
+            "hex_preview": self.data[:64].hex(" ").upper()
+            if len(self.data) > 64
+            else self.data.hex(" ").upper(),
+        }
+
+
+@dataclass
+class Field:
+    """Message field with semantic information."""
+
+    name: str
+    offset: int
+    length: int
+    field_type: FieldType
+    constant: bool
+    values: list[Any] = field(default_factory=list)
+    statistics: dict[str, float] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Export to dictionary."""
+        result = {
+            "name": self.name,
+            "offset": self.offset,
+            "length": self.length,
+            "type": self.field_type.value,
+            "constant": self.constant,
+            "statistics": self.statistics,
+        }
+
+        # Add sample values
+        if self.values:
+            if isinstance(self.values[0], bytes):
+                result["sample_values"] = [v.hex(" ").upper() for v in self.values[:5]]
+            else:
+                result["sample_values"] = self.values[:5]
+
+        # Add metadata
+        if self.metadata:
+            result["metadata"] = self.metadata
+
+        return result
+
+
+@dataclass
+class EncodingResult:
+    """Encoding detection result with validation."""
+
+    dtype: str
+    confidence: float
+    alternatives: list[tuple[str, float]]
+    validation_passed: bool
+    sample_data: NDArray[np.float64]
+    statistics: dict[str, float]
+    issues: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Export to dictionary."""
+        return {
+            "detected_dtype": self.dtype,
+            "confidence": f"{self.confidence:.1%}",
+            "alternatives": [
+                {"dtype": dt, "confidence": f"{conf:.1%}"} for dt, conf in self.alternatives[:3]
+            ],
+            "validation_passed": self.validation_passed,
+            "statistics": {
+                "mean": float(np.mean(self.sample_data)),
+                "std": float(np.std(self.sample_data)),
+                "min": float(np.min(self.sample_data)),
+                "max": float(np.max(self.sample_data)),
+            },
+            "issues": self.issues,
+        }
+
+
+@dataclass
+class StructureResult:
+    """Inferred file structure result."""
+
+    has_messages: bool
+    message_length: int | None
+    message_count: int
+    messages: list[Message]
+    fields: list[Field]
+    confidence: float
+
+    def to_dict(self) -> dict[str, Any]:
+        """Export to dictionary."""
+        result: dict[str, Any] = {
+            "has_messages": self.has_messages,
+            "message_length": self.message_length,
+            "message_count": self.message_count,
+            "confidence": f"{self.confidence:.1%}",
+        }
+
+        if self.fields:
+            result["fields"] = [f.to_dict() for f in self.fields]
+
+        if self.messages:
+            result["sample_messages"] = [m.to_dict() for m in self.messages[:5]]
+
+        return result
+
+
+@dataclass
+class BinaryAnalysisResult:
+    """Complete binary analysis result."""
+
+    encoding: EncodingResult
+    patterns: list[Pattern]
+    structure: StructureResult | None = None
+    confidence: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Export complete results to dictionary."""
+        result = {
+            "encoding": self.encoding.to_dict(),
+            "patterns": [p.to_dict() for p in self.patterns],
+            "confidence": f"{self.confidence:.1%}",
+        }
+
+        if self.structure:
+            result["structure"] = self.structure.to_dict()
+
+        return result
+
+    def __str__(self) -> str:
+        """Human-readable summary."""
+        lines = [
+            "=== Binary Analysis Results ===",
+            f"Encoding: {self.encoding.dtype} ({self.encoding.confidence:.1%} confidence)",
+            f"Patterns found: {len(self.patterns)}",
+        ]
+
+        if self.structure and self.structure.has_messages:
+            lines.append(f"Messages: {self.structure.message_count}")
+            lines.append(f"Message length: {self.structure.message_length} bytes")
+            lines.append(f"Fields identified: {len(self.structure.fields)}")
+
+        lines.append(f"Overall confidence: {self.confidence:.1%}")
+
+        return "\n".join(lines)

oscura/analyzers/binary/detection/__init__.py

@@ -0,0 +1,10 @@
+"""Binary file detection modules.
+
+Encoding detection, pattern mining, and structure inference.
+"""
+
+from oscura.analyzers.binary.detection.encoding import EncodingDetector
+from oscura.analyzers.binary.detection.patterns import PatternMiner
+from oscura.analyzers.binary.detection.structure import StructureInferencer
+
+__all__ = ["EncodingDetector", "PatternMiner", "StructureInferencer"]