oscura 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

oscura/analyzers/binary/detection/encoding.py

@@ -0,0 +1,624 @@
+"""Enhanced encoding detection with multi-stage validation.
+
+Provides robust encoding detection for binary files using:
+- Statistical analysis (entropy, IEEE 754 validation, value ranges)
+- Validation by loading and checking actual data
+- Consensus detection from multiple file locations
+- Prevents common misdetections (e.g., uint16 → float32)
+"""
+
+from __future__ import annotations
+
+import struct
+import warnings
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, TypedDict
+
+import numpy as np
+from numpy.typing import NDArray
+
+from oscura.analyzers.binary.core.results import EncodingResult
+
+if TYPE_CHECKING:
+    from oscura.analyzers.binary.core.file_access import BinaryFile
+
+# Number of float values to validate for IEEE 754 compliance
+ENCODING_VALIDATION_COUNT = 100
+
+# Minimum valid float ratio for IEEE 754 validation
+MIN_VALID_FLOAT_RATIO = 0.8  # 80% of values must be valid
+
+
+class DTypeInfo(TypedDict):
+    """Data type information."""
+
+    itemsize: int
+    min: float
+    max: float
+    signed: bool
+    float: bool
+
+
+# Supported data types with their properties
+SUPPORTED_DTYPES: dict[str, DTypeInfo] = {
+    "uint8": {"itemsize": 1, "min": 0, "max": 255, "signed": False, "float": False},
+    "int8": {"itemsize": 1, "min": -128, "max": 127, "signed": True, "float": False},
+    "uint16": {"itemsize": 2, "min": 0, "max": 65535, "signed": False, "float": False},
+    "int16": {"itemsize": 2, "min": -32768, "max": 32767, "signed": True, "float": False},
+    "uint32": {"itemsize": 4, "min": 0, "max": 4294967295, "signed": False, "float": False},
+    "int32": {
+        "itemsize": 4,
+        "min": -2147483648,
+        "max": 2147483647,
+        "signed": True,
+        "float": False,
+    },
+    "float32": {"itemsize": 4, "min": -3.4e38, "max": 3.4e38, "signed": True, "float": True},
+    "float64": {"itemsize": 8, "min": -1.7e308, "max": 1.7e308, "signed": True, "float": True},
+}
+
+
+@dataclass
+class ValidationResult:
+    """Result of encoding validation."""
+
+    passed: bool
+    issues: list[str] = field(default_factory=list)
+    statistics: dict[str, float] = field(default_factory=dict)
+    sample_data: NDArray[np.float64] = field(default_factory=lambda: np.array([]))
+
+
+class EncodingDetector:
+    """Multi-stage encoding detector with validation.
+
+    Detects binary encoding using statistical analysis and validation.
+    Prevents false positives by actually loading data and checking results.
+
+    Example:
+        >>> detector = EncodingDetector()
+        >>> with BinaryFile("data.bin") as bf:
+        ...     result = detector.detect(bf, validation=True)
+        ...     print(f"Detected: {result.dtype} ({result.confidence:.1%})")
+    """
+
+    def __init__(
+        self,
+        min_std_threshold: float = 0.01,
+        max_constant_ratio: float = 0.95,
+        min_confidence: float = 0.5,
+    ):
+        """Initialize encoding detector.
+
+        Args:
+            min_std_threshold: Minimum standard deviation to avoid constant data.
+            max_constant_ratio: Maximum ratio of constant values allowed.
+            min_confidence: Minimum confidence to return a detection.
+        """
+        self.min_std_threshold = min_std_threshold
+        self.max_constant_ratio = max_constant_ratio
+        self.min_confidence = min_confidence
+
+    def detect(
+        self,
+        file: BinaryFile,
+        validation: bool = True,
+        n_samples: int = 5,
+        max_samples: int = 1000,
+    ) -> EncodingResult:
+        """Detect encoding with optional validation.
+
+        Args:
+            file: Binary file to analyze.
+            validation: Whether to validate detection by loading data.
+            n_samples: Number of file locations to sample.
+            max_samples: Maximum number of data samples to load for validation.
+
+        Returns:
+            EncodingResult with detected dtype and confidence.
+
+        Example:
+            >>> detector = EncodingDetector()
+            >>> with BinaryFile("data.bin") as bf:
+            ...     result = detector.detect(bf)
+            ...     if result.validation_passed:
+            ...         print(f"Confident detection: {result.dtype}")
+        """
+        # Sample data from multiple locations
+        samples = file.sample_locations(n_samples=n_samples, sample_size=8192)
+
+        # Stage 1: Statistical detection
+        candidates = self._statistical_detection(samples)
+
+        if not candidates:
+            # Fallback to uint8 with low confidence
+            sample_data = self._load_as_dtype(file, "uint8", max_samples=max_samples)
+            return EncodingResult(
+                dtype="uint8",
+                confidence=0.3,
+                alternatives=[],
+                validation_passed=False,
+                sample_data=sample_data,
+                statistics=self._compute_statistics(sample_data),
+                issues=["No clear encoding detected, defaulting to uint8"],
+            )
+
+        # Stage 2: Consensus detection from multiple locations
+        if validation and len(samples) > 1:
+            candidates = self._consensus_detection(file, candidates, n_samples=n_samples)
+
+        # Stage 3: Validation by loading and checking data
+        best_dtype, best_confidence = candidates[0]
+        validation_result = ValidationResult(passed=False)
+
+        if validation:
+            validation_result = self._validate_encoding(file, best_dtype, max_samples=max_samples)
+
+            # If validation failed, try alternatives
+            if not validation_result.passed and len(candidates) > 1:
+                for dtype, conf in candidates[1:]:
+                    alt_validation = self._validate_encoding(file, dtype, max_samples=max_samples)
+                    if alt_validation.passed:
+                        validation_result = alt_validation
+                        best_dtype = dtype
+                        best_confidence = conf
+                        break
+
+        # Use validated sample data if available
+        sample_data = (
+            validation_result.sample_data
+            if validation_result.sample_data.size > 0
+            else self._load_as_dtype(file, best_dtype, max_samples=max_samples)
+        )
+
+        statistics = (
+            validation_result.statistics
+            if validation_result.statistics
+            else self._compute_statistics(sample_data)
+        )
+
+        return EncodingResult(
+            dtype=best_dtype,
+            confidence=best_confidence if validation_result.passed else best_confidence * 0.7,
+            alternatives=[(dt, conf) for dt, conf in candidates[1:6]],
+            validation_passed=validation_result.passed,
+            sample_data=sample_data,
+            statistics=statistics,
+            issues=validation_result.issues if not validation_result.passed else [],
+        )
+
+    def _statistical_detection(self, samples: list[bytes]) -> list[tuple[str, float]]:
+        """Detect encoding using statistical analysis.
+
+        Args:
+            samples: List of byte samples from file.
+
+        Returns:
+            List of (dtype, confidence) tuples, sorted by confidence descending.
+        """
+        scores: dict[str, float] = {}
+
+        for dtype_name in SUPPORTED_DTYPES:
+            score = self._score_dtype(samples, dtype_name)
+            if score > 0:
+                scores[dtype_name] = score
+
+        # Sort by score descending
+        candidates = sorted(scores.items(), key=lambda x: x[1], reverse=True)
+
+        return candidates
+
+    def _score_dtype(self, samples: list[bytes], dtype: str) -> float:
+        """Score how well a dtype fits the data.
+
+        Args:
+            samples: List of byte samples.
+            dtype: Data type to score.
+
+        Returns:
+            Score from 0.0 to 1.0, higher is better.
+        """
+        dtype_info = SUPPORTED_DTYPES[dtype]
+        itemsize = dtype_info["itemsize"]
+        is_float = dtype_info["float"]
+
+        total_score = 0.0
+        valid_samples = 0
+
+        for sample in samples:
+            if len(sample) < itemsize:
+                continue
+
+            # Parse sample data
+            count = len(sample) // itemsize
+            try:
+                data = np.frombuffer(sample, dtype=dtype, count=count)
+            except (ValueError, TypeError):
+                continue
+
+            if len(data) == 0:
+                continue
+
+            valid_samples += 1
+            sample_score = 0.0
+
+            # Check 1: Entropy (should be reasonable)
+            entropy = self._compute_entropy(sample)
+            if is_float:
+                # Floats should have higher entropy
+                if entropy > 4.0:
+                    sample_score += 0.3
+            else:
+                # Integers should have moderate entropy
+                if 1.0 < entropy < 7.0:
+                    sample_score += 0.2
+
+            # Check 2: Value range makes sense
+            # Suppress warnings for invalid casts (expected when testing encodings)
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", RuntimeWarning)
+                if is_float:
+                    # No conversion needed for floats
+                    data_min = float(np.min(data))
+                    data_max = float(np.max(data))
+                else:
+                    # Convert integers to float64 for range comparison
+                    data_float = data.astype(np.float64)
+                    data_min = float(np.min(data_float))
+                    data_max = float(np.max(data_float))
+
+            if is_float:
+                # For floats, check IEEE 754 validity
+                if self._is_valid_ieee754(sample, itemsize):
+                    sample_score += 0.4
+                # Check if values are in reasonable range
+                if data_min > -1e6 and data_max < 1e6:
+                    sample_score += 0.2
+            else:
+                # For integers, check range
+                type_min = dtype_info["min"]
+                type_max = dtype_info["max"]
+                range_score = self._score_integer_range(data_min, data_max, type_min, type_max)
+                sample_score += range_score * 0.5
+
+                # Bonus: Prefer types that make better use of the value range
+                # If data uses more of the type's range, that's a better fit
+                data_range = data_max - data_min
+                type_range = type_max - type_min
+                if type_range > 0:
+                    utilization = data_range / type_range
+                    # Reward types with 1%-50% utilization (good fit)
+                    if 0.01 <= utilization <= 0.5:
+                        sample_score += 0.2
+                    # Small reward for reasonable utilization
+                    elif 0.001 <= utilization < 0.01:
+                        sample_score += 0.1
+
+            # Check 3: Not all constant
+            # Suppress overflow warnings when testing invalid dtype interpretations
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", RuntimeWarning)
+                if is_float:
+                    std = float(np.std(data))
+                else:
+                    std = float(np.std(data_float))
+            if std > self.min_std_threshold:
+                sample_score += 0.2
+
+            # Check 4: Alignment check (data should align well)
+            if self._check_alignment(sample, itemsize):
+                sample_score += 0.1
+
+            total_score += sample_score
+
+        if valid_samples == 0:
+            return 0.0
+
+        # Normalize to 0.0-1.0 range
+        avg_score = total_score / valid_samples
+        return min(avg_score, 1.0)
+
+    def _compute_entropy(self, data: bytes) -> float:
+        """Compute Shannon entropy of byte data.
+
+        Args:
+            data: Byte sequence.
+
+        Returns:
+            Entropy in bits (0-8 for bytes).
+        """
+        if len(data) == 0:
+            return 0.0
+
+        # Count byte frequencies
+        counts = np.bincount(np.frombuffer(data, dtype=np.uint8), minlength=256)
+        probabilities = counts[counts > 0] / len(data)
+
+        # Shannon entropy
+        entropy = -np.sum(probabilities * np.log2(probabilities))
+        return float(entropy)
+
+    def _is_valid_ieee754(self, data: bytes, itemsize: int) -> bool:
+        """Check if data looks like valid IEEE 754 floats.
+
+        Args:
+            data: Byte data to check.
+            itemsize: 4 for float32, 8 for float64.
+
+        Returns:
+            True if data appears to be valid IEEE 754.
+        """
+        if itemsize not in (4, 8):
+            return False
+
+        count = min(len(data) // itemsize, ENCODING_VALIDATION_COUNT)
+        if count == 0:
+            return False
+
+        fmt = f"{'f' if itemsize == 4 else 'd'}"
+        valid_count = 0
+
+        for i in range(count):
+            try:
+                offset = i * itemsize
+                value = struct.unpack(fmt, data[offset : offset + itemsize])[0]
+
+                # Check for NaN, Inf
+                if np.isnan(value) or np.isinf(value):
+                    continue
+
+                # Check if value is reasonable
+                if abs(value) < 1e30:  # Not extremely large
+                    valid_count += 1
+
+            except (struct.error, ValueError):
+                continue
+
+        # Validate minimum ratio of valid floats
+        return valid_count / count > MIN_VALID_FLOAT_RATIO
+
+    def _score_integer_range(
+        self, data_min: float, data_max: float, type_min: float, type_max: float
+    ) -> float:
+        """Score how well data fits integer type range.
+
+        Args:
+            data_min: Minimum value in data.
+            data_max: Maximum value in data.
+            type_min: Minimum value for type.
+            type_max: Maximum value for type.
+
+        Returns:
+            Score from 0.0 to 1.0.
+        """
+        # Data must fit in type range
+        if data_min < type_min or data_max > type_max:
+            return 0.0
+
+        # Score based on range utilization
+        data_range = data_max - data_min
+        type_range = type_max - type_min
+
+        if type_range == 0:
+            return 0.0
+
+        utilization = data_range / type_range
+
+        # Prefer types where data uses a reasonable portion of range
+        # but not types that are way too large
+        if utilization > 0.01:  # Uses at least 1% of range
+            return 1.0
+        elif utilization > 0.001:  # Uses at least 0.1%
+            return 0.7
+        else:
+            return 0.3  # Very small utilization, might be oversized type
+
+    def _check_alignment(self, data: bytes, itemsize: int) -> bool:
+        """Check if data aligns well with item size.
+
+        Args:
+            data: Byte data.
+            itemsize: Size of data type in bytes.
+
+        Returns:
+            True if data aligns well.
+        """
+        # File size should be multiple of itemsize (or close)
+        remainder = len(data) % itemsize
+        return remainder < itemsize * 0.1  # Within 10%
+
+    def _consensus_detection(
+        self, file: BinaryFile, candidates: list[tuple[str, float]], n_samples: int
+    ) -> list[tuple[str, float]]:
+        """Refine detection using consensus from multiple locations.
+
+        Args:
+            file: Binary file being analyzed.
+            candidates: Initial candidate list.
+            n_samples: Number of samples to check.
+
+        Returns:
+            Refined candidate list with adjusted confidences.
+        """
+        # Sample additional locations
+        samples = file.sample_locations(n_samples=n_samples, sample_size=4096)
+
+        # Score each candidate on all samples
+        consensus_scores: dict[str, list[float]] = {dt: [] for dt, _ in candidates[:5]}
+
+        for sample in samples:
+            for dtype, scores_list in consensus_scores.items():
+                score = self._score_dtype([sample], dtype)
+                scores_list.append(score)
+
+        # Compute average and consistency
+        refined_candidates: list[tuple[str, float]] = []
+        for dtype, scores in consensus_scores.items():
+            if not scores:
+                continue
+            # Suppress overflow warnings when computing statistics
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", RuntimeWarning)
+                avg_score = np.mean(scores)
+                consistency = 1.0 - np.std(scores)  # Lower std = more consistent
+
+            # Combine average score with consistency
+            final_score = avg_score * 0.7 + consistency * 0.3
+            refined_candidates.append((dtype, float(final_score)))
+
+        # Sort by final score
+        refined_candidates.sort(key=lambda x: x[1], reverse=True)
+
+        return refined_candidates
+
+    def _validate_encoding(
+        self, file: BinaryFile, dtype: str, max_samples: int = 1000
+    ) -> ValidationResult:
+        """Validate encoding by loading and checking data.
+
+        Args:
+            file: Binary file to validate.
+            dtype: Data type to validate.
+            max_samples: Maximum number of data samples to load.
+
+        Returns:
+            ValidationResult with validation status and issues.
+        """
+        issues: list[str] = []
+
+        # Load sample data
+        try:
+            sample_data = self._load_as_dtype(file, dtype, max_samples=max_samples)
+        except (ValueError, TypeError) as e:
+            return ValidationResult(passed=False, issues=[f"Failed to load as {dtype}: {e!s}"])
+
+        if len(sample_data) == 0:
+            return ValidationResult(passed=False, issues=["No data could be loaded"])
+
+        # Convert to float for analysis
+        # Suppress overflow warnings when testing invalid dtype interpretations
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            data_float = sample_data.astype(np.float64)
+
+        # Check 1: Not all constant
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            std = float(np.std(data_float))
+        if std < self.min_std_threshold:
+            issues.append(f"Data appears constant (std={std:.2e})")
+
+        # Check 2: No NaN/Inf for float types
+        dtype_info = SUPPORTED_DTYPES[dtype]
+        if dtype_info["float"]:
+            nan_count = int(np.sum(np.isnan(data_float)))
+            inf_count = int(np.sum(np.isinf(data_float)))
+
+            if nan_count > len(data_float) * 0.1:  # More than 10% NaN
+                issues.append(f"Too many NaN values: {nan_count}/{len(data_float)}")
+
+            if inf_count > len(data_float) * 0.1:  # More than 10% Inf
+                issues.append(f"Too many Inf values: {inf_count}/{len(data_float)}")
+
+        # Check 3: Values in reasonable range
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            data_min = float(np.min(data_float))
+            data_max = float(np.max(data_float))
+        type_min = dtype_info["min"]
+        type_max = dtype_info["max"]
+
+        if dtype_info["float"]:
+            # For floats, check reasonable range
+            if abs(data_min) > 1e20 or abs(data_max) > 1e20:
+                issues.append(f"Values extremely large: [{data_min:.2e}, {data_max:.2e}]")
+        else:
+            # For integers, must fit in type
+            if data_min < type_min or data_max > type_max:
+                issues.append(
+                    f"Values outside type range: [{data_min}, {data_max}] "
+                    f"not in [{type_min}, {type_max}]"
+                )
+
+        # Check 4: Reasonable entropy
+        sample_bytes = file.read_bytes(0, min(8192, file.size))
+        entropy = self._compute_entropy(sample_bytes)
+
+        if entropy < 0.5:
+            issues.append(f"Very low entropy: {entropy:.2f} bits")
+
+        # Check 5: Not too many constant values
+        unique_count = len(np.unique(sample_data))
+        constant_ratio = 1.0 - (unique_count / len(sample_data))
+
+        if constant_ratio > self.max_constant_ratio:
+            issues.append(f"Too many repeated values: {constant_ratio:.1%} repetition")
+
+        # Compute statistics
+        statistics = self._compute_statistics(data_float)
+
+        # Validation passes if no critical issues
+        passed = len(issues) == 0
+
+        return ValidationResult(
+            passed=passed,
+            issues=issues,
+            statistics=statistics,
+            sample_data=data_float,
+        )
+
+    def _load_as_dtype(
+        self, file: BinaryFile, dtype: str, max_samples: int = 1000
+    ) -> NDArray[np.float64]:
+        """Load sample data as specified dtype.
+
+        Args:
+            file: Binary file to load from.
+            dtype: Data type to use.
+            max_samples: Maximum number of samples to load.
+
+        Returns:
+            Sample data as float64 array.
+        """
+        dtype_info = SUPPORTED_DTYPES[dtype]
+        itemsize = dtype_info["itemsize"]
+
+        # Calculate number of samples to load
+        max_elements = file.size // itemsize
+        count = min(max_samples, max_elements)
+
+        if count <= 0:
+            return np.array([], dtype=np.float64)
+
+        # Load data
+        data = file.read_array(offset=0, count=count, dtype=dtype)
+
+        # Convert to float64
+        # Suppress overflow warnings when converting invalid dtype interpretations
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            return data.astype(np.float64)
+
+    def _compute_statistics(self, data: NDArray[np.float64]) -> dict[str, float]:
+        """Compute statistics for sample data.
+
+        Args:
+            data: Sample data array.
+
+        Returns:
+            Dictionary of statistics.
+        """
+        if len(data) == 0:
+            return {}
+
+        # Suppress overflow warnings when computing statistics on invalid dtype interpretations
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", RuntimeWarning)
+            return {
+                "mean": float(np.mean(data)),
+                "std": float(np.std(data)),
+                "min": float(np.min(data)),
+                "max": float(np.max(data)),
+                "median": float(np.median(data)),
+                "unique_count": int(len(np.unique(data))),
+                "total_count": int(len(data)),
+            }