oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

oscura/inference/message_format.py

@@ -172,6 +172,84 @@ class MessageFormatInferrer:
 
         return schema
 
+    def _calculate_entropy_boundaries(
+        self, messages: list[NDArray[np.uint8]], msg_len: int
+    ) -> list[int]:
+        """Detect boundaries using entropy transitions.
+
+        Args:
+            messages: List of message arrays
+            msg_len: Length of messages
+
+        Returns:
+            List of boundary positions
+        """
+        boundaries = []
+        entropies = []
+
+        # Calculate entropy at each byte position
+        for offset in range(msg_len):
+            entropy = self._calculate_byte_entropy(messages, offset)
+            entropies.append(entropy)
+
+        # Find transitions (entropy changes > threshold)
+        entropy_threshold = 1.5  # bits
+        for i in range(1, len(entropies)):
+            delta = abs(entropies[i] - entropies[i - 1])
+            if delta > entropy_threshold:
+                boundaries.append(i)
+
+        return boundaries
+
+    def _calculate_variance_boundaries(
+        self, messages: list[NDArray[np.uint8]], msg_len: int
+    ) -> list[int]:
+        """Detect boundaries using variance transitions.
+
+        Args:
+            messages: List of message arrays
+            msg_len: Length of messages
+
+        Returns:
+            List of boundary positions
+        """
+        boundaries = []
+        variances = []
+
+        # Calculate variance at each byte position
+        for offset in range(msg_len):
+            values = [msg[offset] for msg in messages]
+            variance = np.var(values)
+            variances.append(variance)
+
+        # Find variance transitions
+        var_threshold = 1000.0
+        for i in range(1, len(variances)):
+            delta = abs(variances[i] - variances[i - 1])
+            if delta > var_threshold:
+                boundaries.append(i)
+
+        return boundaries
+
+    def _merge_close_boundaries(self, boundaries: list[int]) -> list[int]:
+        """Merge boundaries that are too close together.
+
+        Args:
+            boundaries: Sorted list of boundaries
+
+        Returns:
+            Merged boundary list
+        """
+        if not boundaries:
+            return []
+
+        merged = [boundaries[0]]
+        for b in boundaries[1:]:
+            if b - merged[-1] >= 2:
+                merged.append(b)
+
+        return merged
+
     def detect_field_boundaries(
         self,
         messages: list[NDArray[np.uint8]],
@@ -194,45 +272,18 @@ class MessageFormatInferrer:
         msg_len = len(messages[0])
         boundaries = [0]  # Always start at offset 0
 
+        # Detect boundaries using selected method
         if method in ["entropy", "combined"]:
-            # Calculate entropy at each byte position
-            entropies = []
-            for offset in range(msg_len):
-                entropy = self._calculate_byte_entropy(messages, offset)
-                entropies.append(entropy)
-
-            # Find transitions (entropy changes > threshold)
-            entropy_threshold = 1.5  # bits
-            for i in range(1, len(entropies)):
-                delta = abs(entropies[i] - entropies[i - 1])
-                if delta > entropy_threshold and i not in boundaries:
-                    boundaries.append(i)
+            entropy_boundaries = self._calculate_entropy_boundaries(messages, msg_len)
+            boundaries.extend(entropy_boundaries)
 
         if method in ["variance", "combined"]:
-            # Calculate variance at each byte position
-            variances = []
-            for offset in range(msg_len):
-                values = [msg[offset] for msg in messages]
-                variance = np.var(values)
-                variances.append(variance)
-
-            # Find variance transitions
-            var_threshold = 1000.0
-            for i in range(1, len(variances)):
-                delta = abs(variances[i] - variances[i - 1])
-                if delta > var_threshold and i not in boundaries:
-                    boundaries.append(i)
-
-        # Sort and ensure we don't have too many tiny fields
-        boundaries = sorted(set(boundaries))
+            variance_boundaries = self._calculate_variance_boundaries(messages, msg_len)
+            boundaries.extend(variance_boundaries)
 
-        # Merge boundaries that are too close (< 2 bytes apart)
-        merged = [boundaries[0]]
-        for b in boundaries[1:]:
-            if b - merged[-1] >= 2:
-                merged.append(b)
-
-        return merged
+        # Merge and sort boundaries
+        boundaries = sorted(set(boundaries))
+        return self._merge_close_boundaries(boundaries)
 
     def detect_boundaries_voting(
         self,
@@ -532,7 +583,30 @@ class MessageFormatInferrer:
         if len(messages) < self.min_samples:
             raise ValueError(f"Need at least {self.min_samples} messages, got {len(messages)}")
 
-        # Convert all to bytes for voting
+        bytes_messages = self._convert_messages_to_bytes(messages)
+        msg_len = self._validate_message_lengths(bytes_messages)
+        boundaries = self.detect_boundaries_voting(
+            bytes_messages, min_confidence=min_boundary_confidence
+        )
+        msg_arrays = [np.frombuffer(msg, dtype=np.uint8) for msg in bytes_messages]
+        fields = self._infer_fields_from_boundaries(
+            boundaries, msg_arrays, msg_len, min_field_confidence
+        )
+        checksum_field, length_field = self._find_special_fields(fields)
+        header_size = self._estimate_header_size(fields)
+
+        return MessageSchema(
+            total_size=msg_len,
+            fields=fields,
+            field_boundaries=boundaries,
+            header_size=header_size,
+            payload_offset=header_size,
+            checksum_field=checksum_field,
+            length_field=length_field,
+        )
+
+    def _convert_messages_to_bytes(self, messages: list[bytes | NDArray[np.uint8]]) -> list[bytes]:
+        """Convert messages to bytes format."""
         bytes_messages = []
         for msg in messages:
             if isinstance(msg, bytes):
@@ -541,99 +615,81 @@ class MessageFormatInferrer:
                 bytes_messages.append(msg.tobytes())
             else:
                 raise ValueError(f"Invalid message type: {type(msg)}")
+        return bytes_messages
 
-        # Check all messages are same length
-        lengths = [len(m) for m in bytes_messages]
+    def _validate_message_lengths(self, messages: list[bytes]) -> int:
+        """Validate all messages have same length and return that length."""
+        lengths = [len(m) for m in messages]
         if len(set(lengths)) > 1:
             raise ValueError(f"Messages have varying lengths: {set(lengths)}")
+        return lengths[0]
 
-        msg_len = lengths[0]
-
-        # Detect boundaries with voting
-        boundaries = self.detect_boundaries_voting(
-            bytes_messages, min_confidence=min_boundary_confidence
-        )
-
-        # Convert to numpy arrays for field type detection
-        msg_arrays = []
-        for msg in bytes_messages:
-            msg_arrays.append(np.frombuffer(msg, dtype=np.uint8))
-
-        # Extract field candidates
+    def _infer_fields_from_boundaries(
+        self,
+        boundaries: list[int],
+        msg_arrays: list[NDArray[np.uint8]],
+        msg_len: int,
+        min_confidence: float,
+    ) -> list[InferredField]:
+        """Infer fields from detected boundaries."""
         fields: list[InferredField] = []
         for i in range(len(boundaries)):
             offset = boundaries[i]
-
-            # Determine field size
-            if i < len(boundaries) - 1:
-                size = boundaries[i + 1] - offset
-            else:
-                size = msg_len - offset
-
-            # Extract field values
+            size = boundaries[i + 1] - offset if i < len(boundaries) - 1 else msg_len - offset
             field_data = self._extract_field_data(msg_arrays, offset, size)
-
-            # Run multiple field type detectors
-            entropy_type, entropy_conf = self._detect_type_entropy(field_data)
-            pattern_type, pattern_conf = self._detect_type_patterns(
-                field_data, offset, size, msg_len
-            )
-            stats_type, stats_conf = self._detect_type_statistics(field_data)
-
-            # Vote on field type
-            field_type, confidence, evidence = self._vote_field_type(
-                [
-                    (entropy_type, entropy_conf),
-                    (pattern_type, pattern_conf),
-                    (stats_type, stats_conf),
-                ]
+            field_obj = self._classify_field_ensemble(
+                field_data, offset, size, msg_len, min_confidence
             )
-
-            if confidence >= min_field_confidence:
-                # Sample values (first 5)
-                sample_values = field_data["values"][:5]
-
-                field_obj = InferredField(
-                    name=f"field_{len(fields)}",
-                    offset=offset,
-                    size=size,
-                    field_type=field_type,  # type: ignore[arg-type]
-                    entropy=float(field_data["entropy"]),
-                    variance=float(field_data["variance"]),
-                    confidence=confidence,
-                    values_seen=sample_values,
-                    evidence=evidence,
-                )
-
+            if field_obj is not None:
+                field_obj.name = f"field_{len(fields)}"
                 fields.append(field_obj)
+        return fields
 
-        # Determine header size
-        header_size = self._estimate_header_size(fields)
+    def _classify_field_ensemble(
+        self,
+        field_data: dict[str, Any],
+        offset: int,
+        size: int,
+        msg_len: int,
+        min_confidence: float,
+    ) -> InferredField | None:
+        """Classify field using ensemble of detectors."""
+        entropy_type, entropy_conf = self._detect_type_entropy(field_data)
+        pattern_type, pattern_conf = self._detect_type_patterns(field_data, offset, size, msg_len)
+        stats_type, stats_conf = self._detect_type_statistics(field_data)
+        field_type, confidence, evidence = self._vote_field_type(
+            [
+                (entropy_type, entropy_conf),
+                (pattern_type, pattern_conf),
+                (stats_type, stats_conf),
+            ]
+        )
+        if confidence < min_confidence:
+            return None
+        return InferredField(
+            name="",  # Will be set by caller
+            offset=offset,
+            size=size,
+            field_type=field_type,  # type: ignore[arg-type]
+            entropy=float(field_data["entropy"]),
+            variance=float(field_data["variance"]),
+            confidence=confidence,
+            values_seen=field_data["values"][:5],
+            evidence=evidence,
+        )
 
-        # Find checksum and length fields
+    def _find_special_fields(
+        self, fields: list[InferredField]
+    ) -> tuple[InferredField | None, InferredField | None]:
+        """Find checksum and length fields."""
         checksum_field = None
         length_field = None
-
         for f in fields:
             if f.field_type == "checksum":
                 checksum_field = f
             elif f.field_type == "length":
                 length_field = f
-
-        # Payload starts after header
-        payload_offset = header_size
-
-        schema = MessageSchema(
-            total_size=msg_len,
-            fields=fields,
-            field_boundaries=boundaries,
-            header_size=header_size,
-            payload_offset=payload_offset,
-            checksum_field=checksum_field,
-            length_field=length_field,
-        )
-
-        return schema
+        return checksum_field, length_field
 
     def _extract_field_data(
         self, messages: list[NDArray[np.uint8]], offset: int, size: int
@@ -669,14 +725,14 @@ class MessageFormatInferrer:
                         int(msg[offset]) << 16 | int(msg[offset + 1]) << 8 | int(msg[offset + 2])
                     )
                 int_values.append(val_int)
-            values = list(int_values)  # type: ignore[assignment]
+            values = list(int_values)
         else:
             # For larger fields, use bytes
             tuple_values: list[tuple[int, ...]] = []
             for msg in messages:
                 val_tuple = tuple(int(b) for b in msg[offset : offset + size])
                 tuple_values.append(val_tuple)
-            values = list(tuple_values)  # type: ignore[assignment]
+            values = list(tuple_values)
 
         # Calculate statistics
         if size > 4:
@@ -733,57 +789,115 @@ class MessageFormatInferrer:
     ) -> tuple[str, float]:
         """Detect field type using pattern matching.
 
-        : Pattern-based field type detection.
-
         Detects:
         - Counters (incrementing values)
+        - Timestamps (steady large increments)
         - Lengths (correlates with message size)
-        - Checksums (high entropy, end of message)
+        - Checksums (end of message)
         - Constants (no variation)
-        - Timestamps (steady increase)
 
         Args:
-            field_data: Field data dictionary
-            offset: Field offset
-            size: Field size
-            msg_len: Total message length
+            field_data: Field data dictionary with 'values' key.
+            offset: Field offset in bytes.
+            size: Field size in bytes.
+            msg_len: Total message length in bytes.
 
         Returns:
-            Tuple of (field_type, confidence)
+            Tuple of (field_type, confidence_score).
         """
         values = field_data["values"]
 
-        # Check for counter (if integer values)
-        if not isinstance(values[0], tuple):
-            int_values = [v for v in values if isinstance(v, int)]
-            if self._detect_counter_field(int_values):
-                return ("counter", 0.9)
-
-            # Check for timestamp (similar to counter but larger values)
-            if len(int_values) >= 3:
-                diffs = [int_values[i + 1] - int_values[i] for i in range(len(int_values) - 1)]
-                positive_diffs = [d for d in diffs if d > 0]
-                if len(positive_diffs) >= len(diffs) * 0.7:
-                    # Check if increments are relatively steady
-                    if len(positive_diffs) > 0:
-                        avg_diff = sum(positive_diffs) / len(positive_diffs)
-                        if avg_diff > 100:  # Large increments suggest timestamp
-                            return ("timestamp", 0.7)
-
-        # Check for length field (small values, near start)
-        if offset < 8 and size <= 2 and not isinstance(values[0], tuple):
-            int_values = [v for v in values if isinstance(v, int)]
-            if int_values:
-                max_val = max(int_values)
-                if max_val < msg_len * 2:
-                    return ("length", 0.6)
-
-        # Check for checksum (near end of message, but not the entire message)
-        if offset + size >= msg_len - 4 and offset > 0:
+        # Skip tuple values (byte arrays)
+        if isinstance(values[0], tuple):
+            return ("unknown", 0.3)
+
+        # Extract integer values
+        int_values = [v for v in values if isinstance(v, int)]
+        if not int_values:
+            return ("unknown", 0.3)
+
+        # Check for counter pattern
+        if self._detect_counter_field(int_values):
+            return ("counter", 0.9)
+
+        # Check for timestamp pattern
+        timestamp_result = self._check_timestamp_pattern(int_values)
+        if timestamp_result is not None:
+            return timestamp_result
+
+        # Check for length field
+        if self._is_likely_length_field(offset, size, msg_len, int_values):
+            return ("length", 0.6)
+
+        # Check for checksum field
+        if self._is_likely_checksum_field(offset, size, msg_len):
             return ("checksum", 0.5)
 
         return ("unknown", 0.3)
 
+    def _check_timestamp_pattern(self, values: list[int]) -> tuple[str, float] | None:
+        """Check if values follow timestamp pattern.
+
+        Args:
+            values: Integer values.
+
+        Returns:
+            ('timestamp', confidence) or None if not a timestamp.
+        """
+        if len(values) < 3:
+            return None
+
+        # Calculate differences between consecutive values
+        diffs = [values[i + 1] - values[i] for i in range(len(values) - 1)]
+        positive_diffs = [d for d in diffs if d > 0]
+
+        # Need mostly increasing values
+        if len(positive_diffs) < len(diffs) * 0.7:
+            return None
+
+        # Check for large increments (typical of timestamps)
+        avg_diff = sum(positive_diffs) / len(positive_diffs)
+        if avg_diff > 100:
+            return ("timestamp", 0.7)
+
+        return None
+
+    def _is_likely_length_field(
+        self, offset: int, size: int, msg_len: int, values: list[int]
+    ) -> bool:
+        """Check if field is likely a length field.
+
+        Args:
+            offset: Field offset.
+            size: Field size.
+            msg_len: Message length.
+            values: Integer values.
+
+        Returns:
+            True if likely a length field.
+        """
+        # Length fields are typically near message start and small
+        if offset >= 8 or size > 2:
+            return False
+
+        # Values should be reasonable for message lengths
+        max_val = max(values)
+        return max_val < msg_len * 2
+
+    def _is_likely_checksum_field(self, offset: int, size: int, msg_len: int) -> bool:
+        """Check if field is likely a checksum.
+
+        Args:
+            offset: Field offset.
+            size: Field size.
+            msg_len: Message length.
+
+        Returns:
+            True if likely a checksum field.
+        """
+        # Checksums are typically near the end but not the whole message
+        return offset + size >= msg_len - 4 and offset > 0
+
     def _detect_type_statistics(self, field_data: dict[str, Any]) -> tuple[str, float]:
         """Detect field type using statistical properties.
 
@@ -909,14 +1023,14 @@ class MessageFormatInferrer:
                             | int(msg[offset + 2])
                         )
                     int_values.append(val_int)
-                values = list(int_values)  # type: ignore[assignment]
+                values = list(int_values)
             else:
                 # For larger fields, use bytes
                 tuple_values: list[tuple[int, ...]] = []
                 for msg in messages:
                     val_tuple = tuple(int(b) for b in msg[offset : offset + size])
                     tuple_values.append(val_tuple)
-                values = list(tuple_values)  # type: ignore[assignment]
+                values = list(tuple_values)
 
             # Calculate statistics
             if size > 4:
@@ -1025,55 +1139,100 @@ class MessageFormatInferrer:
     ) -> tuple[str, float]:
         """Classify field type based on patterns.
 
-        : Field type classification logic.
-
         Args:
-            values: Field values across all messages
-            offset: Field offset
-            size: Field size
-            messages: Original messages
+            values: Field values across all messages.
+            offset: Field offset in bytes.
+            size: Field size in bytes.
+            messages: Original message byte arrays.
 
         Returns:
-            Tuple of (field_type, confidence)
+            Tuple of (field_type, confidence_score).
+
+        Example:
+            >>> field_type, confidence = analyzer._classify_field(values, 0, 1, messages)
         """
-        # Handle byte fields (larger than 4 bytes)
+        # Handle byte array fields (tuples)
         if isinstance(values[0], tuple):
-            # Check if all tuples are identical (truly constant)
-            if len(set(values)) == 1:
-                return ("constant", 1.0)
-
-            entropy = self._calculate_entropy(np.concatenate([np.array(v) for v in values]))
-            if entropy < 1.0:
-                return ("constant", 0.9)
-            elif entropy > 7.0:
-                return ("data", 0.6)
-            else:
-                return ("data", 0.5)
+            return self._classify_byte_array_field(values)
+
+        # Handle scalar fields
+        return self._classify_scalar_field(values, offset, size, messages)
+
+    def _classify_byte_array_field(self, values: list[int | tuple[int, ...]]) -> tuple[str, float]:
+        """Classify byte array field (multi-byte fields).
+
+        Args:
+            values: List of byte tuples.
+
+        Returns:
+            Tuple of (field_type, confidence).
+        """
+        # Check for constant byte arrays
+        if len(set(values)) == 1:
+            return ("constant", 1.0)
+
+        # Calculate entropy across all bytes
+        all_bytes = np.concatenate([np.array(v) for v in values])
+        entropy = self._calculate_entropy(all_bytes)
+
+        if entropy < 1.0:
+            return ("constant", 0.9)
+        elif entropy > 7.0:
+            return ("data", 0.6)
+        else:
+            return ("data", 0.5)
+
+    def _classify_scalar_field(
+        self,
+        values: list[int | tuple[int, ...]],
+        offset: int,
+        size: int,
+        messages: list[NDArray[np.uint8]],
+    ) -> tuple[str, float]:
+        """Classify scalar (integer) field.
 
-        # Check for constant field
+        Args:
+            values: List of integer values.
+            offset: Field offset.
+            size: Field size.
+            messages: Original messages.
+
+        Returns:
+            Tuple of (field_type, confidence).
+        """
+        # Check for constant values
         if len(set(values)) == 1:
             return ("constant", 1.0)
 
-        # Check for counter field
-        if not isinstance(values[0], tuple) and self._detect_counter_field(  # type: ignore[misc, unreachable]
-            [v for v in values if isinstance(v, int)]
-        ):
+        # Check for counter pattern
+        int_values = [v for v in values if isinstance(v, int)]
+        if self._detect_counter_field(int_values):
             return ("counter", 0.9)
 
-        # Check for checksum (if near end of message)
+        # Check for checksum (near end of message)
         msg_len = len(messages[0])
-        if offset + size >= msg_len - 4:  # Within last 4 bytes
+        if offset + size >= msg_len - 4:
             if self._detect_checksum_field(messages, offset, size):
                 return ("checksum", 0.8)
 
-        # Check for length field (small values, near start)
+        # Check for length field (early, small values)
         if offset < 8 and size <= 2:
-            if not isinstance(values[0], tuple):  # type: ignore[unreachable]
-                max_val = max(v for v in values if isinstance(v, int))
-                if max_val < msg_len * 2:  # Reasonable length value
-                    return ("length", 0.6)
+            max_val = max(int_values)
+            if max_val < msg_len * 2:
+                return ("length", 0.6)
 
-        # Check variance for classification
+        # Classify by variance and entropy
+        return self._classify_by_statistics(values)
+
+    def _classify_by_statistics(self, values: list[int | tuple[int, ...]]) -> tuple[str, float]:
+        """Classify field by statistical properties.
+
+        Args:
+            values: Field values.
+
+        Returns:
+            Tuple of (field_type, confidence).
+        """
         variance = np.var(values)
         entropy = self._calculate_entropy(np.array(values))
 
@@ -1304,4 +1463,4 @@ def find_dependencies(
         # Cast to expected type (msg_arrays contains only NDArray after conversion)
         schema = inferrer.infer_format(msg_arrays)  # type: ignore[arg-type]
 
-    return inferrer.find_dependencies(msg_arrays, schema)  # type: ignore[arg-type]
+    return inferrer.find_dependencies(msg_arrays, schema)