oscura 0.0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

oscura/inference/__init__.py

@@ -0,0 +1,251 @@
+"""Auto-inference and smart defaults for Oscura.
+
+This module provides automatic parameter detection and intelligent defaults
+for common analysis tasks.
+
+- RE-SEQ-002: Sequence Pattern Detection
+- RE-SEQ-003: Request-Response Correlation
+- RE-STR-001: UDP Stream Reconstruction
+- RE-STR-002: TCP Stream Reassembly
+- RE-STR-003: Message Framing and Segmentation
+- RE-BIN-001: Magic Byte Detection
+- RE-BIN-002: Structure Alignment Detection
+- RE-BIN-003: Binary Parser DSL
+- RE-DSL-003: Protocol Format Library
+"""
+
+# Active Learning: L* algorithm for DFA inference
+from oscura.inference.active_learning import (
+    LStarLearner,
+    ObservationTable,
+    Oracle,
+    SimulatorTeacher,
+)
+from oscura.inference.adaptive_tuning import (
+    AdaptiveParameterTuner,
+    TunedParameters,
+    get_adaptive_parameters,
+)
+from oscura.inference.alignment import (
+    AlignmentResult,
+    align_global,
+    align_local,
+    align_multiple,
+    compute_similarity,
+    find_conserved_regions,
+    find_variable_regions,
+)
+from oscura.inference.bayesian import (
+    BayesianInference,
+    Posterior,
+    Prior,
+    SequentialBayesian,
+    infer_with_uncertainty,
+)
+
+# RE-BIN-001, RE-BIN-002, RE-BIN-003: Binary Format Inference
+from oscura.inference.binary import (
+    KNOWN_MAGIC_BYTES,
+    AlignmentDetector,
+    BinaryParserGenerator,
+    MagicByteDetector,
+    MagicByteResult,
+    ParserDefinition,
+    ParserField,
+    detect_alignment,
+    detect_magic_bytes,
+    find_all_magic_bytes,
+    generate_parser,
+    parser_to_python,
+    parser_to_yaml,
+)
+from oscura.inference.binary import (
+    AlignmentResult as BinaryAlignmentResult,
+)
+from oscura.inference.crc_reverse import (
+    STANDARD_CRCS,
+    CRCParameters,
+    CRCReverser,
+    verify_crc,
+)
+from oscura.inference.logic import detect_logic_family
+from oscura.inference.message_format import (
+    InferredField,
+    MessageFormatInferrer,
+    MessageSchema,
+    detect_field_types,
+    find_dependencies,
+    infer_format,
+)
+from oscura.inference.protocol import detect_protocol
+from oscura.inference.protocol_dsl import (
+    DecodedMessage,
+    FieldDefinition,
+    ProtocolDecoder,
+    ProtocolDefinition,
+    ProtocolEncoder,
+    decode_message,
+    load_protocol,
+)
+
+# RE-DSL-003: Protocol Format Library
+from oscura.inference.protocol_library import (
+    ProtocolInfo,
+    ProtocolLibrary,
+    get_decoder,
+    get_library,
+    get_protocol,
+    list_protocols,
+)
+
+# RE-SEQ-002, RE-SEQ-003: Sequence Pattern Detection and Correlation
+from oscura.inference.sequences import (
+    CommunicationFlow,
+    RequestResponseCorrelator,
+    RequestResponsePair,
+    SequencePattern,
+    SequencePatternDetector,
+    calculate_latency_stats,
+    correlate_requests,
+    detect_sequence_patterns,
+    find_message_dependencies,
+)
+from oscura.inference.signal_intelligence import (
+    AnalysisRecommendation,
+    assess_signal_quality,
+    check_measurement_suitability,
+    classify_signal,
+    get_optimal_domain_order,
+    recommend_analyses,
+    suggest_measurements,
+)
+from oscura.inference.spectral import auto_spectral_config
+from oscura.inference.state_machine import (
+    FiniteAutomaton,
+    State,
+    StateMachineInferrer,
+    Transition,
+    infer_rpni,
+    minimize_dfa,
+    to_dot,
+    to_networkx,
+)
+
+# RE-STR-001, RE-STR-002, RE-STR-003: Stream Reassembly
+from oscura.inference.stream import (
+    FramingResult,
+    MessageFrame,
+    MessageFramer,
+    ReassembledStream,
+    StreamSegment,
+    TCPStreamReassembler,
+    UDPStreamReassembler,
+    detect_message_framing,
+    extract_messages,
+    reassemble_tcp_stream,
+    reassemble_udp_stream,
+)
+
+__all__ = [
+    "KNOWN_MAGIC_BYTES",
+    "STANDARD_CRCS",
+    "AdaptiveParameterTuner",
+    # RE-BIN-001, RE-BIN-002, RE-BIN-003: Binary Format Inference
+    "AlignmentDetector",
+    "AlignmentResult",
+    "AnalysisRecommendation",
+    "BayesianInference",
+    "BinaryAlignmentResult",
+    "BinaryParserGenerator",
+    # CRC Reverse Engineering
+    "CRCParameters",
+    "CRCReverser",
+    # RE-SEQ-002, RE-SEQ-003: Sequence Patterns and Correlation
+    "CommunicationFlow",
+    "DecodedMessage",
+    "FieldDefinition",
+    "FiniteAutomaton",
+    # RE-STR-001, RE-STR-002, RE-STR-003: Stream Reassembly
+    "FramingResult",
+    "InferredField",
+    # Active Learning
+    "LStarLearner",
+    "MagicByteDetector",
+    "MagicByteResult",
+    "MessageFormatInferrer",
+    "MessageFrame",
+    "MessageFramer",
+    "MessageSchema",
+    "ObservationTable",
+    "Oracle",
+    "ParserDefinition",
+    "ParserField",
+    "Posterior",
+    "Prior",
+    "ProtocolDecoder",
+    "ProtocolDefinition",
+    "ProtocolEncoder",
+    # RE-DSL-003: Protocol Format Library
+    "ProtocolInfo",
+    "ProtocolLibrary",
+    "ReassembledStream",
+    "RequestResponseCorrelator",
+    "RequestResponsePair",
+    "SequencePattern",
+    "SequencePatternDetector",
+    "SequentialBayesian",
+    "SimulatorTeacher",
+    "State",
+    "StateMachineInferrer",
+    "StreamSegment",
+    "TCPStreamReassembler",
+    "Transition",
+    "TunedParameters",
+    "UDPStreamReassembler",
+    "align_global",
+    "align_local",
+    "align_multiple",
+    # Original exports
+    "assess_signal_quality",
+    "auto_spectral_config",
+    "calculate_latency_stats",
+    "check_measurement_suitability",
+    "classify_signal",
+    "compute_similarity",
+    "correlate_requests",
+    "decode_message",
+    "detect_alignment",
+    "detect_field_types",
+    "detect_logic_family",
+    "detect_magic_bytes",
+    "detect_message_framing",
+    "detect_protocol",
+    "detect_sequence_patterns",
+    "extract_messages",
+    "find_all_magic_bytes",
+    "find_conserved_regions",
+    "find_dependencies",
+    "find_message_dependencies",
+    "find_variable_regions",
+    "generate_parser",
+    "get_adaptive_parameters",
+    "get_decoder",
+    "get_library",
+    "get_optimal_domain_order",
+    "get_protocol",
+    "infer_format",
+    "infer_rpni",
+    "infer_with_uncertainty",
+    "list_protocols",
+    "load_protocol",
+    "minimize_dfa",
+    "parser_to_python",
+    "parser_to_yaml",
+    "reassemble_tcp_stream",
+    "reassemble_udp_stream",
+    "recommend_analyses",
+    "suggest_measurements",
+    "to_dot",
+    "to_networkx",
+    "verify_crc",
+]

oscura/inference/active_learning/README.md

@@ -0,0 +1,153 @@
+# Active Learning for DFA Inference
+
+This module implements **Angluin's L\* algorithm** for learning deterministic finite automata (DFAs) through active learning.
+
+## Overview
+
+L*is an active learning algorithm that learns the minimal DFA for a regular language by querying an oracle. Unlike passive learning (RPNI), which learns from a fixed dataset, L* can ask questions to refine its hypothesis.
+
+### Key Features
+
+- **Minimal DFA**: Guaranteed to produce the minimal DFA (fewest states)
+- **Polynomial complexity**: O(|Q|²|Σ|) membership queries, O(|Q|) equivalence queries
+- **No negative examples required**: Only needs access to oracle
+- **Interactive learning**: Can learn from live systems, simulators, or models
+
+## Algorithm
+
+The L\* algorithm maintains an **observation table** with:
+
+- **S**: Set of prefixes (representative strings for states)
+- **E**: Set of suffixes (experiments to distinguish states)
+- **T**: Table mapping (prefix + suffix) to acceptance
+
+The algorithm iterates:
+
+1. Fill observation table using membership queries
+2. Make table **closed** (every extension has equivalent in S)
+3. Make table **consistent** (equivalent rows have equivalent extensions)
+4. Construct hypothesis DFA from table
+5. Ask equivalence query
+6. If counterexample: refine table and repeat
+7. Return final hypothesis
+
+## Usage
+
+### Basic Example
+
+```python
+from oscura.inference.active_learning import LStarLearner, SimulatorTeacher
+
+# Create oracle from captured traces
+traces = [
+    ["CONNECT", "ACK"],
+    ["CONNECT", "ACK", "DATA", "ACK"],
+    ["CONNECT", "ACK", "DISCONNECT"]
+]
+
+teacher = SimulatorTeacher(traces)
+
+# Learn DFA
+learner = LStarLearner(teacher, verbose=True)
+dfa = learner.learn()
+
+print(f"Learned DFA with {len(dfa.states)} states")
+print(f"Membership queries: {learner.membership_queries}")
+print(f"Equivalence queries: {learner.equivalence_queries}")
+```
+
+### Custom Oracle
+
+```python
+from oscura.inference.active_learning import Oracle, LStarLearner
+from oscura.inference.state_machine import FiniteAutomaton
+
+class MyOracle(Oracle):
+    def membership_query(self, word: tuple[str, ...]) -> bool:
+        # Implement your logic here
+        return len(word) % 2 == 0
+
+    def equivalence_query(self, hypothesis: FiniteAutomaton) -> tuple[str, ...] | None:
+        # Test hypothesis and return counterexample if wrong
+        test_words = [(), ("a",), ("a", "a"), ("a", "a", "a")]
+        for word in test_words:
+            if hypothesis.accepts(list(word)) != self.membership_query(word):
+                return word
+        return None
+
+    def get_alphabet(self) -> set[str]:
+        return {"a", "b"}
+
+oracle = MyOracle()
+learner = LStarLearner(oracle)
+dfa = learner.learn()
+```
+
+## Oracles (Teachers)
+
+### SimulatorTeacher
+
+Learn from captured protocol traces. Treats all prefixes of traces as valid.
+
+```python
+from oscura.inference.active_learning import SimulatorTeacher
+
+traces = [["a", "b", "c"], ["a", "b", "d"]]
+teacher = SimulatorTeacher(traces)
+```
+
+**Membership queries**: Returns True if word is a prefix of any trace
+**Equivalence queries**: Tests hypothesis against all traces and their prefixes
+
+### Future Oracles
+
+- **InteractiveTeacher**: Query live device/system
+- **ModelTeacher**: Query formal specification or model
+
+## Observation Table
+
+The observation table is the core data structure:
+
+```python
+from oscura.inference.active_learning import ObservationTable
+
+table = ObservationTable(alphabet={"a", "b"})
+
+# Check properties
+table.is_closed()      # All extensions covered
+table.is_consistent()  # Equivalent rows have equivalent extensions
+
+# Convert to DFA
+dfa = table.to_dfa()
+```
+
+## Performance
+
+For small protocols (5-10 states):
+
+- Membership queries: 50-200
+- Equivalence queries: 3-10
+- Time: < 1 second
+
+The algorithm is polynomial in the number of states and alphabet size.
+
+## Academic Reference
+
+```
+Angluin, D. (1987). Learning regular sets from queries and counterexamples.
+Information and Computation, 75(2), 87-106.
+```
+
+## Comparison with RPNI
+
+|Feature|L\* (Active)|RPNI (Passive)|
+|---|---|---|---|---|---|---|
+|Learning type|Active (queries oracle)|Passive (fixed dataset)|
+|Minimal DFA|Yes|No (may have extra states)|
+|Negative examples|Not required|Optional|
+|Live learning|Yes|No|
+|Query complexity|O(|Q|²|Σ|)|N/A|## See Also
+
+- `oscura.inference.state_machine`: RPNI passive learning
+- `examples/lstar_demo.py`: Complete usage examples
+- `tests/unit/inference/test_lstar.py`: Test cases including academic examples

oscura/inference/active_learning/__init__.py

@@ -0,0 +1,38 @@
+"""Active learning for protocol inference.
+
+This module implements Angluin's L* algorithm for learning deterministic
+finite automata (DFAs) through active learning. Unlike passive learning
+(RPNI), active learning interacts with an oracle to learn the minimal DFA.
+
+Key advantages of L* over passive learning:
+- Learns minimal DFA (fewest states)
+- Polynomial query complexity
+- No negative examples required
+- Interactive learning from live systems
+
+References:
+    Angluin, D. (1987). Learning regular sets from queries and counterexamples.
+    Information and Computation, 75(2), 87-106.
+
+Example:
+    >>> from oscura.inference.active_learning import LStarLearner, SimulatorTeacher
+    >>> # Create oracle from captured traces
+    >>> traces = [['A', 'B', 'C'], ['A', 'B', 'B', 'C']]
+    >>> oracle = SimulatorTeacher(traces)
+    >>> # Learn DFA
+    >>> learner = LStarLearner(oracle)
+    >>> dfa = learner.learn()
+    >>> print(f"Learned DFA with {len(dfa.states)} states")
+"""
+
+from oscura.inference.active_learning.lstar import LStarLearner
+from oscura.inference.active_learning.observation_table import ObservationTable
+from oscura.inference.active_learning.oracle import Oracle
+from oscura.inference.active_learning.teachers import SimulatorTeacher
+
+__all__ = [
+    "LStarLearner",
+    "ObservationTable",
+    "Oracle",
+    "SimulatorTeacher",
+]

oscura/inference/active_learning/lstar.py

@@ -0,0 +1,257 @@
+"""Angluin's L* algorithm for DFA learning.
+
+This module implements the L* algorithm for learning deterministic finite
+automata through active learning. The algorithm interacts with an oracle
+to learn the minimal DFA for the target language.
+
+References:
+    Angluin, D. (1987). Learning regular sets from queries and counterexamples.
+    Information and Computation, 75(2), 87-106.
+
+Algorithm Overview:
+    1. Initialize observation table with S={ε}, E={ε}
+    2. Fill table using membership queries
+    3. While table not closed or not consistent:
+        - If not closed: add row to S
+        - If not consistent: add column to E
+        - Refill table
+    4. Construct hypothesis DFA from table
+    5. Ask equivalence query
+    6. If counterexample exists: process it and goto step 3
+    7. Return hypothesis
+
+Complexity:
+    - O(|Q|²|Σ|) membership queries where |Q| is number of states, |Σ| is alphabet
+    - O(|Q|) equivalence queries
+    - Produces minimal DFA (fewest states)
+"""
+
+from __future__ import annotations
+
+from oscura.inference.active_learning.observation_table import ObservationTable
+from oscura.inference.active_learning.oracle import Oracle  # noqa: TC001
+from oscura.inference.state_machine import FiniteAutomaton  # noqa: TC001
+
+
+class LStarLearner:
+    """Angluin's L* algorithm for learning DFAs.
+
+    The L* algorithm learns the minimal DFA for a regular language by
+    interacting with an oracle through membership and equivalence queries.
+
+    Key properties:
+    - Guaranteed convergence to correct minimal DFA
+    - Polynomial query complexity
+    - No negative examples required
+
+    Attributes:
+        oracle: Oracle for answering queries
+        verbose: Print algorithm progress
+        membership_queries: Count of membership queries made
+        equivalence_queries: Count of equivalence queries made
+    """
+
+    def __init__(self, oracle: Oracle, verbose: bool = False):
+        """Initialize L* learner.
+
+        Args:
+            oracle: Oracle for answering membership and equivalence queries
+            verbose: Print algorithm progress if True
+        """
+        self.oracle = oracle
+        self.verbose = verbose
+        self.membership_queries = 0
+        self.equivalence_queries = 0
+
+    def learn(self, max_iterations: int = 1000) -> FiniteAutomaton:
+        """Run L* algorithm to learn the target DFA.
+
+        The algorithm proceeds as follows:
+        1. Initialize observation table with S={ε}, E={ε}
+        2. Fill table using membership queries
+        3. Make table closed and consistent
+        4. Construct hypothesis DFA
+        5. Ask equivalence query
+        6. If counterexample: update table and repeat from step 3
+        7. Return final hypothesis
+
+        Args:
+            max_iterations: Maximum iterations to prevent infinite loops
+
+        Returns:
+            Learned minimal DFA
+
+        Raises:
+            ValueError: If algorithm doesn't converge within max_iterations
+        """
+        # Initialize observation table
+        alphabet = self.oracle.get_alphabet()
+        table = ObservationTable(alphabet=alphabet)
+
+        if self.verbose:
+            print(f"L* learning started with alphabet: {sorted(alphabet)}")
+
+        # Fill initial table
+        self._fill_table(table)
+
+        iteration = 0
+        while iteration < max_iterations:
+            iteration += 1
+
+            # Make table closed and consistent
+            changed = True
+            while changed:
+                changed = False
+
+                # Make closed
+                if self._make_closed(table):
+                    changed = True
+                    self._fill_table(table)
+
+                # Make consistent
+                if self._make_consistent(table):
+                    changed = True
+                    self._fill_table(table)
+
+            # Construct hypothesis
+            hypothesis = table.to_dfa()
+
+            if self.verbose:
+                print(f"Iteration {iteration}: Hypothesis has {len(hypothesis.states)} states")
+
+            # Equivalence query
+            self.equivalence_queries += 1
+            counterexample = self.oracle.equivalence_query(hypothesis)
+
+            if counterexample is None:
+                # Hypothesis is correct
+                if self.verbose:
+                    print(f"Learning complete after {iteration} iterations")
+                    print(f"Membership queries: {self.membership_queries}")
+                    print(f"Equivalence queries: {self.equivalence_queries}")
+                return hypothesis
+
+            # Process counterexample
+            if self.verbose:
+                print(f"Counterexample found: {counterexample}")
+
+            self._process_counterexample(table, counterexample)
+            self._fill_table(table)
+
+        raise ValueError(f"L* did not converge after {max_iterations} iterations")
+
+    def _fill_table(self, table: ObservationTable) -> None:
+        """Fill table entries using membership queries.
+
+        Fills all missing entries in the observation table by querying
+        the oracle for membership of prefix+suffix combinations.
+
+        For every s in S union SA and every e in E, ensures T[s·e] is filled.
+
+        Args:
+            table: Observation table to fill
+        """
+        # Compute SA = S union (S · Sigma)
+        SA = set(table.S)
+        for s in table.S:
+            for a in table.alphabet:
+                SA.add(s + (a,))
+
+        # Fill all entries
+        for s in SA:
+            for e in table.E:
+                word = s + e
+                if word not in table.T:
+                    self.membership_queries += 1
+                    table.T[word] = self.oracle.membership_query(word)
+
+    def _make_closed(self, table: ObservationTable) -> bool:
+        """Make table closed.
+
+        If table is not closed, adds one row from SA to S to move toward closure.
+
+        A table is closed when every row in SA has an equivalent row in S.
+        If not closed, we add the counterexample string to S.
+
+        Args:
+            table: Observation table to make closed
+
+        Returns:
+            True if table was modified, False if already closed
+        """
+        counterexample = table.find_closing_counterexample()
+
+        if counterexample is None:
+            return False
+
+        # Add counterexample to S
+        table.S.add(counterexample)
+
+        if self.verbose:
+            print(f"Added to S (closing): {counterexample}")
+
+        return True
+
+    def _make_consistent(self, table: ObservationTable) -> bool:
+        """Make table consistent.
+
+        If table is not consistent, adds a distinguishing suffix to E.
+
+        A table is consistent when identical rows in S have identical
+        one-step extensions. If inconsistent, we find the distinguishing
+        suffix and add it to E.
+
+        Args:
+            table: Observation table to make consistent
+
+        Returns:
+            True if table was modified, False if already consistent
+        """
+        result = table.find_consistency_counterexample()
+
+        if result is None:
+            return False
+
+        s1, s2, a = result
+
+        # Find distinguishing suffix
+        # row(s1·a) ≠ row(s2·a), so there exists e ∈ E where they differ
+        for e in table.E:
+            s1ae = s1 + (a,) + e
+            s2ae = s2 + (a,) + e
+
+            if table.T.get(s1ae, False) != table.T.get(s2ae, False):
+                # Found distinguishing suffix: a·e
+                distinguishing_suffix = (a,) + e
+                table.E.add(distinguishing_suffix)
+
+                if self.verbose:
+                    print(f"Added to E (consistency): {distinguishing_suffix}")
+
+                return True
+
+        # Should never reach here if find_consistency_counterexample worked correctly
+        return False
+
+    def _process_counterexample(
+        self, table: ObservationTable, counterexample: tuple[str, ...]
+    ) -> None:
+        """Process counterexample from equivalence query.
+
+        Adds all prefixes of the counterexample and the counterexample
+        itself to the observation table to refine the hypothesis.
+
+        This uses the "add all suffixes" strategy which is simple and
+        ensures progress.
+
+        Args:
+            table: Observation table to update
+            counterexample: Counterexample word from equivalence query
+        """
+        # Add all suffixes of counterexample to E
+        for i in range(len(counterexample) + 1):
+            suffix = counterexample[i:]
+            table.E.add(suffix)
+
+        if self.verbose:
+            print(f"Added {len(counterexample) + 1} suffixes to E from counterexample")