oscura 0.0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

oscura/analyzers/packet/stream.py

@@ -0,0 +1,359 @@
+"""Stream processing utilities for packet analysis.
+
+This module provides generator-based stream processing for
+O(1) memory usage when analyzing large packet captures.
+
+
+Example:
+    >>> from oscura.analyzers.packet.stream import stream_packets
+    >>> for packet in stream_packets(file_path, format="pcap"):
+    ...     process(packet)  # Process one at a time
+
+References:
+    Python generators and itertools patterns
+"""
+
+from __future__ import annotations
+
+import io
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, BinaryIO, TypeVar
+
+from oscura.analyzers.packet.parser import BinaryParser
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterator
+
+T = TypeVar("T")
+
+
+@dataclass
+class StreamPacket:
+    """Packet from a stream.
+
+    Attributes:
+        timestamp: Packet timestamp in seconds.
+        data: Raw packet data.
+        metadata: Additional packet metadata.
+    """
+
+    timestamp: float
+    data: bytes
+    metadata: dict[str, Any]
+
+
+def stream_file(
+    file_path: str | Path,
+    chunk_size: int = 65536,
+) -> Iterator[bytes]:
+    """Stream file in chunks.
+
+    Args:
+        file_path: Path to file.
+        chunk_size: Bytes per chunk (default 64KB).
+
+    Yields:
+        Byte chunks from file.
+
+    Example:
+        >>> for chunk in stream_file("large_capture.bin"):
+        ...     process_chunk(chunk)
+    """
+    path = Path(file_path)
+
+    with open(path, "rb") as f:
+        while True:
+            chunk = f.read(chunk_size)
+            if not chunk:
+                break
+            yield chunk
+
+
+def stream_records(
+    file_or_buffer: str | Path | BinaryIO | bytes,
+    record_size: int,
+) -> Iterator[bytes]:
+    """Stream fixed-size records.
+
+    Args:
+        file_or_buffer: Source file path, file object, or bytes.
+        record_size: Size of each record in bytes.
+
+    Yields:
+        Records as bytes objects.
+
+    Example:
+        >>> for record in stream_records("data.bin", record_size=128):
+        ...     parse_record(record)
+    """
+    if isinstance(file_or_buffer, bytes):
+        buffer: BinaryIO = io.BytesIO(file_or_buffer)
+        should_close = True
+    elif isinstance(file_or_buffer, str | Path):
+        buffer = open(file_or_buffer, "rb")  # noqa: SIM115
+        should_close = True
+    else:
+        buffer = file_or_buffer
+        should_close = False
+
+    try:
+        while True:
+            record = buffer.read(record_size)
+            if len(record) < record_size:
+                break
+            yield record
+    finally:
+        if should_close:
+            buffer.close()
+
+
+def stream_packets(
+    file_or_buffer: str | Path | BinaryIO | bytes,
+    *,
+    header_parser: BinaryParser | None = None,
+    length_field: int = 1,
+    header_included: bool = False,
+) -> Iterator[StreamPacket]:
+    """Stream variable-length packets.
+
+    Parses packets with length-prefixed format.
+
+    Args:
+        file_or_buffer: Source.
+        header_parser: Parser for packet header.
+        length_field: Index of length field in header (default 1).
+        header_included: True if length includes header.
+
+    Yields:
+        StreamPacket objects.
+
+    Example:
+        >>> header = BinaryParser(">HH")  # sync, length
+        >>> for pkt in stream_packets("capture.bin", header_parser=header):
+        ...     print(f"Packet: {len(pkt.data)} bytes")
+    """
+    if header_parser is None:
+        # Default: 2-byte big-endian length prefix
+        header_parser = BinaryParser(">H")
+        length_field = 0
+
+    header_size = header_parser.size
+
+    if isinstance(file_or_buffer, bytes):
+        buffer: BinaryIO = io.BytesIO(file_or_buffer)
+        should_close = True
+    elif isinstance(file_or_buffer, str | Path):
+        buffer = open(file_or_buffer, "rb")  # noqa: SIM115
+        should_close = True
+    else:
+        buffer = file_or_buffer
+        should_close = False
+
+    try:
+        packet_num = 0
+
+        while True:
+            # Read header
+            header_bytes = buffer.read(header_size)
+            if len(header_bytes) < header_size:
+                break
+
+            header = header_parser.unpack(header_bytes)
+            length = header[length_field]
+
+            # Compute payload size
+            payload_size = length - header_size if header_included else length
+
+            if payload_size < 0:
+                break
+
+            # Read payload
+            payload = buffer.read(payload_size)
+            if len(payload) < payload_size:
+                break
+
+            packet_num += 1
+
+            yield StreamPacket(
+                timestamp=packet_num,  # Placeholder
+                data=header_bytes + payload,
+                metadata={"header": header, "packet_num": packet_num},
+            )
+
+    finally:
+        if should_close:
+            buffer.close()
+
+
+def stream_delimited(
+    file_or_buffer: str | Path | BinaryIO | bytes,
+    delimiter: bytes = b"\n",
+    *,
+    max_record_size: int = 1048576,
+) -> Iterator[bytes]:
+    """Stream delimiter-separated records.
+
+    Args:
+        file_or_buffer: Source.
+        delimiter: Record delimiter (default newline).
+        max_record_size: Maximum record size (default 1MB).
+
+    Yields:
+        Records as bytes (without delimiter).
+
+    Example:
+        >>> for line in stream_delimited("log.txt", b"\\n"):
+        ...     process_line(line)
+    """
+    if isinstance(file_or_buffer, bytes):
+        buffer: BinaryIO = io.BytesIO(file_or_buffer)
+        should_close = True
+    elif isinstance(file_or_buffer, str | Path):
+        buffer = open(file_or_buffer, "rb")  # noqa: SIM115
+        should_close = True
+    else:
+        buffer = file_or_buffer
+        should_close = False
+
+    try:
+        partial = b""
+
+        while True:
+            chunk = buffer.read(65536)
+            if not chunk:
+                if partial:
+                    yield partial
+                break
+
+            data = partial + chunk
+            parts = data.split(delimiter)
+
+            # Yield complete records
+            for part in parts[:-1]:
+                if len(part) <= max_record_size:
+                    yield part
+
+            # Keep partial record for next iteration
+            partial = parts[-1]
+
+            # Guard against memory issues
+            if len(partial) > max_record_size:
+                yield partial[:max_record_size]
+                partial = b""
+
+    finally:
+        if should_close:
+            buffer.close()
+
+
+def pipeline(
+    source: Iterator[T],
+    *transforms: Callable[[Iterator[T]], Iterator[T]],
+) -> Iterator[T]:
+    """Chain processing transforms.
+
+    Args:
+        source: Source iterator.
+        *transforms: Transform functions.
+
+    Yields:
+        Transformed items.
+
+    Example:
+        >>> def filter_large(packets):
+        ...     for pkt in packets:
+        ...         if len(pkt.data) > 100:
+        ...             yield pkt
+        ...
+        >>> def decode(packets):
+        ...     for pkt in packets:
+        ...         pkt.metadata["decoded"] = decode_packet(pkt.data)
+        ...         yield pkt
+        ...
+        >>> for pkt in pipeline(stream_packets(f), filter_large, decode):
+        ...     print(pkt)
+    """
+    result: Iterator = source  # type: ignore[type-arg]
+
+    for transform in transforms:
+        result = transform(result)
+
+    yield from result
+
+
+def batch[T](
+    source: Iterator[T],
+    size: int,
+) -> Iterator[list[T]]:
+    """Batch items from iterator.
+
+    Args:
+        source: Source iterator.
+        size: Batch size.
+
+    Yields:
+        Lists of items.
+
+    Example:
+        >>> for batch_items in batch(stream_packets(f), size=100):
+        ...     process_batch(batch_items)
+    """
+    current_batch: list[T] = []
+
+    for item in source:
+        current_batch.append(item)
+        if len(current_batch) >= size:
+            yield current_batch
+            current_batch = []
+
+    if current_batch:
+        yield current_batch
+
+
+def take[T](source: Iterator[T], n: int) -> Iterator[T]:
+    """Take first n items.
+
+    Args:
+        source: Source iterator.
+        n: Number of items to take.
+
+    Yields:
+        First n items.
+    """
+    count = 0
+    for item in source:
+        if count >= n:
+            break
+        yield item
+        count += 1  # noqa: SIM113
+
+
+def skip[T](source: Iterator[T], n: int) -> Iterator[T]:
+    """Skip first n items.
+
+    Args:
+        source: Source iterator.
+        n: Number of items to skip.
+
+    Yields:
+        Items after first n.
+    """
+    count = 0
+    for item in source:
+        if count >= n:
+            yield item
+        count += 1  # noqa: SIM113
+
+
+__all__ = [
+    "StreamPacket",
+    "batch",
+    "pipeline",
+    "skip",
+    "stream_delimited",
+    "stream_file",
+    "stream_packets",
+    "stream_records",
+    "take",
+]

oscura/analyzers/patterns/__init__.py

@@ -0,0 +1,266 @@
+"""Pattern Detection & Analysis module for Oscura.
+
+This module provides comprehensive pattern detection and analysis capabilities
+for digital signals and binary data, including:
+
+- Periodic pattern detection (autocorrelation, FFT, suffix array)
+- Repeating sequence detection (n-grams, LRS, approximate matching)
+- Automatic signature discovery (headers, delimiters, magic bytes)
+- Pattern clustering by similarity (Hamming, edit distance, hierarchical)
+- Binary regex pattern matching
+- Multi-pattern search (Aho-Corasick)
+- Fuzzy/approximate pattern matching
+- Pattern learning and discovery
+
+    - RE-PAT-001: Binary Regex Pattern Matching
+    - RE-PAT-002: Multi-Pattern Search (Aho-Corasick)
+    - RE-PAT-003: Fuzzy Pattern Matching
+    - RE-PAT-004: Pattern Learning and Discovery
+
+Author: Oscura Development Team
+"""
+
+# Periodic pattern detection (PAT-001)
+# Pattern clustering (PAT-004)
+from .clustering import (
+    ClusteringResult,
+    ClusterResult,
+    analyze_cluster,
+    cluster_by_edit_distance,
+    cluster_by_hamming,
+    cluster_hierarchical,
+    compute_distance_matrix,
+)
+
+# Signature discovery (PAT-003)
+from .discovery import (
+    CandidateSignature,
+    SignatureDiscovery,
+    discover_signatures,
+    find_delimiter_candidates,
+    find_header_candidates,
+)
+
+# RE-PAT-004: Pattern Learning and Discovery
+from .learning import (
+    LearnedPattern,
+    NgramModel,
+    PatternLearner,
+    StructureHypothesis,
+    find_recurring_structures,
+    infer_structure,
+    learn_patterns_from_data,
+)
+
+# RE-PAT-001, RE-PAT-002, RE-PAT-003: Advanced pattern matching
+from .matching import (
+    AhoCorasickMatcher,
+    # Classes
+    BinaryRegex,
+    FuzzyMatcher,
+    FuzzyMatchResult,
+    # Data classes
+    PatternMatchResult,
+    # RE-PAT-001: Binary Regex
+    binary_regex_search,
+    count_pattern_occurrences,
+    # Utilities
+    find_pattern_positions,
+    find_similar_sequences,
+    # RE-PAT-003: Fuzzy Matching
+    fuzzy_search,
+    # RE-PAT-002: Multi-Pattern Search
+    multi_pattern_search,
+)
+from .periodic import (
+    PeriodicPatternDetector,
+    PeriodResult,
+    detect_period,
+    detect_periods_autocorr,
+    detect_periods_fft,
+    validate_period,
+)
+
+# Alias for backward compatibility
+detect_period_autocorr = detect_periods_autocorr
+detect_period_fft = detect_periods_fft
+
+# Repeating sequence detection (PAT-002)
+# Motif detection functions (aliases for test compatibility)
+from typing import TYPE_CHECKING, Any, cast
+
+from .sequences import (
+    NgramResult,
+    RepeatingSequence,
+    find_approximate_repeats,
+    find_frequent_ngrams,
+    find_longest_repeat,
+    find_repeating_sequences,
+)
+
+if TYPE_CHECKING:
+    import numpy as np
+    from numpy.typing import NDArray
+
+
+def find_motifs(
+    data: Any, motif_length: int = 8, max_distance: float = 0.1
+) -> list[RepeatingSequence]:
+    """Find motifs (repeating patterns) in data.
+
+    This is an alias for find_repeating_sequences for test compatibility.
+
+    Args:
+        data: Input data array.
+        motif_length: Length of motifs to find.
+        max_distance: Maximum distance for fuzzy matching (unused).
+
+    Returns:
+        List of RepeatingSequence objects.
+    """
+    import numpy as np
+
+    data = np.asarray(data)
+    results = find_repeating_sequences(data, min_length=motif_length, min_count=2)
+    return results
+
+
+def extract_motif(data: Any, start: int, length: int) -> "NDArray[np.generic]":
+    """Extract a motif from data.
+
+    Args:
+        data: Input data array.
+        start: Start index.
+        length: Length to extract.
+
+    Returns:
+        Extracted motif as numpy array.
+    """
+    import numpy as np
+    from numpy.typing import NDArray
+
+    data_arr = np.asarray(data)
+    result: NDArray[np.generic] = data_arr[start : start + length]
+    return result
+
+
+def detect_anomalies(data: Any, threshold: float = 3.0) -> list[int]:
+    """Detect anomalies in data using z-score.
+
+    Args:
+        data: Input data array.
+        threshold: Z-score threshold for anomaly detection.
+
+    Returns:
+        List of anomaly indices.
+    """
+    import numpy as np
+
+    data_arr = np.asarray(data)
+    mean = np.mean(data_arr)
+    std = np.std(data_arr)
+    if std == 0:
+        return []
+
+    z_scores = np.abs((data_arr - mean) / std)
+    indices = np.where(z_scores > threshold)[0].tolist()
+    return cast("list[int]", indices)
+
+
+def cluster_patterns(patterns: Any, method: str = "hamming") -> ClusteringResult:
+    """Cluster patterns by similarity.
+
+    Args:
+        patterns: List of patterns to cluster.
+        method: Clustering method ('hamming' or 'edit').
+
+    Returns:
+        ClusteringResult with cluster assignments.
+    """
+    if method == "hamming":
+        return cluster_by_hamming(patterns)
+    else:
+        return cluster_by_edit_distance(patterns)
+
+
+def pattern_similarity(pattern1: Any, pattern2: Any) -> float:
+    """Calculate similarity between two patterns.
+
+    Args:
+        pattern1: First pattern.
+        pattern2: Second pattern.
+
+    Returns:
+        Similarity score (0-1, 1 = identical).
+    """
+    import numpy as np
+
+    p1 = np.asarray(pattern1)
+    p2 = np.asarray(pattern2)
+
+    if len(p1) != len(p2):
+        return 0.0
+
+    if len(p1) == 0:
+        return 1.0
+
+    matches = int(np.sum(p1 == p2))
+    return float(matches / len(p1))
+
+
+__all__ = [
+    # RE-PAT-002: Multi-Pattern Search
+    "AhoCorasickMatcher",
+    # RE-PAT-001: Binary Regex Pattern Matching
+    "BinaryRegex",
+    "CandidateSignature",
+    "ClusterResult",
+    "ClusteringResult",
+    "FuzzyMatchResult",
+    # RE-PAT-003: Fuzzy Pattern Matching
+    "FuzzyMatcher",
+    # RE-PAT-004: Pattern Learning and Discovery
+    "LearnedPattern",
+    "NgramModel",
+    "NgramResult",
+    "PatternLearner",
+    "PatternMatchResult",
+    "PeriodResult",
+    "PeriodicPatternDetector",
+    "RepeatingSequence",
+    "SignatureDiscovery",
+    "StructureHypothesis",
+    "analyze_cluster",
+    "binary_regex_search",
+    "cluster_by_edit_distance",
+    "cluster_by_hamming",
+    "cluster_hierarchical",
+    "cluster_patterns",
+    "compute_distance_matrix",
+    "count_pattern_occurrences",
+    # Motif detection (compatibility)
+    "detect_anomalies",
+    "detect_period",
+    "detect_period_autocorr",
+    "detect_period_fft",
+    "detect_periods_autocorr",
+    "detect_periods_fft",
+    "discover_signatures",
+    "extract_motif",
+    "find_approximate_repeats",
+    "find_delimiter_candidates",
+    "find_frequent_ngrams",
+    "find_header_candidates",
+    "find_longest_repeat",
+    "find_motifs",
+    "find_pattern_positions",
+    "find_recurring_structures",
+    "find_repeating_sequences",
+    "find_similar_sequences",
+    "fuzzy_search",
+    "infer_structure",
+    "learn_patterns_from_data",
+    "multi_pattern_search",
+    "pattern_similarity",
+    "validate_period",
+]