oscura 0.0.1__py3-none-any.whl → 0.1.1__py3-none-any.whl

oscura/exploratory/parse.py

@@ -0,0 +1,358 @@
+"""Error-tolerant protocol parsing with timestamp correction.
+
+
+This module provides robust protocol decoding that continues after errors
+and timestamp correction for jittery captures.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Literal
+
+import numpy as np
+from numpy.typing import NDArray
+from scipy import signal
+
+
+class ErrorTolerance(Enum):
+    """Error tolerance modes for protocol decoding.
+
+    Attributes:
+        STRICT: Abort on first error (backward compatible)
+        TOLERANT: Skip error frame, resync, continue (default)
+        PERMISSIVE: Best-effort decode, report all errors
+    """
+
+    STRICT = "strict"
+    TOLERANT = "tolerant"
+    PERMISSIVE = "permissive"
+
+
+@dataclass
+class DecodedFrame:
+    """Decoded protocol frame with error annotation.
+
+    Attributes:
+        data: Decoded data bytes
+        timestamp: Frame timestamp in seconds
+        valid: Whether frame is valid or has errors
+        error_type: Type of error if invalid (e.g., 'framing', 'parity')
+        position: Byte position in original trace
+    """
+
+    data: bytes
+    timestamp: float
+    valid: bool
+    error_type: str | None
+    position: int
+
+
+@dataclass
+class TimestampCorrection:
+    """Result from timestamp jitter correction.
+
+    Attributes:
+        corrected_timestamps: Array of corrected timestamps
+        original_jitter_rms: RMS jitter before correction
+        corrected_jitter_rms: RMS jitter after correction
+        reduction_ratio: Jitter reduction factor (before/after)
+        samples_corrected: Number of samples that were adjusted
+        max_correction: Maximum correction applied to any sample
+    """
+
+    corrected_timestamps: NDArray[np.float64]
+    original_jitter_rms: float
+    corrected_jitter_rms: float
+    reduction_ratio: float
+    samples_corrected: int
+    max_correction: float
+
+
+def correct_timestamp_jitter(
+    timestamps: NDArray[np.float64],
+    expected_rate: float,
+    *,
+    method: Literal["lowpass", "pll"] = "lowpass",
+    max_correction_factor: float = 2.0,
+) -> TimestampCorrection:
+    """Correct timestamp jitter using filtering or PLL model.
+
+    : Compensates for clock jitter in logic analyzer
+    captures (e.g., USB transmission jitter) while preserving phase.
+
+    Correction constraints (DAQ-003):
+    - Max correction per sample: ±max_correction_factor × expected_period  # noqa: RUF002, RUF003
+    - Filter cutoff: expected_rate / 10 (removes 10× jitter frequency)  # noqa: RUF002, RUF003
+    - Target reduction: ≥5× for typical USB jitter  # noqa: RUF002, RUF003
+
+    Args:
+        timestamps: Original jittery timestamps in seconds
+        expected_rate: Expected nominal sample rate in Hz
+        method: Correction method ('lowpass' or 'pll')
+        max_correction_factor: Max correction as multiple of period
+
+    Returns:
+        TimestampCorrection with corrected timestamps and metrics
+
+    Raises:
+        ValueError: If timestamps array is empty
+        ValueError: If expected_rate <= 0
+        ValueError: If max_correction_factor <= 0
+
+    Examples:
+        >>> # Correct jittery timestamps from USB logic analyzer
+        >>> import numpy as np
+        >>> timestamps = np.linspace(0, 1e-3, 1000)
+        >>> jitter = np.random.normal(0, 1e-7, 1000)  # 100ns jitter
+        >>> jittery = timestamps + jitter
+        >>> result = correct_timestamp_jitter(jittery, expected_rate=1e6)
+        >>> print(f"Jitter reduced by {result.reduction_ratio:.1f}x")
+
+    References:
+        DAQ-003: Timestamp Jitter Compensation and Clock Correction
+    """
+    if len(timestamps) == 0:
+        raise ValueError("Timestamps array cannot be empty")
+
+    if expected_rate <= 0:
+        raise ValueError("expected_rate must be positive")
+
+    if max_correction_factor <= 0:
+        raise ValueError("max_correction_factor must be positive")
+
+    if len(timestamps) < 3:
+        # Not enough data to filter
+        return TimestampCorrection(
+            corrected_timestamps=timestamps.copy(),
+            original_jitter_rms=0.0,
+            corrected_jitter_rms=0.0,
+            reduction_ratio=1.0,
+            samples_corrected=0,
+            max_correction=0.0,
+        )
+
+    expected_period = 1.0 / expected_rate
+    max_correction = max_correction_factor * expected_period
+
+    # Calculate original jitter
+    diffs = np.diff(timestamps)
+    original_jitter = diffs - expected_period
+    original_jitter_rms = float(np.sqrt(np.mean(original_jitter**2)))
+
+    # If jitter is negligible (below 1 ns), no correction needed
+    # This avoids correcting floating-point rounding errors in perfect timestamps
+    if original_jitter_rms < 1e-9:
+        return TimestampCorrection(
+            corrected_timestamps=timestamps.copy(),
+            original_jitter_rms=original_jitter_rms,
+            corrected_jitter_rms=original_jitter_rms,
+            reduction_ratio=1.0,
+            samples_corrected=0,
+            max_correction=0.0,
+        )
+
+    if method == "lowpass":
+        # Low-pass filter approach
+        # Design Butterworth filter: cutoff at expected_rate / 10
+        cutoff_freq = expected_rate / 10.0
+        nyquist = 0.5 * expected_rate
+
+        # Ensure cutoff is valid
+        if cutoff_freq >= nyquist:
+            cutoff_freq = nyquist * 0.8
+
+        # Design 2nd order Butterworth
+        sos = signal.butter(2, cutoff_freq / nyquist, btype="low", output="sos")
+
+        # Filter the timestamps
+        # Need to detrend first to avoid edge effects
+        t_mean = np.mean(timestamps)
+        t_detrended = timestamps - t_mean
+
+        # Apply filter
+        filtered = signal.sosfiltfilt(sos, t_detrended)
+        corrected = filtered + t_mean
+
+    else:  # pll
+        # Phase-locked loop model
+        # Simple PLL: track expected phase and correct deviations
+        corrected = np.zeros_like(timestamps)
+        corrected[0] = timestamps[0]
+
+        # PLL state
+        phase = 0.0
+        phase_increment = 2 * np.pi * expected_rate
+
+        for i in range(1, len(timestamps)):
+            # Predict next timestamp based on expected rate
+            predicted = corrected[i - 1] + expected_period
+
+            # Measure phase error
+            actual = timestamps[i]
+            error = actual - predicted
+
+            # Apply correction with limiting
+            correction = np.clip(error * 0.5, -max_correction, max_correction)
+            corrected[i] = predicted + correction
+
+            # Update phase
+            phase += phase_increment * (corrected[i] - corrected[i - 1])
+
+    # Limit corrections to max_correction
+    corrections = corrected - timestamps
+    exceeded = np.abs(corrections) > max_correction
+    corrections[exceeded] = np.sign(corrections[exceeded]) * max_correction
+    corrected = timestamps + corrections
+
+    # Calculate corrected jitter
+    corrected_diffs = np.diff(corrected)
+    corrected_jitter = corrected_diffs - expected_period
+    corrected_jitter_rms = float(np.sqrt(np.mean(corrected_jitter**2)))
+
+    # Calculate metrics
+    samples_corrected = int(np.sum(np.abs(corrections) > 1e-12))
+    max_correction_applied = float(np.max(np.abs(corrections)))
+
+    # original_jitter_rms is always > 0 here (early return handles negligible jitter)
+    reduction_ratio = original_jitter_rms / max(corrected_jitter_rms, 1e-15)
+
+    return TimestampCorrection(
+        corrected_timestamps=corrected,
+        original_jitter_rms=original_jitter_rms,
+        corrected_jitter_rms=corrected_jitter_rms,
+        reduction_ratio=reduction_ratio,
+        samples_corrected=samples_corrected,
+        max_correction=max_correction_applied,
+    )
+
+
+def decode_with_error_tolerance(
+    data: NDArray[np.uint8],
+    protocol: Literal["uart", "spi", "i2c", "can"],
+    *,
+    tolerance: ErrorTolerance = ErrorTolerance.TOLERANT,
+    **protocol_params: Any,
+) -> list[DecodedFrame]:
+    """Decode protocol with error tolerance and resynchronization.
+
+    : Continues decoding after framing/parity/stop-bit
+    errors instead of aborting. Applies to all protocol decoders.
+
+    Error tolerance modes (DAQ-004):
+    - STRICT: Abort on first error (backward compatible)
+    - TOLERANT: Skip error frame, resync, continue (default)
+    - PERMISSIVE: Best-effort decode, report all errors
+
+    Resynchronization strategies (DAQ-004):
+    - UART: Search for next valid start bit + stop bit pattern
+    - SPI: Re-align on next CS edge
+    - I2C: Search for next START condition
+    - CAN: Wait for recessive bus + next SOF
+
+    Args:
+        data: Raw protocol data bytes
+        protocol: Protocol type ('uart', 'spi', 'i2c', 'can')
+        tolerance: Error tolerance mode
+        **protocol_params: Protocol-specific parameters (baud, parity, etc.)
+
+    Returns:
+        List of DecodedFrame objects with data and error annotations
+
+    Raises:
+        ValueError: If protocol not supported
+        ValueError: If required protocol_params missing
+        Exception: Re-raised in STRICT mode if decoding fails
+
+    Examples:
+        >>> # Decode UART with error tolerance
+        >>> data = np.array([0xFF, 0x55, 0xAA, 0x00], dtype=np.uint8)
+        >>> frames = decode_with_error_tolerance(
+        ...     data, 'uart', tolerance=ErrorTolerance.TOLERANT, baud=9600
+        ... )
+        >>> valid_frames = [f for f in frames if f.valid]
+
+    References:
+        DAQ-004: Error-Tolerant Protocol Decoding with Resynchronization
+    """
+    if protocol not in ("uart", "spi", "i2c", "can"):
+        raise ValueError(f"Unsupported protocol: {protocol}")
+
+    frames: list[DecodedFrame] = []
+    pos = 0
+
+    # Protocol-specific decode logic
+    # This is a simplified implementation showing the error handling pattern
+    # Full protocol decoders are in oscura.analyzers.protocols
+
+    if protocol == "uart":
+        # UART parameters
+        if "baud" not in protocol_params:
+            raise ValueError("UART requires 'baud' parameter")
+
+        # Simplified UART frame extraction with error tolerance
+        while pos < len(data):
+            try:
+                # Try to decode frame at current position
+                # This is simplified - real UART decoder would analyze bit timing
+
+                # Check for valid frame (simplified)
+                if pos + 1 >= len(data):
+                    break
+
+                frame_data = bytes([data[pos]])
+                timestamp = float(pos) / protocol_params["baud"]
+
+                # Validate frame (simplified - would check start/stop bits)
+                is_valid = True
+                error_type = None
+
+                # Example: detect framing error (no proper stop bit)
+                if data[pos] == 0xFF:  # Example error condition
+                    is_valid = False
+                    error_type = "framing"
+
+                frames.append(
+                    DecodedFrame(
+                        data=frame_data,
+                        timestamp=timestamp,
+                        valid=is_valid,
+                        error_type=error_type,
+                        position=pos,
+                    )
+                )
+
+                if not is_valid and tolerance == ErrorTolerance.STRICT:
+                    # Strict mode: abort on error
+                    break
+                elif not is_valid and tolerance == ErrorTolerance.TOLERANT:
+                    # Tolerant: skip error frame, resync
+                    # Search for next valid start bit
+                    pos += 1
+                    # In real implementation, would search for start bit pattern
+                else:
+                    # Permissive: record error, continue
+                    pos += 1
+
+            except Exception:
+                if tolerance == ErrorTolerance.STRICT:
+                    raise
+                else:
+                    # Log error and continue
+                    pos += 1
+
+    elif protocol == "spi":
+        # SPI: Re-align on CS edge
+        # Simplified placeholder
+        pass
+
+    elif protocol == "i2c":
+        # I2C: Search for START condition
+        # Simplified placeholder
+        pass
+
+    elif protocol == "can":
+        # CAN: Wait for SOF after error
+        # Simplified placeholder
+        pass
+
+    return frames

oscura/exploratory/recovery.py

@@ -0,0 +1,275 @@
+"""Bit error pattern analysis and capture diagnostics.
+
+
+This module characterizes bit error patterns to diagnose capture quality
+issues (EMI, USB problems, clock jitter) and suggests likely causes.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+
+import numpy as np
+from numpy.typing import NDArray
+
+
+class ErrorPattern(Enum):
+    """Classified error pattern types.
+
+    Attributes:
+        RANDOM: Errors uniformly distributed, no clustering (likely EMI)
+        BURST: Errors clustered together (likely USB transmission issue)
+        PERIODIC: Errors repeat at regular intervals (likely clock jitter)
+        UNKNOWN: Pattern doesn't match known types
+    """
+
+    RANDOM = "random"
+    BURST = "burst"
+    PERIODIC = "periodic"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class ErrorAnalysis:
+    """Result from bit error pattern analysis.
+
+    Attributes:
+        bit_error_rate: Ratio of errors to total bits
+        error_count: Total number of bit errors detected
+        total_bits: Total bits examined
+        pattern_type: Classified error pattern (random, burst, periodic)
+        mean_error_gap: Mean number of bits between errors
+        error_positions: Array of bit positions where errors occurred
+        diagnosis: Suggested cause based on pattern
+        severity: Error severity level (low, moderate, severe)
+    """
+
+    bit_error_rate: float
+    error_count: int
+    total_bits: int
+    pattern_type: ErrorPattern
+    mean_error_gap: float
+    error_positions: NDArray[np.int64]
+    diagnosis: str
+    severity: str
+
+
+def analyze_bit_errors(
+    received: NDArray[np.uint8],
+    expected: NDArray[np.uint8],
+    *,
+    burst_threshold: int = 100,
+    periodicity_threshold: float = 0.1,
+) -> ErrorAnalysis:
+    """Characterize bit error patterns for capture diagnostics.
+
+    : Analyzes bit errors to diagnose capture quality
+    issues and distinguish between EMI, USB problems, and clock jitter.
+
+    Error pattern classification (DAQ-005):
+    - Random: Errors uniformly distributed, no clustering
+    - Burst: Errors clustered, mean_gap < 100 bits
+    - Periodic: Errors repeat at regular intervals (FFT peak in positions)
+
+    Diagnosis suggestions (DAQ-005):
+    - BER > 0.01: Severe capture issue, check connections
+    - BER 0.001-0.01: Moderate errors, reduce sample rate
+    - BER < 0.001: Acceptable, likely EMI
+    - Burst errors: USB transmission issue
+    - Periodic errors: Clock jitter or interference
+
+    Args:
+        received: Received bit array (actual capture)
+        expected: Expected bit array (golden reference)
+        burst_threshold: Mean gap threshold for burst classification
+        periodicity_threshold: FFT peak threshold for periodic detection
+
+    Returns:
+        ErrorAnalysis with BER, pattern type, and diagnosis
+
+    Raises:
+        ValueError: If received and expected have different lengths
+        ValueError: If arrays are empty
+
+    Examples:
+        >>> # Analyze random EMI errors
+        >>> import numpy as np
+        >>> expected = np.random.randint(0, 2, 10000, dtype=np.uint8)
+        >>> received = expected.copy()
+        >>> errors = np.random.choice(10000, 50, replace=False)
+        >>> received[errors] = 1 - received[errors]  # Flip bits
+        >>> analysis = analyze_bit_errors(received, expected)
+        >>> print(f"BER: {analysis.bit_error_rate:.6f}")
+        >>> print(f"Pattern: {analysis.pattern_type.value}")
+
+        >>> # Analyze burst errors (USB issue)
+        >>> received = expected.copy()
+        >>> received[1000:1050] = 1 - received[1000:1050]  # 50-bit burst
+        >>> analysis = analyze_bit_errors(received, expected)
+        >>> print(analysis.diagnosis)
+        'USB transmission issue'
+
+    References:
+        DAQ-005: Bit Error Pattern Analysis and Capture Diagnostics
+    """
+    if len(received) != len(expected):
+        raise ValueError("Received and expected arrays must have same length")
+
+    if len(received) == 0:
+        raise ValueError("Arrays cannot be empty")
+
+    # Find bit errors (XOR)
+    errors = received != expected
+    error_positions = np.where(errors)[0]
+    error_count = len(error_positions)
+    total_bits = len(received)
+
+    # Calculate BER
+    bit_error_rate = error_count / total_bits if total_bits > 0 else 0.0
+
+    if error_count == 0:
+        # No errors
+        return ErrorAnalysis(
+            bit_error_rate=0.0,
+            error_count=0,
+            total_bits=total_bits,
+            pattern_type=ErrorPattern.RANDOM,
+            mean_error_gap=float(total_bits),
+            error_positions=error_positions,
+            diagnosis="No errors detected - good capture quality",
+            severity="low",
+        )
+
+    # Calculate error gaps
+    if error_count > 1:
+        error_gaps = np.diff(error_positions)
+        mean_gap = float(np.mean(error_gaps))
+    else:
+        mean_gap = float(total_bits)
+
+    # Classify error pattern
+    pattern_type = ErrorPattern.UNKNOWN
+    diagnosis = ""
+
+    # Check for burst pattern (errors clustered)
+    if error_count > 1 and mean_gap < burst_threshold:
+        pattern_type = ErrorPattern.BURST
+        diagnosis = "Burst errors detected - likely USB transmission issue"
+
+    # Check for periodic pattern (FFT analysis)
+    # Need at least 10 errors for reliable periodicity detection
+    elif error_count >= 10:
+        # Create binary error signal
+        error_signal = errors.astype(float)
+
+        # Compute FFT to detect periodicity
+        fft = np.fft.rfft(error_signal)
+        fft_mag = np.abs(fft[1:])  # Skip DC component
+
+        if len(fft_mag) > 0:
+            # Check if there's a strong peak relative to mean (not just max)
+            mean_mag = np.mean(fft_mag)
+            max_mag = np.max(fft_mag)
+            peak_ratio = max_mag / (mean_mag + 1e-12)
+
+            # Require strong peak (>10x mean) and exceeds threshold
+            if peak_ratio > 10 and (max_mag / (np.max(fft_mag) + 1e-12)) > periodicity_threshold:
+                pattern_type = ErrorPattern.PERIODIC
+                diagnosis = "Periodic errors detected - likely clock jitter or interference"
+
+    # If not burst or periodic, classify as random
+    if pattern_type == ErrorPattern.UNKNOWN:
+        # Check if errors are uniformly distributed
+        if error_count > 2:
+            # Use coefficient of variation of gaps
+            if error_count > 1:
+                gap_std = float(np.std(error_gaps))
+                gap_cv = gap_std / (mean_gap + 1e-12)
+
+                if gap_cv < 1.0:  # Relatively uniform spacing
+                    pattern_type = ErrorPattern.RANDOM
+                    diagnosis = "Random errors detected - likely EMI or noise"
+                else:
+                    diagnosis = "Mixed error pattern - multiple causes possible"
+            else:
+                pattern_type = ErrorPattern.RANDOM
+                diagnosis = "Single error - insufficient data for classification"
+        else:
+            pattern_type = ErrorPattern.RANDOM
+            diagnosis = "Few errors - likely random EMI or noise"
+
+    # Determine severity based on BER
+    if bit_error_rate > 0.01:
+        severity = "severe"
+        diagnosis += ". SEVERE: Check connections and hardware"
+    elif bit_error_rate > 0.001:
+        severity = "moderate"
+        diagnosis += ". MODERATE: Consider reducing sample rate"
+    else:
+        severity = "low"
+        diagnosis += ". Acceptable error rate"
+
+    return ErrorAnalysis(
+        bit_error_rate=bit_error_rate,
+        error_count=error_count,
+        total_bits=total_bits,
+        pattern_type=pattern_type,
+        mean_error_gap=mean_gap,
+        error_positions=error_positions,
+        diagnosis=diagnosis,
+        severity=severity,
+    )
+
+
+def generate_error_visualization_data(
+    analysis: ErrorAnalysis,
+    *,
+    histogram_bins: int = 50,
+) -> dict[str, NDArray[np.float64]]:
+    """Generate data for error distribution visualization.
+
+    Creates histogram and timeline data suitable for plotting error patterns.
+
+    Args:
+        analysis: ErrorAnalysis result from analyze_bit_errors()
+        histogram_bins: Number of bins for error position histogram
+
+    Returns:
+        Dictionary with 'histogram_counts', 'histogram_edges', and
+        'timeline' arrays for visualization
+
+    Examples:
+        >>> # Generate visualization data
+        >>> analysis = analyze_bit_errors(received, expected)
+        >>> viz_data = generate_error_visualization_data(analysis)
+        >>> # Plot with matplotlib
+        >>> import matplotlib.pyplot as plt
+        >>> plt.hist(analysis.error_positions, bins=viz_data['histogram_edges'])
+        >>> plt.xlabel('Bit Position')
+        >>> plt.ylabel('Error Count')
+        >>> plt.show()
+
+    References:
+        DAQ-005: Bit Error Pattern Analysis and Capture Diagnostics
+    """
+    if len(analysis.error_positions) == 0:
+        # No errors - return empty data
+        return {
+            "histogram_counts": np.array([], dtype=np.float64),
+            "histogram_edges": np.array([], dtype=np.float64),
+            "timeline": np.array([], dtype=np.float64),
+        }
+
+    # Generate histogram over full bit range
+    counts, edges = np.histogram(
+        analysis.error_positions, bins=histogram_bins, range=(0, analysis.total_bits), density=False
+    )
+
+    # Timeline: binary array with 1s at error positions
+    timeline = np.zeros(analysis.total_bits, dtype=np.float64)
+    timeline[analysis.error_positions] = 1.0
+
+    return {
+        "histogram_counts": counts.astype(np.float64),
+        "histogram_edges": edges.astype(np.float64),
+        "timeline": timeline,
+    }