oscura 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

oscura/{exploratory → jupyter/exploratory}/__init__.py

@@ -10,18 +10,18 @@ legacy hardware, and corrupted or noisy data.
 - UNKNOWN-005: Reverse Engineering Workflow
 """
 
-from oscura.exploratory.error_recovery import (
+from oscura.jupyter.exploratory.error_recovery import (
     ErrorContext,
     partial_decode,
     recover_corrupted_data,
     retry_with_adjustment,
 )
-from oscura.exploratory.fuzzy import (
+from oscura.jupyter.exploratory.fuzzy import (
     fuzzy_pattern_match,
     fuzzy_protocol_detect,
     fuzzy_timing_match,
 )
-from oscura.exploratory.fuzzy_advanced import (
+from oscura.jupyter.exploratory.fuzzy_advanced import (
     AlignmentResult,
     PositionAnalysis,
     VariantCharacterization,
@@ -30,33 +30,33 @@ from oscura.exploratory.fuzzy_advanced import (
     characterize_variants,
     compute_conservation_scores,
 )
-from oscura.exploratory.legacy import (
+from oscura.jupyter.exploratory.legacy import (
     assess_signal_quality,
     characterize_test_points,
     cross_correlate_multi_reference,
     detect_logic_families_multi_channel,
 )
-from oscura.exploratory.parse import (
+from oscura.jupyter.exploratory.parse import (
     DecodedFrame,
     ErrorTolerance,
     TimestampCorrection,
     correct_timestamp_jitter,
     decode_with_error_tolerance,
 )
-from oscura.exploratory.recovery import (
+from oscura.jupyter.exploratory.recovery import (
     ErrorAnalysis,
     ErrorPattern,
     analyze_bit_errors,
     generate_error_visualization_data,
 )
-from oscura.exploratory.sync import (
+from oscura.jupyter.exploratory.sync import (
     PacketParseResult,
     RecoveryStrategy,
     SyncMatch,
     fuzzy_sync_search,
     parse_variable_length_packets,
 )
-from oscura.exploratory.unknown import (
+from oscura.jupyter.exploratory.unknown import (
     analyze_pattern_frequency,
     characterize_unknown_signal,
     detect_binary_fields,

oscura/{exploratory → jupyter/exploratory}/error_recovery.py

@@ -5,7 +5,7 @@ noisy, or incomplete signal data.
 
 
 Example:
-    >>> from oscura.exploratory.error_recovery import recover_corrupted_data
+    >>> from oscura.jupyter.exploratory.error_recovery import recover_corrupted_data
     >>> recovered, stats = recover_corrupted_data(trace)
     >>> print(f"Recovered {stats.recovered_samples} samples")
 """
@@ -83,7 +83,64 @@ def recover_corrupted_data(
     data = trace.data.copy()
     n = len(data)
 
-    # Detect corrupted samples using statistical outlier detection
+    # Detect corrupted samples
+    corrupted_mask, corrupted_indices = _detect_corrupted_samples(data, corruption_threshold)
+    n_corrupted = len(corrupted_indices)
+
+    if n_corrupted == 0:
+        return trace, RecoveryStats(
+            total_samples=n,
+            corrupted_samples=0,
+            recovered_samples=0,
+            unrecoverable_samples=0,
+            recovery_method="none",
+            confidence=1.0,
+        )
+
+    # Group corrupted samples into contiguous regions (gaps)
+    gaps = _find_corruption_gaps(corrupted_indices)
+
+    # Attempt recovery for each gap
+    recovered, unrecoverable = _recover_gaps(
+        data, gaps, corrupted_mask, n, recovery_method, max_gap_samples
+    )
+
+    # Create recovered trace
+    recovered_trace = WaveformTrace(
+        data=data,
+        metadata=trace.metadata,
+    )
+
+    # Calculate confidence
+    recovery_ratio = recovered / max(n_corrupted, 1)
+    gap_sizes = [end - start + 1 for start, end in gaps]
+    avg_gap_size = np.mean(gap_sizes) if gap_sizes else 0
+    confidence = recovery_ratio * (1 - avg_gap_size / max_gap_samples)
+
+    confidence_clamped: float = float(max(0.0, min(1.0, confidence)))
+    return recovered_trace, RecoveryStats(
+        total_samples=n,
+        corrupted_samples=n_corrupted,
+        recovered_samples=recovered,
+        unrecoverable_samples=unrecoverable,
+        recovery_method=recovery_method,
+        confidence=confidence_clamped,
+    )
+
+
+def _detect_corrupted_samples(
+    data: NDArray[np.float64],
+    corruption_threshold: float,
+) -> tuple[NDArray[np.bool_], NDArray[np.int_]]:
+    """Detect corrupted samples using statistical outlier detection.
+
+    Args:
+        data: Waveform data array.
+        corruption_threshold: Threshold in standard deviations.
+
+    Returns:
+        Tuple of (corrupted_mask, corrupted_indices).
+    """
     # Filter out nan/inf for initial statistics calculation
     valid_mask = np.isfinite(data)
     valid_data = data[valid_mask] if np.any(valid_mask) else data
@@ -94,46 +151,70 @@ def recover_corrupted_data(
     if mad < 1e-10:
         mad = np.std(valid_data) if len(valid_data) > 0 else 1.0
 
-    # Z-score based on MAD
+    # Z-score based on MAD (Median Absolute Deviation)
     z_scores = np.abs(data - median) / (1.4826 * mad + 1e-10)
 
-    # Find corrupted samples
+    # Find corrupted samples (statistical outliers or invalid values)
     corrupted_mask = z_scores > corruption_threshold
-
-    # Also detect NaN and Inf
     corrupted_mask |= np.isnan(data)
     corrupted_mask |= np.isinf(data)
 
     corrupted_indices = np.where(corrupted_mask)[0]
-    n_corrupted = len(corrupted_indices)
 
-    if n_corrupted == 0:
-        return trace, RecoveryStats(
-            total_samples=n,
-            corrupted_samples=0,
-            recovered_samples=0,
-            unrecoverable_samples=0,
-            recovery_method="none",
-            confidence=1.0,
-        )
+    return corrupted_mask, corrupted_indices
+
+
+def _find_corruption_gaps(
+    corrupted_indices: NDArray[np.int_],
+) -> list[tuple[int, int]]:
+    """Group corrupted indices into contiguous regions (gaps).
+
+    Args:
+        corrupted_indices: Indices of corrupted samples.
+
+    Returns:
+        List of (start, end) tuples for each gap.
+    """
+    if len(corrupted_indices) == 0:
+        return []
 
-    # Group corrupted samples into contiguous regions
     gaps = []
-    if len(corrupted_indices) > 0:
-        gap_start = corrupted_indices[0]
-        gap_end = corrupted_indices[0]
+    gap_start = corrupted_indices[0]
+    gap_end = corrupted_indices[0]
 
-        for idx in corrupted_indices[1:]:
-            if idx == gap_end + 1:
-                gap_end = idx
-            else:
-                gaps.append((gap_start, gap_end))
-                gap_start = idx
-                gap_end = idx
+    for idx in corrupted_indices[1:]:
+        if idx == gap_end + 1:
+            gap_end = idx
+        else:
+            gaps.append((gap_start, gap_end))
+            gap_start = idx
+            gap_end = idx
 
-        gaps.append((gap_start, gap_end))
+    gaps.append((gap_start, gap_end))
+    return gaps
 
-    # Attempt recovery
+
+def _recover_gaps(
+    data: NDArray[np.float64],
+    gaps: list[tuple[int, int]],
+    corrupted_mask: NDArray[np.bool_],
+    n: int,
+    recovery_method: str,
+    max_gap_samples: int,
+) -> tuple[int, int]:
+    """Attempt recovery for each corruption gap.
+
+    Args:
+        data: Waveform data array (modified in place).
+        gaps: List of (start, end) tuples for gaps.
+        corrupted_mask: Boolean mask of corrupted samples.
+        n: Total number of samples.
+        recovery_method: Recovery method ('interpolate', 'median', 'zero').
+        max_gap_samples: Maximum recoverable gap size.
+
+    Returns:
+        Tuple of (recovered_count, unrecoverable_count).
+    """
     recovered = 0
     unrecoverable = 0
 
@@ -145,68 +226,93 @@ def recover_corrupted_data(
             continue
 
         if recovery_method == "interpolate":
-            # Linear interpolation from surrounding samples
-            left_idx = max(0, start - 1)
-            right_idx = min(n - 1, end + 1)
-
-            if left_idx < start and right_idx > end:
-                # Can interpolate
-                left_val = data[left_idx]
-                right_val = data[right_idx]
-                for i, idx in enumerate(range(start, end + 1)):
-                    t = (i + 1) / (gap_length + 1)
-                    data[idx] = left_val * (1 - t) + right_val * t
-                recovered += gap_length
-            else:
-                # Edge case - use nearest valid value
-                if left_idx >= start:
-                    data[start : end + 1] = data[right_idx]
-                else:
-                    data[start : end + 1] = data[left_idx]
-                recovered += gap_length
-
+            recovered += _recover_gap_interpolate(data, start, end, gap_length, n)
         elif recovery_method == "median":
-            # Replace with local median
-            window_start = max(0, start - 50)
-            window_end = min(n, end + 50)
-            window_data = data[window_start:window_end]
-            valid_data = window_data[~corrupted_mask[window_start:window_end]]
-
-            if len(valid_data) > 0:
-                fill_value = np.median(valid_data)
-                data[start : end + 1] = fill_value
+            result = _recover_gap_median(data, corrupted_mask, start, end, gap_length, n)
+            if result:
                 recovered += gap_length
             else:
                 unrecoverable += gap_length
-
         elif recovery_method == "zero":
-            # Replace with zero
             data[start : end + 1] = 0
             recovered += gap_length
-
         else:
             unrecoverable += gap_length
 
-    # Create recovered trace
-    recovered_trace = WaveformTrace(
-        data=data,
-        metadata=trace.metadata,
-    )
+    return recovered, unrecoverable
 
-    # Calculate confidence
-    recovery_ratio = recovered / max(n_corrupted, 1)
-    gap_sizes = [end - start + 1 for start, end in gaps]
-    avg_gap_size = np.mean(gap_sizes) if gap_sizes else 0
-    confidence = recovery_ratio * (1 - avg_gap_size / max_gap_samples)
 
-    return recovered_trace, RecoveryStats(
-        total_samples=n,
-        corrupted_samples=n_corrupted,
-        recovered_samples=recovered,
-        unrecoverable_samples=unrecoverable,
-        recovery_method=recovery_method,
-        confidence=max(0.0, min(1.0, confidence)),
-    )
+def _recover_gap_interpolate(
+    data: NDArray[np.float64],
+    start: int,
+    end: int,
+    gap_length: int,
+    n: int,
+) -> int:
+    """Recover gap using linear interpolation.
+
+    Args:
+        data: Waveform data array (modified in place).
+        start: Gap start index.
+        end: Gap end index.
+        gap_length: Gap length.
+        n: Total number of samples.
+
+    Returns:
+        Number of samples recovered.
+    """
+    left_idx = max(0, start - 1)
+    right_idx = min(n - 1, end + 1)
+
+    if left_idx < start and right_idx > end:
+        # Can interpolate between valid samples
+        left_val = data[left_idx]
+        right_val = data[right_idx]
+        for i, idx in enumerate(range(start, end + 1)):
+            t = (i + 1) / (gap_length + 1)
+            data[idx] = left_val * (1 - t) + right_val * t
+    else:
+        # Edge case - use nearest valid value
+        if left_idx >= start:
+            data[start : end + 1] = data[right_idx]
+        else:
+            data[start : end + 1] = data[left_idx]
+
+    return gap_length
+
+
+def _recover_gap_median(
+    data: NDArray[np.float64],
+    corrupted_mask: NDArray[np.bool_],
+    start: int,
+    end: int,
+    gap_length: int,
+    n: int,
+) -> bool:
+    """Recover gap using local median value.
+
+    Args:
+        data: Waveform data array (modified in place).
+        corrupted_mask: Boolean mask of corrupted samples.
+        start: Gap start index.
+        end: Gap end index.
+        gap_length: Gap length.
+        n: Total number of samples.
+
+    Returns:
+        True if recovery succeeded, False otherwise.
+    """
+    window_start = max(0, start - 50)
+    window_end = min(n, end + 50)
+    window_data = data[window_start:window_end]
+    valid_data = window_data[~corrupted_mask[window_start:window_end]]
+
+    if len(valid_data) > 0:
+        fill_value = np.median(valid_data)
+        data[start : end + 1] = fill_value
+        return True
+    else:
+        return False
 
 
 @dataclass
@@ -358,73 +464,20 @@ def partial_decode(
     References:
         ERROR-003: Partial Decode Support
     """
-    data = trace.data
-    n = len(data)
-
-    complete_packets: list[dict[str, Any]] = []
-    partial_packets: list[dict[str, Any]] = []
-    error_regions: list[dict[str, Any]] = []
-
-    total_samples = 0
-    decoded_samples = 0
-
-    # Try to decode entire trace first
-    try:
-        full_result = decode_func(trace)
-        if full_result:
-            complete_packets.extend(full_result)
-            decoded_samples = n
-            total_samples = n
-    except Exception as e:
-        logger.info("Full decode failed, falling back to segment decode: %s", e)
-        # Fall back to segment-by-segment decode
-        for start in range(0, n, segment_size):
-            end = min(start + segment_size, n)
-            segment_data = data[start:end]
-
-            # Create segment trace
-            segment_trace = WaveformTrace(
-                data=segment_data,
-                metadata=trace.metadata,
-            )
+    n = len(trace.data)
 
-            total_samples += len(segment_data)
+    # Try full decode first
+    full_decode_result = _try_full_decode(trace, decode_func, n)
+    if full_decode_result is not None:
+        return full_decode_result
 
-            try:
-                segment_result = decode_func(segment_trace)
-
-                if segment_result:
-                    # Adjust timestamps
-                    for packet in segment_result:
-                        if "timestamp" in packet:
-                            packet["timestamp"] += start / trace.metadata.sample_rate
-                        if "sample" in packet:
-                            packet["sample"] += start
-
-                    # Check if segment is valid
-                    valid_ratio = len(segment_result) / max(len(segment_data) / 100, 1)
-
-                    if valid_ratio >= min_valid_ratio:
-                        complete_packets.extend(segment_result)
-                        decoded_samples += len(segment_data)
-                    else:
-                        partial_packets.extend(segment_result)
-                        decoded_samples += len(segment_data) // 2
-
-            except Exception as e:
-                logger.debug("Segment decode failed at sample %d: %s", start, e)
-                error_regions.append(
-                    {
-                        "start_sample": start,
-                        "end_sample": end,
-                        "error": str(e),
-                    }
-                )
+    # Fall back to segment decode
+    complete_packets, partial_packets, error_regions, decoded_samples, total_samples = (
+        _segment_decode(trace, decode_func, segment_size, min_valid_ratio, n)
+    )
 
     # Calculate statistics
     decode_rate = decoded_samples / max(total_samples, 1)
-
-    # Calculate confidence
     error_ratio = len(error_regions) / max((n // segment_size), 1)
     confidence = decode_rate * (1 - error_ratio)
 
@@ -437,6 +490,110 @@ def partial_decode(
     )
 
 
+def _try_full_decode(
+    trace: WaveformTrace,
+    decode_func: Callable[[WaveformTrace], list[dict[str, Any]]],
+    n: int,
+) -> PartialDecodeResult | None:
+    """Try to decode entire trace at once.
+
+    Args:
+        trace: Trace to decode.
+        decode_func: Protocol decode function.
+        n: Number of samples.
+
+    Returns:
+        PartialDecodeResult if successful, None if failed.
+    """
+    try:
+        full_result = decode_func(trace)
+        if full_result:
+            return PartialDecodeResult(
+                complete_packets=full_result,
+                partial_packets=[],
+                error_regions=[],
+                decode_rate=1.0,
+                confidence=1.0,
+            )
+    except Exception as e:
+        logger.info("Full decode failed, falling back to segment decode: %s", e)
+
+    return None
+
+
+def _segment_decode(
+    trace: WaveformTrace,
+    decode_func: Callable[[WaveformTrace], list[dict[str, Any]]],
+    segment_size: int,
+    min_valid_ratio: float,
+    n: int,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]], list[dict[str, Any]], int, int]:
+    """Decode trace segment by segment.
+
+    Args:
+        trace: Trace to decode.
+        decode_func: Protocol decode function.
+        segment_size: Size of segments to try independently.
+        min_valid_ratio: Minimum valid ratio to accept segment.
+        n: Number of samples.
+
+    Returns:
+        Tuple of (complete_packets, partial_packets, error_regions, decoded_samples, total_samples).
+    """
+    complete_packets: list[dict[str, Any]] = []
+    partial_packets: list[dict[str, Any]] = []
+    error_regions: list[dict[str, Any]] = []
+    total_samples = 0
+    decoded_samples = 0
+
+    for start in range(0, n, segment_size):
+        end = min(start + segment_size, n)
+        segment_data = trace.data[start:end]
+
+        segment_trace = WaveformTrace(data=segment_data, metadata=trace.metadata)
+        total_samples += len(segment_data)
+
+        try:
+            segment_result = decode_func(segment_trace)
+
+            if segment_result:
+                _adjust_packet_timestamps(segment_result, start, trace.metadata.sample_rate)
+
+                # Classify segment as complete or partial
+                valid_ratio = len(segment_result) / max(len(segment_data) / 100, 1)
+                if valid_ratio >= min_valid_ratio:
+                    complete_packets.extend(segment_result)
+                    decoded_samples += len(segment_data)
+                else:
+                    partial_packets.extend(segment_result)
+                    decoded_samples += len(segment_data) // 2
+
+        except Exception as e:
+            logger.debug("Segment decode failed at sample %d: %s", start, e)
+            error_regions.append({"start_sample": start, "end_sample": end, "error": str(e)})
+
+    return complete_packets, partial_packets, error_regions, decoded_samples, total_samples
+
+
+def _adjust_packet_timestamps(
+    packets: list[dict[str, Any]],
+    start_sample: int,
+    sample_rate: float,
+) -> None:
+    """Adjust packet timestamps to account for segment offset.
+
+    Args:
+        packets: List of decoded packets to adjust.
+        start_sample: Start sample of this segment.
+        sample_rate: Sample rate in Hz.
+    """
+    for packet in packets:
+        if "timestamp" in packet:
+            packet["timestamp"] += start_sample / sample_rate
+        if "sample" in packet:
+            packet["sample"] += start_sample
+
+
 @dataclass
 class ErrorContext:
     """Preserved error context for debugging.