oscura 0.5.0__py3-none-any.whl → 0.6.0__py3-none-any.whl

oscura/utils/memory.py

@@ -245,86 +245,149 @@ def estimate_memory(
         >>> estimate = estimate_memory('fft', samples=1e9, nfft=8192)
         >>> print(f"Required: {estimate.total / 1e9:.2f} GB")
     """
-    # Bytes per element
     bytes_per_sample = 4 if dtype == "float32" else 8
-
     samples = int(samples or 0)
 
-    # Calculate based on operation
-    if operation == "fft":
-        nfft = nfft or _next_power_of_2(samples)
-        data_mem = samples * bytes_per_sample * channels
-        # FFT needs complex output (2x) plus work buffer
-        intermediate_mem = nfft * bytes_per_sample * 2 * 2  # complex, work buffer
-        output_mem = (nfft // 2 + 1) * bytes_per_sample * 2 * channels  # complex output
+    # Dispatch to operation-specific estimator (returns computed params)
+    data_mem, intermediate_mem, output_mem, computed_params = _estimate_for_operation(
+        operation, samples, bytes_per_sample, channels, nfft, nperseg, noverlap, kwargs
+    )
 
-    elif operation == "psd":
-        nperseg = nperseg or 256
-        nfft = nfft or nperseg
-        data_mem = samples * bytes_per_sample * channels
-        # Welch needs segment buffer plus FFT work
-        intermediate_mem = nperseg * bytes_per_sample * 2 + nfft * bytes_per_sample * 2
-        output_mem = (nfft // 2 + 1) * bytes_per_sample * channels
+    total_mem = data_mem + intermediate_mem + output_mem
 
-    elif operation == "spectrogram":
-        nperseg = nperseg or 256
-        noverlap = noverlap or nperseg // 2
-        nfft = nfft or nperseg
-        hop = nperseg - noverlap
-        num_segments = max(1, (samples - noverlap) // hop)
+    # Merge provided and computed parameters
+    all_params = {
+        "samples": samples,
+        "dtype": dtype,
+        "channels": channels,
+        **computed_params,  # Include computed defaults
+        **kwargs,
+    }
 
-        data_mem = samples * bytes_per_sample * channels
-        # STFT needs segment buffer
-        intermediate_mem = nperseg * bytes_per_sample * 2 + nfft * bytes_per_sample * 2
-        # Output: (nfft//2+1) frequencies x num_segments times
-        output_mem = (nfft // 2 + 1) * num_segments * bytes_per_sample * 2 * channels
+    return MemoryEstimate(
+        data=data_mem,
+        intermediate=intermediate_mem,
+        output=output_mem,
+        total=total_mem,
+        operation=operation,
+        parameters=all_params,
+    )
 
-    elif operation == "eye_diagram":
-        samples_per_ui = kwargs.get("samples_per_ui", 100)
-        num_uis = kwargs.get("num_uis", 1000)
-        data_mem = samples * bytes_per_sample * channels
-        # Eye diagram accumulates traces
-        intermediate_mem = samples_per_ui * num_uis * bytes_per_sample
-        output_mem = samples_per_ui * num_uis * bytes_per_sample
 
-    elif operation == "correlate":
-        data_mem = samples * bytes_per_sample * 2 * channels  # Two signals
-        # FFT-based correlation
-        nfft = _next_power_of_2(samples * 2)
-        intermediate_mem = nfft * bytes_per_sample * 2 * 2  # Two FFTs
-        output_mem = (samples * 2 - 1) * bytes_per_sample * channels
+def _estimate_for_operation(
+    operation: str,
+    samples: int,
+    bytes_per_sample: int,
+    channels: int,
+    nfft: int | None,
+    nperseg: int | None,
+    noverlap: int | None,
+    kwargs: dict[str, Any],
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for specific operation.
 
+    Returns:
+        Tuple of (data_mem, intermediate_mem, output_mem, computed_params)
+    """
+    if operation == "fft":
+        return _estimate_fft(samples, bytes_per_sample, channels, nfft)
+    elif operation == "psd":
+        return _estimate_psd(samples, bytes_per_sample, channels, nfft, nperseg)
+    elif operation == "spectrogram":
+        return _estimate_spectrogram(samples, bytes_per_sample, channels, nfft, nperseg, noverlap)
+    elif operation == "eye_diagram":
+        return _estimate_eye_diagram(samples, bytes_per_sample, channels, kwargs)
+    elif operation == "correlate":
+        return _estimate_correlate(samples, bytes_per_sample, channels)
     elif operation == "filter":
-        filter_order = kwargs.get("filter_order", 8)
-        data_mem = samples * bytes_per_sample * channels
-        # Filter state and buffer
-        intermediate_mem = (filter_order + samples) * bytes_per_sample
-        output_mem = samples * bytes_per_sample * channels
-
+        return _estimate_filter(samples, bytes_per_sample, channels, kwargs)
     else:
         # Generic estimate
         data_mem = samples * bytes_per_sample * channels
-        intermediate_mem = samples * bytes_per_sample
-        output_mem = samples * bytes_per_sample * channels
+        return data_mem, data_mem, data_mem, {}
+
+
+def _estimate_fft(
+    samples: int, bytes_per_sample: int, channels: int, nfft: int | None
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for FFT operation."""
+    nfft = nfft or _next_power_of_2(samples)
+    data_mem = samples * bytes_per_sample * channels
+    intermediate_mem = nfft * bytes_per_sample * 2 * 2  # complex, work buffer
+    output_mem = (nfft // 2 + 1) * bytes_per_sample * 2 * channels
+    return data_mem, intermediate_mem, output_mem, {"nfft": nfft}
+
+
+def _estimate_psd(
+    samples: int, bytes_per_sample: int, channels: int, nfft: int | None, nperseg: int | None
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for PSD (Welch) operation."""
+    nperseg = nperseg or 256
+    nfft = nfft or nperseg
+    data_mem = samples * bytes_per_sample * channels
+    intermediate_mem = nperseg * bytes_per_sample * 2 + nfft * bytes_per_sample * 2
+    output_mem = (nfft // 2 + 1) * bytes_per_sample * channels
+    return data_mem, intermediate_mem, output_mem, {"nfft": nfft, "nperseg": nperseg}
+
+
+def _estimate_spectrogram(
+    samples: int,
+    bytes_per_sample: int,
+    channels: int,
+    nfft: int | None,
+    nperseg: int | None,
+    noverlap: int | None,
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for spectrogram (STFT) operation."""
+    nperseg = nperseg or 256
+    noverlap = noverlap or nperseg // 2
+    nfft = nfft or nperseg
+    hop = nperseg - noverlap
+    num_segments = max(1, (samples - noverlap) // hop)
+
+    data_mem = samples * bytes_per_sample * channels
+    intermediate_mem = nperseg * bytes_per_sample * 2 + nfft * bytes_per_sample * 2
+    output_mem = (nfft // 2 + 1) * num_segments * bytes_per_sample * 2 * channels
+    return (
+        data_mem,
+        intermediate_mem,
+        output_mem,
+        {"nfft": nfft, "nperseg": nperseg, "noverlap": noverlap},
+    )
 
-    total_mem = data_mem + intermediate_mem + output_mem
 
-    return MemoryEstimate(
-        data=data_mem,
-        intermediate=intermediate_mem,
-        output=output_mem,
-        total=total_mem,
-        operation=operation,
-        parameters={
-            "samples": samples,
-            "nfft": nfft,
-            "nperseg": nperseg,
-            "noverlap": noverlap,
-            "dtype": dtype,
-            "channels": channels,
-            **kwargs,
-        },
-    )
+def _estimate_eye_diagram(
+    samples: int, bytes_per_sample: int, channels: int, kwargs: dict[str, Any]
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for eye diagram generation."""
+    samples_per_ui = kwargs.get("samples_per_ui", 100)
+    num_uis = kwargs.get("num_uis", 1000)
+    data_mem = samples * bytes_per_sample * channels
+    intermediate_mem = samples_per_ui * num_uis * bytes_per_sample
+    output_mem = samples_per_ui * num_uis * bytes_per_sample
+    return data_mem, intermediate_mem, output_mem, {}
+
+
+def _estimate_correlate(
+    samples: int, bytes_per_sample: int, channels: int
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for correlation operation."""
+    data_mem = samples * bytes_per_sample * 2 * channels  # Two signals
+    nfft = _next_power_of_2(samples * 2)
+    intermediate_mem = nfft * bytes_per_sample * 2 * 2  # Two FFTs
+    output_mem = (samples * 2 - 1) * bytes_per_sample * channels
+    return data_mem, intermediate_mem, output_mem, {"nfft": nfft}
+
+
+def _estimate_filter(
+    samples: int, bytes_per_sample: int, channels: int, kwargs: dict[str, Any]
+) -> tuple[int, int, int, dict[str, Any]]:
+    """Estimate memory for filter operation."""
+    filter_order = kwargs.get("filter_order", 8)
+    data_mem = samples * bytes_per_sample * channels
+    intermediate_mem = (filter_order + samples) * bytes_per_sample
+    output_mem = samples * bytes_per_sample * channels
+    return data_mem, intermediate_mem, output_mem, {}
 
 
 def check_memory_available(
@@ -552,7 +615,7 @@ def configure_memory(
         >>> configure_memory(max_memory="4GB", warn_threshold=0.7, critical_threshold=0.9)
         >>> configure_memory(auto_degrade=True)
     """
-    global _memory_config  # noqa: PLW0602
+    global _memory_config
 
     if max_memory is not None:
         if isinstance(max_memory, str):

oscura/utils/memory_advanced.py

@@ -10,9 +10,11 @@ from __future__ import annotations
 import contextlib
 import gc
 import hashlib
+import hmac
 import json
 import logging
 import pickle
+import secrets
 import tempfile
 import threading
 import time
@@ -24,6 +26,8 @@ from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 import numpy as np
 
+from oscura.core.exceptions import SecurityError
+
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
@@ -573,7 +577,7 @@ class AdaptiveMeasurementSelector:
     """
 
     # Default size thresholds (in samples)
-    THRESHOLDS = {  # noqa: RUF012
+    THRESHOLDS = {
         "eye_diagram": 1e8,  # 100M samples
         "spectrogram": 5e8,  # 500M samples
         "full_correlation": 1e9,  # 1B samples
@@ -730,13 +734,21 @@ class CacheInvalidationStrategy:
         self._misses = 0
 
     def _compute_hash(self, data: Any) -> str:
-        """Compute hash of data for comparison."""
+        """Compute hash of data for comparison.
+
+        Note:
+            Uses MD5 for cache invalidation checksums only (not for security).
+            MD5 is appropriate here for non-cryptographic data comparison.
+        """
         if isinstance(data, np.ndarray):
-            return hashlib.md5(data.tobytes()[:1024]).hexdigest()
+            # Sample first 1KB for performance (cache invalidation only, not security)
+            return hashlib.md5(data.tobytes()[:1024], usedforsecurity=False).hexdigest()
         elif isinstance(data, dict | list):
-            return hashlib.md5(json.dumps(data, sort_keys=True).encode()).hexdigest()
+            return hashlib.md5(
+                json.dumps(data, sort_keys=True).encode(), usedforsecurity=False
+            ).hexdigest()
         else:
-            return hashlib.md5(str(data).encode()).hexdigest()
+            return hashlib.md5(str(data).encode(), usedforsecurity=False).hexdigest()
 
     def get(
         self,
@@ -938,6 +950,9 @@ class DiskCache:
         self._memory_used = 0
         self._lock = threading.Lock()
 
+        # Security: HMAC signing key for cache integrity (SEC-003 fix)
+        self._cache_key = self._load_or_create_cache_key()
+
     def _get_cache_path(self, key: str) -> Path:
         """Get cache file path for key."""
         key_hash = hashlib.sha256(key.encode()).hexdigest()[:16]
@@ -946,10 +961,44 @@ class DiskCache:
     def _estimate_size(self, value: Any) -> int:
         """Estimate memory size of value."""
         if isinstance(value, np.ndarray):
-            return value.nbytes  # type: ignore[no-any-return]
+            return value.nbytes
         else:
             return len(pickle.dumps(value))
 
+    def _load_or_create_cache_key(self) -> bytes:
+        """Load or create HMAC signing key for cache integrity.
+
+        Returns:
+            256-bit signing key.
+
+        Security:
+            SEC-003 fix: Protects cached pickle files from tampering.
+            Key is persistent per cache directory and stored with 0o600 permissions.
+            Each cache directory has its own unique key.
+
+        References:
+            https://owasp.org/www-project-top-ten/
+        """
+        key_file = self._cache_dir / ".cache_key"
+
+        # Load existing key
+        if key_file.exists():
+            with open(key_file, "rb") as f:
+                return f.read()
+
+        # Create new 256-bit key
+        key = secrets.token_bytes(32)
+
+        # Save with restrictive permissions
+        with open(key_file, "wb") as f:
+            f.write(key)
+
+        # Set owner read/write only (0o600)
+        key_file.chmod(0o600)
+
+        logger.info(f"Created new cache signing key: {key_file}")
+        return key
+
     def get(self, key: str) -> tuple[Any, bool]:
         """Get value from cache.
 
@@ -976,7 +1025,23 @@ class DiskCache:
 
             try:
                 with open(cache_path, "rb") as f:
-                    value = pickle.load(f)
+                    signature = f.read(32)  # SHA256 = 32 bytes
+                    data = f.read()
+
+                # Verify HMAC signature (SEC-003 fix)
+                expected_signature = hmac.new(self._cache_key, data, hashlib.sha256).digest()
+
+                if not hmac.compare_digest(signature, expected_signature):
+                    logger.error(f"Cache integrity check failed for {key}")
+                    # Delete corrupted cache file
+                    cache_path.unlink()
+                    raise SecurityError(
+                        f"Cache file integrity verification failed: {key}. "
+                        "File may have been tampered with and has been removed."
+                    )
+
+                # Deserialize only after HMAC verification
+                value = pickle.loads(data)
 
                 # Promote to memory cache if space
                 size = self._estimate_size(value)
@@ -984,6 +1049,9 @@ class DiskCache:
                     self._add_to_memory(key, value, size)
 
                 return value, True
+
+            except SecurityError:
+                raise  # Re-raise security errors
             except Exception as e:
                 logger.warning(f"Failed to load cache: {e}")
                 return None, False
@@ -1004,14 +1072,29 @@ class DiskCache:
             self._memory_used += size
 
     def _write_to_disk(self, key: str, value: Any) -> None:
-        """Write value to disk cache."""
+        """Write value to disk cache with HMAC signature.
+
+        Security:
+            SEC-003 fix: Writes HMAC-SHA256 signature + pickled data.
+            Format: [32 bytes signature][pickled data]
+            Signature computed over pickled data using self._cache_key.
+        """
         # Check disk space
         self._cleanup_disk()
 
         cache_path = self._get_cache_path(key)
         try:
+            # Serialize value
+            data = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)
+
+            # Compute HMAC-SHA256 signature
+            signature = hmac.new(self._cache_key, data, hashlib.sha256).digest()
+
+            # Write signature + data
             with open(cache_path, "wb") as f:
-                pickle.dump(value, f)
+                f.write(signature)  # First 32 bytes
+                f.write(data)  # Rest is pickled data
+
         except Exception as e:
             logger.warning(f"Failed to write cache: {e}")
 

oscura/utils/memory_extensions.py

@@ -94,7 +94,7 @@ class ArrayManager(ResourceManager):
     """
 
     def __init__(self, array: NDArray[Any]) -> None:
-        super().__init__(array, cleanup_func=lambda x: None)  # noqa: ARG005
+        super().__init__(array, cleanup_func=lambda x: None)
 
 
 # =============================================================================
@@ -252,7 +252,7 @@ def get_result_cache() -> LRUCache[Any]:
     global _result_cache
     if _result_cache is None:
         # Default: 2 GB cache
-        max_cache_size = int(os.environ.get("TK_CACHE_SIZE", 2 * 1024 * 1024 * 1024))  # noqa: PLW1508
+        max_cache_size = int(os.environ.get("TK_CACHE_SIZE", 2 * 1024 * 1024 * 1024))
         _result_cache = LRUCache(max_memory_bytes=max_cache_size)
     return _result_cache
 
@@ -286,6 +286,10 @@ def show_cache_stats() -> dict[str, int | float]:
 def cache_key(*args: Any, **kwargs: Any) -> str:
     """Generate cache key from arguments.
 
+    Note:
+        Uses MD5 for cache key generation only (not for security).
+        MD5 is appropriate here for non-cryptographic checksums.
+
     Args:
         *args: Positional arguments.
         **kwargs: Keyword arguments.
@@ -302,8 +306,8 @@ def cache_key(*args: Any, **kwargs: Any) -> str:
     parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))
     key_str = "|".join(parts)
 
-    # Hash for consistent key
-    return hashlib.md5(key_str.encode()).hexdigest()
+    # Hash for consistent key (MD5 used for cache keys only, not security)
+    return hashlib.md5(key_str.encode(), usedforsecurity=False).hexdigest()
 
 
 # =============================================================================
@@ -344,9 +348,7 @@ def load_hdf5_lazy(
     try:
         import h5py
     except ImportError:
-        raise ImportError(  # noqa: B904
-            "h5py required for lazy HDF5 loading. Install with: pip install h5py"
-        )
+        raise ImportError("h5py required for lazy HDF5 loading. Install with: pip install h5py")
 
     from pathlib import Path
 
@@ -404,7 +406,7 @@ class LazyHDF5Array:
         try:
             import h5py
         except ImportError:
-            raise ImportError("h5py required. Install with: pip install h5py")  # noqa: B904
+            raise ImportError("h5py required. Install with: pip install h5py")
 
         self._file = h5py.File(self._file_path, "r")
         self._dataset = self._file[self._dataset_path]  # type: ignore[index]

oscura/{optimization → utils/optimization}/__init__.py

@@ -4,7 +4,7 @@ This module provides grid search and randomized search for finding optimal
 analysis parameters.
 """
 
-from oscura.optimization.search import (
+from oscura.utils.optimization.search import (
     GridSearchCV,
     RandomizedSearchCV,
     ScoringFunction,

oscura/{optimization → utils/optimization}/search.py

@@ -99,7 +99,7 @@ class GridSearchCV:
     optimal configuration.
 
     Example:
-        >>> from oscura.optimization.search import GridSearchCV
+        >>> from oscura.utils.optimization.search import GridSearchCV
         >>> param_grid = {
         ...     'cutoff': [1e5, 5e5, 1e6],
         ...     'order': [2, 4, 6]
@@ -373,7 +373,7 @@ class RandomizedSearchCV:
     exhaustively evaluating all combinations.
 
     Example:
-        >>> from oscura.optimization.search import RandomizedSearchCV
+        >>> from oscura.utils.optimization.search import RandomizedSearchCV
         >>> import numpy as np
         >>> param_distributions = {
         ...     'cutoff': lambda: np.random.uniform(1e5, 1e7),

oscura/utils/performance/__init__.py

@@ -0,0 +1,58 @@
+"""Performance optimization utilities for Oscura.
+
+This package provides memory optimization, streaming analysis, parallel processing,
+performance profiling, caching, and monitoring capabilities for processing large
+signal files efficiently.
+"""
+
+from oscura.utils.performance.caching import (
+    CacheBackend,
+    CacheEntry,
+    CacheManager,
+    CachePolicy,
+    CacheStats,
+    cache,
+    get_global_cache,
+)
+from oscura.utils.performance.memory_optimizer import (
+    ChunkingConfig,
+    ChunkingStrategy,
+    MemoryOptimizer,
+    MemoryStats,
+    StreamProcessor,
+)
+from oscura.utils.performance.parallel import (
+    ParallelConfig,
+    ParallelProcessor,
+    ParallelResult,
+    WorkerStats,
+)
+from oscura.utils.performance.profiling import (
+    FunctionStats,
+    PerformanceProfiler,
+    ProfilingMode,
+    ProfilingResult,
+)
+
+__all__ = [
+    "CacheBackend",
+    "CacheEntry",
+    "CacheManager",
+    "CachePolicy",
+    "CacheStats",
+    "ChunkingConfig",
+    "ChunkingStrategy",
+    "FunctionStats",
+    "MemoryOptimizer",
+    "MemoryStats",
+    "ParallelConfig",
+    "ParallelProcessor",
+    "ParallelResult",
+    "PerformanceProfiler",
+    "ProfilingMode",
+    "ProfilingResult",
+    "StreamProcessor",
+    "WorkerStats",
+    "cache",
+    "get_global_cache",
+]