oscura 0.5.1__py3-none-any.whl → 0.7.0__py3-none-any.whl

oscura/core/cache.py

@@ -21,7 +21,10 @@ from __future__ import annotations
 
 import contextlib
 import hashlib
+import hmac
+import logging
 import pickle
+import secrets
 import tempfile
 import threading
 import time
@@ -32,10 +35,14 @@ from typing import TYPE_CHECKING, Any, TypeVar
 
 import numpy as np
 
+from oscura.core.exceptions import SecurityError
+
 if TYPE_CHECKING:
     from collections.abc import Callable
 
 
+logger = logging.getLogger(__name__)
+
 T = TypeVar("T")
 
 
@@ -180,6 +187,9 @@ class OscuraCache:
         self._disk_spills = 0
         self._current_memory = 0
 
+        # Security: HMAC signing key for cache integrity (SEC-003 fix)
+        self._cache_key = self._load_or_create_cache_key()
+
     def __enter__(self) -> OscuraCache:
         """Enter context."""
         return self
@@ -227,12 +237,21 @@ class OscuraCache:
 
             # Load from disk if needed
             if not entry.in_memory:
-                entry.value = self._load_from_disk(entry.disk_path)  # type: ignore[arg-type]
-                entry.in_memory = True
-                self._current_memory += entry.size_bytes
-
-                # Check if we need to spill to make room
-                self._ensure_memory_limit()
+                try:
+                    entry.value = self._load_from_disk(entry.disk_path)  # type: ignore[arg-type]
+                    entry.in_memory = True
+                    self._current_memory += entry.size_bytes
+
+                    # Check if we need to spill to make room
+                    self._ensure_memory_limit()
+                except SecurityError:
+                    # Re-raise security errors (tampered data)
+                    raise
+                except (OSError, pickle.UnpicklingError):
+                    # Remove corrupted entry from cache for non-security errors
+                    del self._cache[key]
+                    self._misses += 1
+                    return None
 
             return entry.value
 
@@ -441,8 +460,7 @@ class OscuraCache:
             self._evictions += 1
 
     def _spill_to_disk(self, key: str, value: Any) -> Path:
-        """Write value to disk.
-
+        """Write value to disk with HMAC signature.
 
         Args:
             key: Cache key.
@@ -450,29 +468,88 @@ class OscuraCache:
 
         Returns:
             Path to disk file.
+
+        Security:
+            SEC-003 fix: Writes HMAC-SHA256 signature + pickled data.
+            Format: [32 bytes signature][pickled data]
+            Signature computed over pickled data using self._cache_key.
+
+        References:
+            MEM-031: Persistent Cache (Disk-Based)
         """
         disk_path = self.cache_dir / f"{key}.pkl"
+
+        # Serialize data
+        data = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL)
+
+        # Compute HMAC-SHA256 signature
+        signature = hmac.new(self._cache_key, data, hashlib.sha256).digest()
+
+        # Write signature + data
         with open(disk_path, "wb") as f:
-            pickle.dump(value, f, protocol=pickle.HIGHEST_PROTOCOL)
+            f.write(signature)  # First 32 bytes
+            f.write(data)  # Rest is pickled data
+
         return disk_path
 
     def _load_from_disk(self, disk_path: Path) -> Any:
-        """Load value from disk.
-
+        """Load value from disk with HMAC verification.
 
         Args:
             disk_path: Path to disk file.
 
         Returns:
             Loaded value.
+
+        Raises:
+            SecurityError: If HMAC verification fails (tampered cache file).
+
+        Security:
+            SEC-003 fix: Verifies HMAC-SHA256 signature before unpickling.
+            Prevents code execution from tampered cache files.
+            Uses constant-time comparison (hmac.compare_digest).
+
+        References:
+            MEM-031: Persistent Cache (Disk-Based)
         """
-        with open(disk_path, "rb") as f:
-            return pickle.load(f)
+        try:
+            with open(disk_path, "rb") as f:
+                signature = f.read(32)  # SHA256 = 32 bytes
+                data = f.read()
+
+            # Check if file is too short (corrupted, not tampered)
+            if len(signature) < 32:
+                logger.warning(f"Cache file too short (corrupted): {disk_path.name}")
+                disk_path.unlink(missing_ok=True)
+                raise OSError(f"Cache file corrupted (too short): {disk_path.name}")
+
+            # Verify HMAC signature
+            expected_signature = hmac.new(self._cache_key, data, hashlib.sha256).digest()
+
+            if not hmac.compare_digest(signature, expected_signature):
+                logger.error(f"Cache integrity check failed for {disk_path.name}")
+                # Delete corrupted cache file
+                disk_path.unlink(missing_ok=True)
+                raise SecurityError(
+                    f"Cache file integrity verification failed: {disk_path.name}. "
+                    "File may have been tampered with and has been removed."
+                )
+
+            # Deserialize only after HMAC verification
+            return pickle.loads(data)
+
+        except SecurityError:
+            raise  # Re-raise security errors
+        except Exception as e:
+            logger.warning(f"Failed to load cache file {disk_path.name}: {e}")
+            # Clean up corrupted file
+            disk_path.unlink(missing_ok=True)
+            raise
 
     def _estimate_size(self, value: Any) -> int:
         """Estimate size of value in bytes."""
         if isinstance(value, np.ndarray):
-            return value.nbytes  # type: ignore[no-any-return]
+            return value.nbytes
         elif isinstance(value, list | tuple):
             return sum(self._estimate_size(item) for item in value)
         elif isinstance(value, dict):
@@ -488,7 +565,7 @@ class OscuraCache:
         """Convert object to hashable bytes."""
         if isinstance(obj, np.ndarray):
             # Use array bytes for hashing
-            return obj.tobytes()  # type: ignore[no-any-return]
+            return obj.tobytes()
         elif isinstance(obj, str | bytes):
             return obj.encode() if isinstance(obj, str) else obj
         elif isinstance(obj, int | float | bool):
@@ -512,6 +589,40 @@ class OscuraCache:
         else:
             return int(memory_str)
 
+    def _load_or_create_cache_key(self) -> bytes:
+        """Load or create HMAC signing key for cache integrity.
+
+        Returns:
+            256-bit signing key.
+
+        Security:
+            SEC-003 fix: Protects cached pickle files from tampering.
+            Key is persistent per cache directory and stored with 0o600 permissions.
+            Each cache directory has its own unique key.
+
+        References:
+            https://owasp.org/www-project-top-ten/
+        """
+        key_file = self.cache_dir / ".cache_key"
+
+        # Load existing key
+        if key_file.exists():
+            with open(key_file, "rb") as f:
+                return f.read()
+
+        # Create new 256-bit key
+        key = secrets.token_bytes(32)
+
+        # Save with restrictive permissions
+        with open(key_file, "wb") as f:
+            f.write(key)
+
+        # Set owner read/write only (0o600)
+        key_file.chmod(0o600)
+
+        logger.info(f"Created new cache signing key: {key_file}")
+        return key
+
 
 # Global cache instance
 _global_cache: OscuraCache | None = None

oscura/core/cancellation.py

@@ -260,7 +260,7 @@ class CancellationManager:
         except KeyboardInterrupt:
             self.cancel("Interrupted by user (Ctrl+C)")
             self._cleanup()
-            raise CancelledException(  # noqa: B904
+            raise CancelledException(
                 f"{name} interrupted by user",
                 partial_results=self._partial_results,
                 elapsed_time=time.time() - self._start_time,

oscura/{config → core/config}/__init__.py

@@ -10,23 +10,23 @@ This package provides:
 
 
 Example:
-    >>> from oscura.config import load_config, validate_config
+    >>> from oscura.core.config import load_config, validate_config
     >>> config = load_config("pipeline.yaml")
     >>> validate_config(config, schema="pipeline")
 """
 
-from oscura.config.defaults import (
+from oscura.core.config.defaults import (
     DEFAULT_CONFIG,
     get_effective_config,
     inject_defaults,
 )
-from oscura.config.loader import (
+from oscura.core.config.loader import (
     get_config_value,
     load_config,
     load_config_file,
     save_config,
 )
-from oscura.config.memory import (
+from oscura.core.config.memory import (
     MemoryConfiguration,
     configure_from_environment,
     enable_auto_degrade,
@@ -36,7 +36,7 @@ from oscura.config.memory import (
     set_memory_reserve,
     set_memory_thresholds,
 )
-from oscura.config.migration import (
+from oscura.core.config.migration import (
     Migration,
     MigrationFunction,
     SchemaMigration,
@@ -46,7 +46,7 @@ from oscura.config.migration import (
     migrate_config,
     register_migration,
 )
-from oscura.config.pipeline import (
+from oscura.core.config.pipeline import (
     Pipeline,
     PipelineDefinition,
     PipelineExecutionError,
@@ -57,7 +57,7 @@ from oscura.config.pipeline import (
     load_pipeline,
     resolve_includes,
 )
-from oscura.config.preferences import (
+from oscura.core.config.preferences import (
     DefaultsPreferences,
     EditorPreferences,
     ExportPreferences,
@@ -69,7 +69,7 @@ from oscura.config.preferences import (
     get_preferences_manager,
     save_preferences,
 )
-from oscura.config.protocol import (
+from oscura.core.config.protocol import (
     ProtocolCapabilities,
     ProtocolDefinition,
     ProtocolRegistry,
@@ -78,7 +78,7 @@ from oscura.config.protocol import (
     load_protocol,
     resolve_inheritance,
 )
-from oscura.config.schema import (
+from oscura.core.config.schema import (
     ConfigSchema,
     SchemaRegistry,
     ValidationError,
@@ -86,7 +86,13 @@ from oscura.config.schema import (
     register_schema,
     validate_against_schema,
 )
-from oscura.config.settings import (
+
+# Alias for backward compatibility with oscura.core.config module
+validate_config = validate_against_schema
+
+# Legacy imports for backward compatibility
+from oscura.core.config.legacy import SmartDefaults, _deep_merge
+from oscura.core.config.settings import (
     AnalysisSettings,
     CLIDefaults,
     OutputSettings,
@@ -97,7 +103,7 @@ from oscura.config.settings import (
     save_settings,
     set_settings,
 )
-from oscura.config.thresholds import (
+from oscura.core.config.thresholds import (
     LogicFamily,
     ThresholdProfile,
     ThresholdRegistry,
@@ -144,11 +150,13 @@ __all__ = [
     "SchemaMigration",
     "SchemaRegistry",
     "Settings",
+    "SmartDefaults",
     "ThresholdProfile",
     "ThresholdRegistry",
     "UserPreferences",
     "ValidationError",
     "VisualizationPreferences",
+    "_deep_merge",
     "configure_from_environment",
     "enable_auto_degrade",
     "get_config_value",
@@ -188,4 +196,5 @@ __all__ = [
     "set_memory_thresholds",
     "set_settings",
     "validate_against_schema",
+    "validate_config",
 ]

oscura/{config → core/config}/defaults.py

@@ -5,7 +5,7 @@ for injecting defaults into user configurations.
 
 
 Example:
-    >>> from oscura.config.defaults import inject_defaults
+    >>> from oscura.core.config.defaults import inject_defaults
     >>> config = {"name": "test"}
     >>> full_config = inject_defaults(config, "protocol")
 """

oscura/{config → core/config}/loader.py

@@ -5,7 +5,7 @@ with support for schema validation, default injection, and path resolution.
 
 
 Example:
-    >>> from oscura.config.loader import load_config_file
+    >>> from oscura.core.config.loader import load_config_file
     >>> config = load_config_file("pipeline.yaml", schema="pipeline")
 """
 
@@ -76,7 +76,7 @@ def load_config_file(
             try:
                 config = _load_json(path)
             except ConfigurationError:
-                raise ConfigurationError(  # noqa: B904
+                raise ConfigurationError(
                     f"Unsupported configuration format: {ext}",
                     config_key=str(path),
                     fix_hint="Use .yaml, .yml, or .json extension",
@@ -84,13 +84,13 @@ def load_config_file(
 
     # Validate against schema if requested
     if validate and schema is not None:
-        from oscura.config.schema import validate_against_schema
+        from oscura.core.config.schema import validate_against_schema
 
         validate_against_schema(config, schema)
 
     # Inject defaults if requested
     if inject_defaults and schema is not None:
-        from oscura.config.defaults import inject_defaults as do_inject
+        from oscura.core.config.defaults import inject_defaults as do_inject
 
         config = do_inject(config, schema)
 
@@ -215,7 +215,7 @@ def load_config(
         >>> config = load_config()  # Auto-find config
         >>> config = load_config("~/.oscura/config.yaml")
     """
-    from oscura.config.defaults import DEFAULT_CONFIG, deep_merge
+    from oscura.core.config.defaults import DEFAULT_CONFIG, deep_merge
 
     config: dict[str, Any] = {}
 
@@ -224,6 +224,8 @@ def load_config(
 
         config = copy.deepcopy(DEFAULT_CONFIG)
 
+    # Search for config files if no explicit path provided
+    # use_defaults flag only controls whether to merge with DEFAULT_CONFIG
     if config_path is None:
         # Search standard locations
         search_paths = [

oscura/{config → core/config}/memory.py

@@ -4,7 +4,7 @@ This module provides global memory limit configuration and settings.
 
 
 Example:
-    >>> from oscura.config.memory import set_memory_limit, get_memory_config
+    >>> from oscura.core.config.memory import set_memory_limit, get_memory_config
     >>> set_memory_limit("4GB")
     >>> config = get_memory_config()
     >>> print(f"Max memory: {config.max_memory / 1e9:.1f} GB")
@@ -90,7 +90,7 @@ def set_memory_limit(limit: int | str | None) -> None:
     Environment:
         Can also be set via TK_MAX_MEMORY environment variable.
     """
-    global _global_config  # noqa: PLW0602
+    global _global_config
 
     if limit is None:
         _global_config.max_memory = None
@@ -117,7 +117,7 @@ def set_memory_thresholds(
         >>> set_memory_thresholds(warn_threshold=0.7, critical_threshold=0.9)
         >>> set_memory_thresholds(critical_threshold=0.95)  # Keep warn unchanged
     """
-    global _global_config  # noqa: PLW0602
+    global _global_config
 
     if warn_threshold is not None:
         _global_config.warn_threshold = warn_threshold
@@ -139,7 +139,7 @@ def enable_auto_degrade(enabled: bool = True) -> None:
         >>> enable_auto_degrade(True)
         >>> # Operations will now auto-downsample if memory insufficient
     """
-    global _global_config  # noqa: PLW0602
+    global _global_config
     _global_config.auto_degrade = enabled
 
 
@@ -154,7 +154,7 @@ def set_memory_reserve(reserve: int | str) -> None:
         >>> set_memory_reserve("1GB")  # Reserve 1 GB for system
         >>> set_memory_reserve(512 * 1024**2)  # 512 MB
     """
-    global _global_config  # noqa: PLW0602
+    global _global_config
 
     if isinstance(reserve, str):
         _global_config.memory_reserve = _parse_memory_string(reserve)

oscura/{config → core/config}/migration.py

@@ -5,7 +5,7 @@ configuration files between schema versions while preserving user data.
 
 
 Example:
-    >>> from oscura.config.migration import migrate_config, register_migration
+    >>> from oscura.core.config.migration import migrate_config, register_migration
     >>> # Register a migration function
     >>> def migrate_1_0_to_1_1(config: dict) -> dict:
     ...     config['new_field'] = 'default_value'

oscura/{config → core/config}/pipeline.py

@@ -17,7 +17,7 @@ from typing import TYPE_CHECKING, Any
 
 import yaml
 
-from oscura.config.schema import validate_against_schema
+from oscura.core.config.schema import validate_against_schema
 from oscura.core.exceptions import ConfigurationError
 
 if TYPE_CHECKING:
@@ -690,7 +690,7 @@ def _substitute_variables(content: str, variables: dict[str, Any], max_depth: in
     pattern = re.compile(r"\$\{(\w+)\}")
     depth = 0
 
-    for depth in range(max_depth):  # noqa: B007
+    for depth in range(max_depth):
         prev_content = content
         substitutions_made = False
 
@@ -768,33 +768,99 @@ def resolve_includes(
     if not pipeline.includes:
         return pipeline
 
-    # Normalize source file path for comparison
-    source_key = str(Path(pipeline.source_file).resolve()) if pipeline.source_file else None
+    source_key = _get_source_key(pipeline)
+    _validate_circular_dependency(source_key, _visited)
+    _validate_depth_limit(_depth, max_depth, _visited, source_key)
 
-    # Cycle detection using DFS with visited set
-    if source_key and source_key in _visited:
-        cycle_list = [*list(_visited), source_key]
+    if source_key:
+        _visited.add(source_key)
+
+    merged_steps = _merge_included_pipelines(
+        pipeline, base_path, max_depth, namespace_isolation, _visited, _depth
+    )
+    merged_steps.extend(pipeline.steps)
+
+    return _build_resolved_pipeline(pipeline, merged_steps)
+
+
+def _get_source_key(pipeline: PipelineDefinition) -> str | None:
+    """Get normalized source file path for pipeline.
+
+    Args:
+        pipeline: Pipeline definition.
+
+    Returns:
+        Normalized source path or None.
+    """
+    return str(Path(pipeline.source_file).resolve()) if pipeline.source_file else None
+
+
+def _validate_circular_dependency(source_key: str | None, visited: set[str]) -> None:
+    """Validate no circular dependency.
+
+    Args:
+        source_key: Source file path.
+        visited: Set of visited paths.
+
+    Raises:
+        PipelineValidationError: If circular dependency detected.
+    """
+    if source_key and source_key in visited:
+        cycle_list = [*list(visited), source_key]
         cycle = " → ".join([Path(p).name for p in cycle_list])
         raise PipelineValidationError(
             f"Circular pipeline include detected: {cycle}",
             suggestion=f"Remove circular dependency from {Path(source_key).name}",
         )
 
-    # Depth limit check
-    if _depth >= max_depth:
+
+def _validate_depth_limit(
+    depth: int, max_depth: int, visited: set[str], source_key: str | None
+) -> None:
+    """Validate include depth within limit.
+
+    Args:
+        depth: Current depth.
+        max_depth: Maximum allowed depth.
+        visited: Set of visited paths.
+        source_key: Current source key.
+
+    Raises:
+        PipelineValidationError: If depth exceeded.
+    """
+    if depth >= max_depth:
         chain = " → ".join(
-            [Path(p).name for p in _visited] + [Path(source_key).name if source_key else "?"]
+            [Path(p).name for p in visited] + [Path(source_key).name if source_key else "?"]
         )
         raise PipelineValidationError(
             f"Pipeline include depth exceeded maximum of {max_depth}",
             suggestion=f"Reduce nesting. Current chain: {chain}",
         )
 
-    if source_key:
-        _visited.add(source_key)
 
-    # Merge included pipelines
+def _merge_included_pipelines(
+    pipeline: PipelineDefinition,
+    base_path: Path,
+    max_depth: int,
+    namespace_isolation: bool,
+    visited: set[str],
+    depth: int,
+) -> list[PipelineStep]:
+    """Merge all included pipelines.
+
+    Args:
+        pipeline: Main pipeline.
+        base_path: Base path for includes.
+        max_depth: Maximum depth.
+        namespace_isolation: Enable namespacing.
+        visited: Visited paths.
+        depth: Current depth.
+
+    Returns:
+        List of merged steps.
+    """
     merged_steps = []
+
     for include_path in pipeline.includes:
         include_full = base_path / include_path
 
@@ -803,22 +869,18 @@ def resolve_includes(
             continue
 
         try:
-            # Load included pipeline
             included = load_pipeline(include_full, pipeline.variables)
-
-            # Recursively resolve nested includes
             resolved = resolve_includes(
                 included,
                 include_full.parent,
                 max_depth=max_depth,
                 namespace_isolation=namespace_isolation,
-                _visited=_visited.copy(),  # Copy to avoid mutation across branches
-                _depth=_depth + 1,
+                _visited=visited.copy(),
+                _depth=depth + 1,
             )
 
-            # Apply namespace isolation
             if namespace_isolation:
-                namespace = Path(include_path).stem  # Use filename as namespace
+                namespace = Path(include_path).stem
                 namespaced_steps = _apply_namespace(resolved.steps, namespace)
                 merged_steps.extend(namespaced_steps)
                 logger.debug(f"Included pipeline '{namespace}' with {len(namespaced_steps)} steps")
@@ -832,9 +894,21 @@ def resolve_includes(
                 suggestion=f"Check file exists and is valid YAML: {e}",
             ) from e
 
-    # Add main pipeline steps after includes
-    merged_steps.extend(pipeline.steps)
+    return merged_steps
+
 
+def _build_resolved_pipeline(
+    pipeline: PipelineDefinition, merged_steps: list[PipelineStep]
+) -> PipelineDefinition:
+    """Build resolved pipeline with merged steps.
+
+    Args:
+        pipeline: Original pipeline.
+        merged_steps: Merged steps list.
+
+    Returns:
+        New PipelineDefinition with resolved includes.
+    """
     return PipelineDefinition(
         name=pipeline.name,
         version=pipeline.version,
@@ -842,7 +916,7 @@ def resolve_includes(
         steps=merged_steps,
         parallel_groups=pipeline.parallel_groups,
         variables=pipeline.variables,
-        includes=[],  # Clear after resolution
+        includes=[],
         source_file=pipeline.source_file,
     )
 
@@ -865,6 +939,8 @@ def _apply_namespace(steps: list[PipelineStep], namespace: str) -> list[Pipeline
     namespaced = []
     for step in steps:
         # Create a copy with namespaced name
+        # NECESSARY COPIES: All .copy() calls create isolated params/conditions.
+        # This prevents parameter mutations from affecting original step definitions.
         namespaced_step = PipelineStep(
             name=f"{namespace}.{step.name}",
             type=step.type,

oscura/{config → core/config}/preferences.py

@@ -15,7 +15,7 @@ from typing import Any
 
 import yaml
 
-from oscura.config.schema import validate_against_schema
+from oscura.core.config.schema import validate_against_schema
 from oscura.core.exceptions import ConfigurationError
 
 logger = logging.getLogger(__name__)

oscura/{config → core/config}/protocol.py

@@ -18,7 +18,7 @@ from typing import TYPE_CHECKING, Any
 
 import yaml
 
-from oscura.config.schema import validate_against_schema
+from oscura.core.config.schema import validate_against_schema
 from oscura.core.exceptions import ConfigurationError
 
 if TYPE_CHECKING:
@@ -644,7 +644,7 @@ class ProtocolWatcher:
                 continue
 
             try:
-                mtime = os.path.getmtime(file_path)  # noqa: PTH204
+                mtime = os.path.getmtime(file_path)
             except OSError:
                 continue
 
@@ -674,7 +674,7 @@ class ProtocolWatcher:
         for file_path in self.directory.glob("**/*.yaml"):
             if file_path.is_file():
                 with contextlib.suppress(OSError):
-                    self._file_mtimes[str(file_path)] = os.path.getmtime(file_path)  # noqa: PTH204
+                    self._file_mtimes[str(file_path)] = os.path.getmtime(file_path)
 
     def _notify(self, protocol: ProtocolDefinition) -> None:
         """Notify callbacks of protocol change."""