kailash 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

kailash/security.py

@@ -0,0 +1,845 @@
+"""
+Comprehensive Security Framework for the Kailash SDK.
+
+This module provides an extensive security framework designed to protect against
+common vulnerabilities and ensure safe execution of workflows, particularly in
+cyclic patterns where long-running processes may be exposed to additional risks.
+It implements defense-in-depth strategies with configurable policies.
+
+Design Philosophy:
+    Implements a comprehensive security-first approach with fail-safe defaults,
+    defense-in-depth strategies, and extensive monitoring. Designed to protect
+    against both common web vulnerabilities and workflow-specific attack vectors
+    while maintaining usability and performance.
+
+Key Security Features:
+    - **Path Security**: Comprehensive path traversal prevention
+    - **Input Validation**: Multi-layer input sanitization and validation
+    - **Execution Security**: Safe code execution with sandboxing
+    - **Resource Limits**: Memory, CPU, and execution time constraints
+    - **Injection Protection**: Command and code injection prevention
+    - **Audit Logging**: Comprehensive security event logging
+
+Cycle Security Enhancements (v0.2.0):
+    Enhanced security specifically for cyclic workflows including:
+    - Long-running process monitoring and limits
+    - Iteration-based resource accumulation detection
+    - Parameter injection attack prevention in cycles
+    - State corruption detection and prevention
+    - Convergence manipulation attack detection
+
+Security Layers:
+    1. **Input Layer**: Validation and sanitization of all inputs
+    2. **Execution Layer**: Sandboxed execution with resource limits
+    3. **File System Layer**: Controlled file access with path validation
+    4. **Network Layer**: Controlled external communication
+    5. **Monitoring Layer**: Real-time security event detection
+
+Vulnerability Protection:
+    - **Path Traversal**: Comprehensive path validation and canonicalization
+    - **Command Injection**: Input sanitization and safe command execution
+    - **Code Injection**: AST validation and safe code execution
+    - **Resource Exhaustion**: Memory, CPU, and time limits
+    - **Information Disclosure**: Controlled error messages and logging
+    - **Privilege Escalation**: Sandboxed execution environments
+
+Core Components:
+    - SecurityConfig: Centralized security policy configuration
+    - ValidationFramework: Multi-layer input validation system
+    - ExecutionSandbox: Safe code execution environment
+    - ResourceMonitor: Real-time resource usage monitoring
+    - AuditLogger: Comprehensive security event logging
+
+Upstream Dependencies:
+    - Operating system security features for sandboxing
+    - Python security libraries for validation and monitoring
+    - Workflow execution framework for integration points
+
+Downstream Consumers:
+    - All workflow execution components requiring security
+    - Node implementations with external resource access
+    - Runtime engines executing user-provided code
+    - API endpoints handling external workflow requests
+
+Examples:
+    Basic security configuration:
+
+    >>> from kailash.security import SecurityConfig, validate_node_parameters
+    >>> # Configure security policy
+    >>> config = SecurityConfig(
+    ...     max_execution_time=300,
+    ...     max_memory_mb=1024,
+    ...     allowed_paths=["/safe/directory"]
+    ... )
+    >>> # Validate node parameters
+    >>> validate_node_parameters(parameters, config)
+
+    Secure file operations:
+
+    >>> from kailash.security import safe_file_operation, validate_path
+    >>> # Validate and access file safely
+    >>> safe_path = validate_path("/user/input/path", base_dir="/safe/root")
+    >>> with safe_file_operation(safe_path, "r") as f:
+    ...     content = f.read()
+
+    Execution timeout protection:
+
+    >>> from kailash.security import execution_timeout
+    >>> @execution_timeout(seconds=30)
+    ... def potentially_long_running_function():
+    ...     # Function will be terminated if it runs longer than 30 seconds
+    ...     return process_data()
+
+    Comprehensive monitoring:
+
+    >>> from kailash.security import SecurityMonitor
+    >>> monitor = SecurityMonitor()
+    >>> with monitor.track_execution("workflow_execution"):
+    ...     # All security events will be monitored and logged
+    ...     runtime.execute(workflow)
+
+Security Policies:
+    Configurable security policies allow adaptation to different environments:
+    - **Development**: Relaxed policies for debugging and testing
+    - **Staging**: Moderate policies balancing security and functionality
+    - **Production**: Strict policies prioritizing security
+    - **High-Security**: Maximum security for sensitive environments
+
+See Also:
+    - :mod:`kailash.nodes.code.python` for secure code execution
+    - :mod:`kailash.workflow.safety` for workflow-specific safety measures
+    - :doc:`/guides/security` for comprehensive security best practices
+"""
+
+import logging
+import os
+import re
+import tempfile
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Union
+
+logger = logging.getLogger(__name__)
+
+
+class SecurityError(Exception):
+    """Raised when a security policy violation is detected."""
+
+    pass
+
+
+class PathTraversalError(SecurityError):
+    """Raised when path traversal attempt is detected."""
+
+    pass
+
+
+class CommandInjectionError(SecurityError):
+    """Raised when command injection attempt is detected."""
+
+    pass
+
+
+class ExecutionTimeoutError(SecurityError):
+    """Raised when execution exceeds allowed time limit."""
+
+    pass
+
+
+class MemoryLimitError(SecurityError):
+    """Raised when memory usage exceeds allowed limit."""
+
+    pass
+
+
+class SecurityConfig:
+    """Configuration for security policies and limits."""
+
+    def __init__(
+        self,
+        allowed_directories: Optional[List[str]] = None,
+        max_file_size: int = 100 * 1024 * 1024,  # 100MB
+        execution_timeout: float = 300.0,  # 5 minutes
+        memory_limit: int = 512 * 1024 * 1024,  # 512MB
+        allowed_file_extensions: Optional[List[str]] = None,
+        enable_audit_logging: bool = True,
+        enable_path_validation: bool = True,
+        enable_command_validation: bool = True,
+    ):
+        """
+        Initialize security configuration.
+
+        Args:
+            allowed_directories: List of directories where file operations are permitted
+            max_file_size: Maximum file size in bytes
+            execution_timeout: Maximum execution time in seconds
+            memory_limit: Maximum memory usage in bytes
+            allowed_file_extensions: List of allowed file extensions
+            enable_audit_logging: Whether to log security events
+            enable_path_validation: Whether to validate file paths
+            enable_command_validation: Whether to validate command strings
+        """
+        default_dirs = [
+            os.path.expanduser("~/.kailash"),
+            tempfile.gettempdir(),  # Allow all temp directories
+            os.getcwd(),
+            "/tmp",  # Unix temp directory
+            "/var/tmp",  # Unix temp directory
+        ]
+
+        # Check for additional allowed directories from environment
+        env_dirs = os.environ.get("KAILASH_ALLOWED_DIRS", "")
+        if env_dirs:
+            for dir_path in env_dirs.split(":"):
+                if dir_path and os.path.isdir(dir_path):
+                    default_dirs.append(os.path.abspath(dir_path))
+
+        self.allowed_directories = allowed_directories or default_dirs
+        self.max_file_size = max_file_size
+        self.execution_timeout = execution_timeout
+        self.memory_limit = memory_limit
+        self.allowed_file_extensions = allowed_file_extensions or [
+            ".txt",
+            ".csv",
+            ".tsv",
+            ".json",
+            ".yaml",
+            ".yml",
+            ".py",
+            ".md",
+            ".xml",
+            ".log",
+            ".dat",
+            ".conf",
+            ".cfg",
+            ".ini",
+            ".properties",
+            ".html",
+            ".htm",
+            ".xhtml",
+            ".jsonl",
+            ".ndjson",
+        ]
+        self.enable_audit_logging = enable_audit_logging
+        self.enable_path_validation = enable_path_validation
+        self.enable_command_validation = enable_command_validation
+
+
+# Global security configuration
+_security_config = SecurityConfig()
+
+
+def get_security_config() -> SecurityConfig:
+    """Get the current security configuration."""
+    return _security_config
+
+
+def set_security_config(config: SecurityConfig) -> None:
+    """Set the global security configuration."""
+    global _security_config
+    _security_config = config
+
+
+def validate_file_path(
+    file_path: Union[str, Path],
+    config: Optional[SecurityConfig] = None,
+    operation: str = "access",
+) -> Path:
+    """
+    Validate and sanitize file paths to prevent traversal attacks.
+
+    Args:
+        file_path: The file path to validate
+        config: Security configuration (uses global if None)
+        operation: Description of the operation for logging
+
+    Returns:
+        Validated and normalized Path object
+
+    Raises:
+        PathTraversalError: If path traversal attempt is detected
+        SecurityError: If path is outside allowed directories
+
+    Examples:
+        >>> # Safe paths
+        >>> validate_file_path("data/file.txt")
+        PosixPath('data/file.txt')
+
+        >>> # Blocked paths
+        >>> validate_file_path("../../../etc/passwd")
+        Traceback (most recent call last):
+        PathTraversalError: Path traversal attempt detected
+    """
+    if config is None:
+        config = get_security_config()
+
+    if not config.enable_path_validation:
+        return Path(file_path)
+
+    try:
+        # Convert to Path and resolve to absolute path
+        path = Path(file_path).resolve()
+
+        # Check for path traversal indicators
+        path_str = str(path)
+        if ".." in str(file_path):
+            if config.enable_audit_logging:
+                logger.warning(
+                    f"Path traversal attempt detected: {file_path} -> {path}"
+                )
+            raise PathTraversalError(f"Path traversal attempt detected: {file_path}")
+
+        # Check for access to sensitive system directories
+        sensitive_dirs = ["/etc", "/var", "/usr", "/root", "/boot", "/sys", "/proc"]
+        if any(path_str.startswith(sensitive) for sensitive in sensitive_dirs):
+            if config.enable_audit_logging:
+                logger.warning(
+                    f"Path traversal attempt detected: {file_path} -> {path}"
+                )
+            raise PathTraversalError(f"Path traversal attempt detected: {file_path}")
+
+        # Validate file extension
+        if path.suffix and path.suffix.lower() not in config.allowed_file_extensions:
+            if config.enable_audit_logging:
+                logger.warning(f"File extension not allowed: {path.suffix} in {path}")
+            raise SecurityError(f"File extension not allowed: {path.suffix}")
+
+        # Check if path is within allowed directories
+        path_in_allowed_dir = False
+        for allowed_dir in config.allowed_directories:
+            try:
+                allowed_path = Path(allowed_dir).resolve()
+                # Use more robust relative path checking
+                try:
+                    path.relative_to(allowed_path)
+                    path_in_allowed_dir = True
+                    break
+                except ValueError:
+                    # Try alternative method for compatibility
+                    if str(path).startswith(str(allowed_path)):
+                        path_in_allowed_dir = True
+                        break
+            except (ValueError, OSError):
+                # Handle cases where path resolution fails
+                if str(path).startswith(str(allowed_dir)):
+                    path_in_allowed_dir = True
+                    break
+
+        if not path_in_allowed_dir:
+            if config.enable_audit_logging:
+                logger.warning(f"Path outside allowed directories: {path}")
+            raise SecurityError(f"Path outside allowed directories: {path}")
+
+        if config.enable_audit_logging:
+            logger.info(f"File path validated for {operation}: {path}")
+
+        return path
+
+    except (OSError, ValueError) as e:
+        if config.enable_audit_logging:
+            logger.error(f"Path validation error: {e}")
+        raise SecurityError(f"Invalid file path: {file_path}")
+
+
+def safe_open(
+    file_path: Union[str, Path],
+    mode: str = "r",
+    config: Optional[SecurityConfig] = None,
+    **kwargs,
+):
+    """
+    Safely open a file with security validation.
+
+    Args:
+        file_path: Path to the file
+        mode: File open mode
+        config: Security configuration
+        **kwargs: Additional arguments for open()
+
+    Returns:
+        File handle
+
+    Raises:
+        SecurityError: If security validation fails
+
+    Examples:
+        >>> with safe_open("data/file.txt", "r") as f:
+        ...     content = f.read()
+    """
+    if config is None:
+        config = get_security_config()
+
+    # Validate the file path
+    validated_path = validate_file_path(file_path, config, f"open({mode})")
+
+    # Check file size for read operations
+    if "r" in mode and validated_path.exists():
+        file_size = validated_path.stat().st_size
+        if file_size > config.max_file_size:
+            raise SecurityError(
+                f"File too large: {file_size} bytes > {config.max_file_size}"
+            )
+
+    # Create directory if writing and it doesn't exist
+    if "w" in mode or "a" in mode:
+        validated_path.parent.mkdir(parents=True, exist_ok=True)
+
+    if config.enable_audit_logging:
+        logger.info(f"Opening file: {validated_path} (mode: {mode})")
+
+    return open(validated_path, mode, **kwargs)
+
+
+def validate_command_string(
+    command: str, config: Optional[SecurityConfig] = None
+) -> str:
+    """
+    Validate command strings to prevent injection attacks.
+
+    Args:
+        command: Command string to validate
+        config: Security configuration
+
+    Returns:
+        Validated command string
+
+    Raises:
+        CommandInjectionError: If command injection attempt is detected
+    """
+    if config is None:
+        config = get_security_config()
+
+    if not config.enable_command_validation:
+        return command
+
+    # Check for common injection patterns
+    dangerous_patterns = [
+        r";",  # Command chaining
+        r"&&",  # Logical AND command chaining
+        r"\|\|",  # Logical OR command chaining
+        r"\|",  # Pipe operations
+        r"\$\(",  # Command substitution
+        r"`.*`",  # Backtick command substitution
+        r">\s*/dev/",  # Redirect to devices
+        r"<.*>",  # Input/output redirection
+        r"\beval\b",  # eval command
+        r"\bexec\b",  # exec command
+        r"rm\s+.*(\/|\*)",  # rm with dangerous paths
+        r"cat\s+\/etc\/",  # reading system files
+    ]
+
+    for pattern in dangerous_patterns:
+        if re.search(pattern, command, re.IGNORECASE):
+            if config.enable_audit_logging:
+                logger.warning(f"Command injection attempt detected: {command}")
+            raise CommandInjectionError(f"Potentially dangerous command: {command}")
+
+    if config.enable_audit_logging:
+        logger.info(
+            f"Command validated: {command[:100]}{'...' if len(command) > 100 else ''}"
+        )
+
+    return command
+
+
+@contextmanager
+def execution_timeout(
+    timeout: Optional[float] = None, config: Optional[SecurityConfig] = None
+):
+    """
+    Context manager to enforce execution timeouts.
+
+    Args:
+        timeout: Timeout in seconds (uses config default if None)
+        config: Security configuration
+
+    Raises:
+        ExecutionTimeoutError: If execution exceeds timeout
+
+    Examples:
+        >>> with execution_timeout(30.0):
+        ...     # Code that should complete within 30 seconds
+        ...     time.sleep(5)
+    """
+    if config is None:
+        config = get_security_config()
+
+    if timeout is None:
+        timeout = config.execution_timeout
+
+    start_time = time.time()
+
+    try:
+        yield
+    finally:
+        elapsed_time = time.time() - start_time
+        if elapsed_time > timeout:
+            if config.enable_audit_logging:
+                logger.warning(f"Execution timeout: {elapsed_time:.2f}s > {timeout}s")
+            raise ExecutionTimeoutError(
+                f"Execution timeout: {elapsed_time:.2f}s > {timeout}s"
+            )
+
+
+def sanitize_input(
+    value: Any,
+    max_length: int = 10000,
+    allowed_types: Optional[List[type]] = None,
+    config: Optional[SecurityConfig] = None,
+) -> Any:
+    """
+    Sanitize input values to prevent injection attacks.
+
+    Args:
+        value: Value to sanitize
+        max_length: Maximum string length
+        allowed_types: List of allowed types
+        config: Security configuration
+
+    Returns:
+        Sanitized value
+
+    Raises:
+        SecurityError: If input fails validation
+    """
+    if config is None:
+        config = get_security_config()
+
+    if allowed_types is None:
+        allowed_types = [str, int, float, bool, list, dict, tuple, set, type(None)]
+
+        # Core data science types
+        try:
+            import pandas as pd
+
+            allowed_types.extend(
+                [
+                    pd.DataFrame,
+                    pd.Series,
+                    pd.Index,
+                    pd.MultiIndex,
+                    pd.Categorical,
+                    pd.Timestamp,
+                    pd.Timedelta,
+                    pd.Period,
+                    pd.DatetimeIndex,
+                    pd.TimedeltaIndex,
+                    pd.PeriodIndex,
+                ]
+            )
+        except ImportError:
+            pass
+
+        try:
+            import numpy as np
+
+            numpy_types = [
+                np.ndarray,
+                np.ma.MaskedArray,
+                # All numpy scalar types
+                np.int8,
+                np.int16,
+                np.int32,
+                np.int64,
+                np.uint8,
+                np.uint16,
+                np.uint32,
+                np.uint64,
+                np.float16,
+                np.float32,
+                np.float64,
+                np.complex64,
+                np.complex128,
+                np.bool_,
+                np.object_,
+                np.datetime64,
+                np.timedelta64,
+            ]
+
+            # Add matrix if available (deprecated in NumPy 2.0)
+            if hasattr(np, "matrix"):
+                numpy_types.append(np.matrix)
+
+            # Handle NumPy version differences
+            if hasattr(np, "string_"):
+                numpy_types.append(np.string_)
+            elif hasattr(np, "bytes_"):
+                numpy_types.append(np.bytes_)
+
+            if hasattr(np, "unicode_"):
+                numpy_types.append(np.unicode_)
+            elif hasattr(np, "str_"):
+                numpy_types.append(np.str_)
+
+            # Add platform-specific types if available
+            if hasattr(np, "float128"):
+                numpy_types.append(np.float128)
+            if hasattr(np, "complex256"):
+                numpy_types.append(np.complex256)
+
+            # Add generic numpy type to catch all numpy scalars
+            if hasattr(np, "generic"):
+                numpy_types.append(np.generic)
+
+            allowed_types.extend(numpy_types)
+        except ImportError:
+            pass
+
+        # Deep learning frameworks
+        try:
+            import torch
+
+            allowed_types.extend(
+                [
+                    torch.Tensor,
+                    torch.nn.Module,
+                    torch.nn.Parameter,
+                    torch.cuda.FloatTensor,
+                    torch.cuda.DoubleTensor,
+                    torch.cuda.IntTensor,
+                    torch.cuda.LongTensor,
+                ]
+            )
+        except ImportError:
+            pass
+
+        try:
+            import tensorflow as tf
+
+            allowed_types.extend(
+                [
+                    tf.Tensor,
+                    tf.Variable,
+                    tf.constant,
+                    tf.keras.Model,
+                    tf.keras.layers.Layer,
+                    tf.data.Dataset,
+                ]
+            )
+        except ImportError:
+            pass
+
+        # Scientific computing
+        try:
+            import scipy.sparse
+
+            allowed_types.extend(
+                [
+                    scipy.sparse.csr_matrix,
+                    scipy.sparse.csc_matrix,
+                    scipy.sparse.coo_matrix,
+                    scipy.sparse.dia_matrix,
+                    scipy.sparse.dok_matrix,
+                    scipy.sparse.lil_matrix,
+                ]
+            )
+        except ImportError:
+            pass
+
+        # Machine learning frameworks
+        try:
+            from sklearn.base import BaseEstimator, TransformerMixin
+
+            allowed_types.extend([BaseEstimator, TransformerMixin])
+        except ImportError:
+            pass
+
+        try:
+            import xgboost as xgb
+
+            allowed_types.extend([xgb.DMatrix, xgb.Booster])
+        except ImportError:
+            pass
+
+        try:
+            import lightgbm as lgb
+
+            allowed_types.extend([lgb.Dataset, lgb.Booster])
+        except ImportError:
+            pass
+
+        # Data visualization
+        try:
+            from matplotlib.axes import Axes
+            from matplotlib.figure import Figure
+
+            allowed_types.extend([Figure, Axes])
+        except ImportError:
+            pass
+
+        try:
+            import plotly.graph_objects as go
+
+            allowed_types.append(go.Figure)
+        except ImportError:
+            pass
+
+        # Statistical modeling
+        try:
+            import statsmodels.api as sm
+
+            allowed_types.extend([sm.OLS, sm.GLM, sm.GLS, sm.WLS, sm.RegressionResults])
+        except ImportError:
+            pass
+
+        # Image processing
+        try:
+            from PIL import Image
+
+            allowed_types.append(Image.Image)
+        except ImportError:
+            pass
+
+        try:
+            # OpenCV uses numpy arrays, already covered
+            import cv2  # noqa: F401
+        except ImportError:
+            pass
+
+        # NLP libraries
+        try:
+            from spacy.tokens import Doc, Span, Token
+
+            allowed_types.extend([Doc, Span, Token])
+        except ImportError:
+            pass
+
+        # Graph/Network analysis
+        try:
+            import networkx as nx
+
+            allowed_types.extend([nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph])
+        except ImportError:
+            pass
+
+        # Time series
+        try:
+            from prophet import Prophet
+            from prophet.forecaster import Prophet as ProphetModel
+
+            allowed_types.extend([Prophet, ProphetModel])
+        except ImportError:
+            pass
+
+    # Type validation - allow data science types
+    type_allowed = any(isinstance(value, t) for t in allowed_types)
+
+    # Additional check for numpy scalar types
+    if not type_allowed:
+        try:
+            import numpy as np
+
+            # Check if it's any numpy type
+            if isinstance(value, np.generic):
+                type_allowed = True
+        except ImportError:
+            pass
+
+    if not type_allowed:
+        raise SecurityError(f"Input type not allowed: {type(value)}")
+
+    # String sanitization
+    if isinstance(value, str):
+        if len(value) > max_length:
+            raise SecurityError(f"Input too long: {len(value)} > {max_length}")
+
+        # Remove potentially dangerous characters and patterns
+        sanitized = re.sub(r"[<>;&|`$()]", "", value)
+        # Remove script tags and javascript
+        sanitized = re.sub(
+            r"<script.*?</script>", "", sanitized, flags=re.IGNORECASE | re.DOTALL
+        )
+        sanitized = re.sub(r"javascript:", "", sanitized, flags=re.IGNORECASE)
+
+        if sanitized != value and config.enable_audit_logging:
+            logger.warning(f"Input sanitized: {value[:50]}... -> {sanitized[:50]}...")
+
+        return sanitized
+
+    # Dictionary sanitization (recursive)
+    if isinstance(value, dict):
+        return {
+            sanitize_input(k, max_length, allowed_types, config): sanitize_input(
+                v, max_length, allowed_types, config
+            )
+            for k, v in value.items()
+        }
+
+    # List sanitization (recursive)
+    if isinstance(value, list):
+        return [
+            sanitize_input(item, max_length, allowed_types, config) for item in value
+        ]
+
+    return value
+
+
+def create_secure_temp_dir(
+    prefix: str = "kailash_", config: Optional[SecurityConfig] = None
+) -> Path:
+    """
+    Create a secure temporary directory.
+
+    Args:
+        prefix: Prefix for the directory name
+        config: Security configuration
+
+    Returns:
+        Path to the secure temporary directory
+    """
+    if config is None:
+        config = get_security_config()
+
+    # Create temp directory with secure permissions
+    temp_dir = Path(tempfile.mkdtemp(prefix=prefix))
+
+    # Set restrictive permissions (owner only)
+    temp_dir.chmod(0o700)
+
+    if config.enable_audit_logging:
+        logger.info(f"Created secure temp directory: {temp_dir}")
+
+    return temp_dir
+
+
+def validate_node_parameters(
+    parameters: Dict[str, Any], config: Optional[SecurityConfig] = None
+) -> Dict[str, Any]:
+    """
+    Validate and sanitize node parameters.
+
+    Args:
+        parameters: Node parameters to validate
+        config: Security configuration
+
+    Returns:
+        Validated and sanitized parameters
+
+    Raises:
+        SecurityError: If parameters fail validation
+    """
+    if config is None:
+        config = get_security_config()
+
+    validated_params = {}
+
+    for key, value in parameters.items():
+        # Sanitize parameter key
+        clean_key = sanitize_input(key, config=config)
+
+        # Special handling for file paths
+        if "path" in key.lower() or "file" in key.lower():
+            if isinstance(value, (str, Path)):
+                validated_value = validate_file_path(value, config, f"parameter {key}")
+            else:
+                validated_value = sanitize_input(value, config=config)
+        else:
+            validated_value = sanitize_input(value, config=config)
+
+        validated_params[clean_key] = validated_value
+
+    if config.enable_audit_logging:
+        logger.info(f"Node parameters validated: {list(validated_params.keys())}")
+
+    return validated_params