kailash 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

kailash/__init__.py

@@ -52,7 +52,7 @@ except ImportError:
 # For backward compatibility
 WorkflowGraph = Workflow
 
-__version__ = "0.8.0"
+__version__ = "0.8.2"
 
 __all__ = [
     # Core workflow components

kailash/middleware/communication/realtime.py

@@ -177,87 +177,108 @@ class ConnectionManager:
             "active_users": len(self.user_connections),
             "total_messages_sent": total_messages,
         }
-    
-    def filter_events(self, events: List[BaseEvent], event_filter: EventFilter = None) -> List[BaseEvent]:
+
+    def filter_events(
+        self, events: List[BaseEvent], event_filter: EventFilter = None
+    ) -> List[BaseEvent]:
         """Filter events based on event filter criteria."""
         if not event_filter:
             return events
-        
+
         filtered = []
         for event in events:
             # Apply session filter
-            if event_filter.session_id and hasattr(event, 'session_id'):
+            if event_filter.session_id and hasattr(event, "session_id"):
                 if event.session_id != event_filter.session_id:
                     continue
-            
+
             # Apply user filter
-            if event_filter.user_id and hasattr(event, 'user_id'):
+            if event_filter.user_id and hasattr(event, "user_id"):
                 if event.user_id != event_filter.user_id:
                     continue
-            
+
             # Apply event type filter
-            if event_filter.event_types and event.event_type not in event_filter.event_types:
+            if (
+                event_filter.event_types
+                and event.event_type not in event_filter.event_types
+            ):
                 continue
-            
+
             filtered.append(event)
-        
+
         return filtered
-    
+
     def set_event_filter(self, connection_id: str, event_filter: EventFilter):
         """Set event filter for a specific connection."""
         if connection_id in self.connections:
             self.connections[connection_id]["event_filter"] = event_filter
-    
+
     def get_event_filter(self, connection_id: str) -> Optional[EventFilter]:
         """Get event filter for a specific connection."""
         if connection_id in self.connections:
             return self.connections[connection_id].get("event_filter")
         return None
-    
+
     # Alias methods for compatibility
-    def event_filter(self, events: List[BaseEvent], filter_criteria: EventFilter = None) -> List[BaseEvent]:
+    def event_filter(
+        self, events: List[BaseEvent], filter_criteria: EventFilter = None
+    ) -> List[BaseEvent]:
         """Alias for filter_events method."""
         return self.filter_events(events, filter_criteria)
-    
+
     async def on_event(self, event: BaseEvent):
         """Handle incoming event - route to appropriate connections."""
         await self.handle_event(event)
-    
+
     async def handle_event(self, event: BaseEvent):
         """Handle and route event to matching connections."""
         await self.process_event(event)
-    
+
     async def process_event(self, event: BaseEvent):
         """Process event and broadcast to matching connections."""
         message = {
             "type": "event",
-            "event_type": event.event_type.value if hasattr(event.event_type, 'value') else str(event.event_type),
+            "event_type": (
+                event.event_type.value
+                if hasattr(event.event_type, "value")
+                else str(event.event_type)
+            ),
             "data": event.data,
-            "timestamp": event.timestamp.isoformat() if hasattr(event, 'timestamp') else datetime.now(timezone.utc).isoformat(),
-            "session_id": getattr(event, 'session_id', None),
-            "user_id": getattr(event, 'user_id', None),
+            "timestamp": (
+                event.timestamp.isoformat()
+                if hasattr(event, "timestamp")
+                else datetime.now(timezone.utc).isoformat()
+            ),
+            "session_id": getattr(event, "session_id", None),
+            "user_id": getattr(event, "user_id", None),
         }
-        
+
         # Broadcast to all matching connections
         for connection_id, connection in self.connections.items():
             event_filter = connection.get("event_filter")
-            
+
             # Check if this connection should receive this event
             should_send = True
             if event_filter:
                 # Apply session filter
-                if event_filter.session_id and connection["session_id"] != event_filter.session_id:
+                if (
+                    event_filter.session_id
+                    and connection["session_id"] != event_filter.session_id
+                ):
                     should_send = False
-                
+
                 # Apply user filter
-                if event_filter.user_id and connection["user_id"] != event_filter.user_id:
+                if (
+                    event_filter.user_id
+                    and connection["user_id"] != event_filter.user_id
+                ):
                     should_send = False
-                
+
                 # Apply event type filter
-                if hasattr(event_filter, 'event_types') and event_filter.event_types:
+                if hasattr(event_filter, "event_types") and event_filter.event_types:
                     if event.event_type not in event_filter.event_types:
                         should_send = False
-            
+
             if should_send:
                 await self.send_to_connection(connection_id, message)
 

kailash/nodes/code/python.py

@@ -50,6 +50,7 @@ import ast
 import importlib.util
 import inspect
 import logging
+import os
 import resource
 import traceback
 from collections.abc import Callable
@@ -465,6 +466,10 @@ class CodeExecutor:
             # Normal operation - eagerly load all modules
             for module_name in self.allowed_modules:
                 try:
+                    # Skip scipy in CI due to version conflicts
+                    if module_name == "scipy" and os.environ.get("CI"):
+                        logger.warning("Skipping scipy import in CI environment")
+                        continue
                     module = importlib.import_module(module_name)
                     namespace[module_name] = module
                 except ImportError:

kailash/runtime/local.py

@@ -43,6 +43,7 @@ import networkx as nx
 
 from kailash.nodes import Node
 from kailash.runtime.parameter_injector import WorkflowParameterInjector
+from kailash.runtime.secret_provider import EnvironmentSecretProvider, SecretProvider
 from kailash.sdk_exceptions import (
     RuntimeExecutionError,
     WorkflowExecutionError,
@@ -84,6 +85,7 @@ class LocalRuntime:
         enable_security: bool = False,
         enable_audit: bool = False,
         resource_limits: Optional[dict[str, Any]] = None,
+        secret_provider: Optional[Any] = None,
     ):
         """Initialize the unified runtime.
 
@@ -97,12 +99,14 @@ class LocalRuntime:
             enable_security: Whether to enable security features.
             enable_audit: Whether to enable audit logging.
             resource_limits: Resource limits (memory_mb, cpu_cores, etc.).
+            secret_provider: Optional secret provider for runtime secret injection.
         """
         self.debug = debug
         self.enable_cycles = enable_cycles
         self.enable_async = enable_async
         self.max_concurrency = max_concurrency
         self.user_context = user_context
+        self.secret_provider = secret_provider
         self.enable_monitoring = enable_monitoring
         self.enable_security = enable_security
         self.enable_audit = enable_audit
@@ -132,6 +136,22 @@ class LocalRuntime:
             "user_context": user_context,
         }
 
+    def _extract_secret_requirements(self, workflow: "Workflow") -> list:
+        """Extract secret requirements from workflow nodes.
+
+        Args:
+            workflow: Workflow to analyze
+
+        Returns:
+            List of secret requirements
+        """
+        requirements = []
+        for node_id, node in workflow.nodes.items():
+            if hasattr(node, "get_secret_requirements"):
+                node_requirements = node.get_secret_requirements()
+                requirements.extend(node_requirements)
+        return requirements
+
     def execute(
         self,
         workflow: Workflow,
@@ -1057,6 +1077,52 @@ class LocalRuntime:
                 for warning in warnings:
                     self.logger.warning(f"Parameter validation: {warning}")
 
+        # Inject secrets into the processed parameters
+        if self.secret_provider:
+            # Get secret requirements from workflow nodes
+            requirements = self._extract_secret_requirements(workflow)
+            if requirements:
+                # Fetch secrets from provider
+                secrets = self.secret_provider.get_secrets(requirements)
+
+                # Inject secrets into workflow-level parameters
+                if secrets:
+                    # If we have workflow-level parameters, add secrets to them
+                    if workflow_level_params:
+                        workflow_level_params.update(secrets)
+
+                        # Re-inject workflow parameters with secrets
+                        injector = WorkflowParameterInjector(workflow, debug=self.debug)
+                        injected_params = injector.transform_workflow_parameters(
+                            workflow_level_params
+                        )
+
+                        # Merge secret-enhanced parameters
+                        for node_id, node_params in injected_params.items():
+                            if node_id not in result:
+                                result[node_id] = {}
+                            for param_name, param_value in node_params.items():
+                                if param_name not in result[node_id]:
+                                    result[node_id][param_name] = param_value
+                    else:
+                        # Create workflow-level parameters from secrets only
+                        injector = WorkflowParameterInjector(workflow, debug=self.debug)
+                        injected_params = injector.transform_workflow_parameters(
+                            secrets
+                        )
+
+                        # Merge secret parameters
+                        for node_id, node_params in injected_params.items():
+                            if node_id not in result:
+                                result[node_id] = {}
+                            for param_name, param_value in node_params.items():
+                                if param_name not in result[node_id]:
+                                    result[node_id][param_name] = param_value
+
+                    # Ensure result is not None if we added secrets
+                    if result is None:
+                        result = {}
+
         return result if result else None
 
     def _separate_parameter_formats(

kailash/runtime/secret_provider.py

@@ -0,0 +1,293 @@
+"""Runtime secret management interface and providers.
+
+This module provides the SecretProvider interface and implementations for
+injecting secrets at runtime, eliminating the need to embed secrets in
+environment variables or workflow parameters.
+"""
+
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SecretRequirement:
+    """Metadata for a required secret."""
+
+    def __init__(
+        self,
+        name: str,
+        parameter_name: str,
+        version: Optional[str] = None,
+        optional: bool = False,
+    ):
+        """Initialize secret requirement.
+
+        Args:
+            name: Secret name in the provider (e.g., "jwt-signing-key")
+            parameter_name: Parameter name in the node (e.g., "secret_key")
+            version: Optional version identifier
+            optional: Whether this secret is optional
+        """
+        self.name = name
+        self.parameter_name = parameter_name
+        self.version = version
+        self.optional = optional
+
+
+class SecretProvider(ABC):
+    """Base interface for secret providers."""
+
+    @abstractmethod
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Fetch a secret by name and optional version.
+
+        Args:
+            name: Secret name
+            version: Optional version identifier
+
+        Returns:
+            Secret value as string
+
+        Raises:
+            SecretNotFoundError: If secret doesn't exist
+            SecretProviderError: If provider operation fails
+        """
+        pass
+
+    @abstractmethod
+    def list_secrets(self) -> List[str]:
+        """List available secrets.
+
+        Returns:
+            List of secret names
+        """
+        pass
+
+    def get_secrets(self, requirements: List[SecretRequirement]) -> Dict[str, str]:
+        """Fetch multiple secrets based on requirements.
+
+        Args:
+            requirements: List of secret requirements
+
+        Returns:
+            Dictionary mapping parameter names to secret values
+        """
+        secrets = {}
+        for req in requirements:
+            try:
+                secret_value = self.get_secret(req.name, req.version)
+                secrets[req.parameter_name] = secret_value
+            except Exception as e:
+                if req.optional:
+                    logger.warning(f"Optional secret {req.name} not found: {e}")
+                    continue
+                else:
+                    raise
+        return secrets
+
+
+class EnvironmentSecretProvider(SecretProvider):
+    """Secret provider that fetches secrets from environment variables.
+
+    This provider maintains backward compatibility by reading secrets from
+    environment variables, but provides a secure interface for runtime injection.
+    """
+
+    def __init__(self, prefix: str = "KAILASH_SECRET_"):
+        """Initialize environment secret provider.
+
+        Args:
+            prefix: Prefix for environment variables containing secrets
+        """
+        self.prefix = prefix
+
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Get secret from environment variable.
+
+        Args:
+            name: Secret name (will be prefixed and uppercased)
+            version: Ignored for environment provider
+
+        Returns:
+            Secret value from environment
+
+        Raises:
+            SecretNotFoundError: If environment variable not found
+        """
+        # Convert name to environment variable format
+        env_name = f"{self.prefix}{name.upper().replace('-', '_')}"
+
+        secret_value = os.environ.get(env_name)
+        if secret_value is None:
+            # Try without prefix for backward compatibility
+            secret_value = os.environ.get(name.upper().replace("-", "_"))
+
+        if secret_value is None:
+            raise SecretNotFoundError(
+                f"Secret '{name}' not found in environment variables"
+            )
+
+        return secret_value
+
+    def list_secrets(self) -> List[str]:
+        """List all secrets available in environment.
+
+        Returns:
+            List of secret names (without prefix)
+        """
+        secrets = []
+        for key in os.environ:
+            if key.startswith(self.prefix):
+                # Remove prefix and convert back to secret name format
+                secret_name = key[len(self.prefix) :].lower().replace("_", "-")
+                secrets.append(secret_name)
+        return secrets
+
+
+class VaultSecretProvider(SecretProvider):
+    """Secret provider for HashiCorp Vault.
+
+    This provider integrates with HashiCorp Vault for enterprise secret management.
+    """
+
+    def __init__(self, vault_url: str, vault_token: str, mount_path: str = "secret"):
+        """Initialize Vault secret provider.
+
+        Args:
+            vault_url: Vault server URL
+            vault_token: Vault authentication token
+            mount_path: Vault mount path for secrets
+        """
+        self.vault_url = vault_url
+        self.vault_token = vault_token
+        self.mount_path = mount_path
+        self._client = None
+
+    @property
+    def client(self):
+        """Lazy initialization of Vault client."""
+        if self._client is None:
+            try:
+                import hvac
+
+                self._client = hvac.Client(url=self.vault_url, token=self.vault_token)
+            except ImportError:
+                raise RuntimeError(
+                    "hvac library not installed. Install with: pip install hvac"
+                )
+        return self._client
+
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Get secret from Vault.
+
+        Args:
+            name: Secret path in Vault
+            version: Optional version (for KV v2)
+
+        Returns:
+            Secret value
+        """
+        try:
+            # Try KV v2 first
+            response = self.client.secrets.kv.v2.read_secret_version(
+                path=name, version=version, mount_point=self.mount_path
+            )
+            return response["data"]["data"]["value"]
+        except Exception:
+            # Fall back to KV v1
+            response = self.client.secrets.kv.v1.read_secret(
+                path=name, mount_point=self.mount_path
+            )
+            return response["data"]["value"]
+
+    def list_secrets(self) -> List[str]:
+        """List all secrets in Vault.
+
+        Returns:
+            List of secret paths
+        """
+        try:
+            response = self.client.secrets.kv.v2.list_secrets(
+                path="", mount_point=self.mount_path
+            )
+            return response["data"]["keys"]
+        except Exception:
+            # Fall back to KV v1
+            response = self.client.secrets.kv.v1.list_secrets(
+                path="", mount_point=self.mount_path
+            )
+            return response["data"]["keys"]
+
+
+class AWSSecretProvider(SecretProvider):
+    """Secret provider for AWS Secrets Manager.
+
+    This provider integrates with AWS Secrets Manager for cloud-native secret management.
+    """
+
+    def __init__(self, region_name: str = "us-east-1"):
+        """Initialize AWS secret provider.
+
+        Args:
+            region_name: AWS region
+        """
+        self.region_name = region_name
+        self._client = None
+
+    @property
+    def client(self):
+        """Lazy initialization of AWS client."""
+        if self._client is None:
+            try:
+                import boto3
+
+                self._client = boto3.client(
+                    "secretsmanager", region_name=self.region_name
+                )
+            except ImportError:
+                raise RuntimeError(
+                    "boto3 library not installed. Install with: pip install boto3"
+                )
+        return self._client
+
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Get secret from AWS Secrets Manager.
+
+        Args:
+            name: Secret name in AWS
+            version: Optional version ID
+
+        Returns:
+            Secret value
+        """
+        kwargs = {"SecretId": name}
+        if version:
+            kwargs["VersionId"] = version
+
+        response = self.client.get_secret_value(**kwargs)
+        return response["SecretString"]
+
+    def list_secrets(self) -> List[str]:
+        """List all secrets in AWS Secrets Manager.
+
+        Returns:
+            List of secret names
+        """
+        response = self.client.list_secrets()
+        return [secret["Name"] for secret in response["SecretList"]]
+
+
+class SecretNotFoundError(Exception):
+    """Raised when a secret cannot be found."""
+
+    pass
+
+
+class SecretProviderError(Exception):
+    """Raised when a secret provider operation fails."""
+
+    pass