kailash 0.8.1__py3-none-any.whl → 0.8.4__py3-none-any.whl

kailash/nodes/code/python.py

@@ -103,6 +103,7 @@ ALLOWED_MODULES = {
     "glob",  # For file pattern matching
     "xml",  # For XML processing
     "uuid",  # For generating unique identifiers (safe, no I/O)
+    "hashlib",  # For cryptographic hashing (safe for common use cases)
 }
 
 

kailash/runtime/local.py

@@ -43,6 +43,7 @@ import networkx as nx
 
 from kailash.nodes import Node
 from kailash.runtime.parameter_injector import WorkflowParameterInjector
+from kailash.runtime.secret_provider import EnvironmentSecretProvider, SecretProvider
 from kailash.sdk_exceptions import (
     RuntimeExecutionError,
     WorkflowExecutionError,
@@ -84,6 +85,7 @@ class LocalRuntime:
         enable_security: bool = False,
         enable_audit: bool = False,
         resource_limits: Optional[dict[str, Any]] = None,
+        secret_provider: Optional[Any] = None,
     ):
         """Initialize the unified runtime.
 
@@ -97,12 +99,14 @@ class LocalRuntime:
             enable_security: Whether to enable security features.
             enable_audit: Whether to enable audit logging.
             resource_limits: Resource limits (memory_mb, cpu_cores, etc.).
+            secret_provider: Optional secret provider for runtime secret injection.
         """
         self.debug = debug
         self.enable_cycles = enable_cycles
         self.enable_async = enable_async
         self.max_concurrency = max_concurrency
         self.user_context = user_context
+        self.secret_provider = secret_provider
         self.enable_monitoring = enable_monitoring
         self.enable_security = enable_security
         self.enable_audit = enable_audit
@@ -132,6 +136,22 @@ class LocalRuntime:
             "user_context": user_context,
         }
 
+    def _extract_secret_requirements(self, workflow: "Workflow") -> list:
+        """Extract secret requirements from workflow nodes.
+
+        Args:
+            workflow: Workflow to analyze
+
+        Returns:
+            List of secret requirements
+        """
+        requirements = []
+        for node_id, node in workflow.nodes.items():
+            if hasattr(node, "get_secret_requirements"):
+                node_requirements = node.get_secret_requirements()
+                requirements.extend(node_requirements)
+        return requirements
+
     def execute(
         self,
         workflow: Workflow,
@@ -1057,6 +1077,52 @@ class LocalRuntime:
                 for warning in warnings:
                     self.logger.warning(f"Parameter validation: {warning}")
 
+        # Inject secrets into the processed parameters
+        if self.secret_provider:
+            # Get secret requirements from workflow nodes
+            requirements = self._extract_secret_requirements(workflow)
+            if requirements:
+                # Fetch secrets from provider
+                secrets = self.secret_provider.get_secrets(requirements)
+
+                # Inject secrets into workflow-level parameters
+                if secrets:
+                    # If we have workflow-level parameters, add secrets to them
+                    if workflow_level_params:
+                        workflow_level_params.update(secrets)
+
+                        # Re-inject workflow parameters with secrets
+                        injector = WorkflowParameterInjector(workflow, debug=self.debug)
+                        injected_params = injector.transform_workflow_parameters(
+                            workflow_level_params
+                        )
+
+                        # Merge secret-enhanced parameters
+                        for node_id, node_params in injected_params.items():
+                            if node_id not in result:
+                                result[node_id] = {}
+                            for param_name, param_value in node_params.items():
+                                if param_name not in result[node_id]:
+                                    result[node_id][param_name] = param_value
+                    else:
+                        # Create workflow-level parameters from secrets only
+                        injector = WorkflowParameterInjector(workflow, debug=self.debug)
+                        injected_params = injector.transform_workflow_parameters(
+                            secrets
+                        )
+
+                        # Merge secret parameters
+                        for node_id, node_params in injected_params.items():
+                            if node_id not in result:
+                                result[node_id] = {}
+                            for param_name, param_value in node_params.items():
+                                if param_name not in result[node_id]:
+                                    result[node_id][param_name] = param_value
+
+                    # Ensure result is not None if we added secrets
+                    if result is None:
+                        result = {}
+
         return result if result else None
 
     def _separate_parameter_formats(

kailash/runtime/secret_provider.py

@@ -0,0 +1,293 @@
+"""Runtime secret management interface and providers.
+
+This module provides the SecretProvider interface and implementations for
+injecting secrets at runtime, eliminating the need to embed secrets in
+environment variables or workflow parameters.
+"""
+
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SecretRequirement:
+    """Metadata for a required secret."""
+
+    def __init__(
+        self,
+        name: str,
+        parameter_name: str,
+        version: Optional[str] = None,
+        optional: bool = False,
+    ):
+        """Initialize secret requirement.
+
+        Args:
+            name: Secret name in the provider (e.g., "jwt-signing-key")
+            parameter_name: Parameter name in the node (e.g., "secret_key")
+            version: Optional version identifier
+            optional: Whether this secret is optional
+        """
+        self.name = name
+        self.parameter_name = parameter_name
+        self.version = version
+        self.optional = optional
+
+
+class SecretProvider(ABC):
+    """Base interface for secret providers."""
+
+    @abstractmethod
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Fetch a secret by name and optional version.
+
+        Args:
+            name: Secret name
+            version: Optional version identifier
+
+        Returns:
+            Secret value as string
+
+        Raises:
+            SecretNotFoundError: If secret doesn't exist
+            SecretProviderError: If provider operation fails
+        """
+        pass
+
+    @abstractmethod
+    def list_secrets(self) -> List[str]:
+        """List available secrets.
+
+        Returns:
+            List of secret names
+        """
+        pass
+
+    def get_secrets(self, requirements: List[SecretRequirement]) -> Dict[str, str]:
+        """Fetch multiple secrets based on requirements.
+
+        Args:
+            requirements: List of secret requirements
+
+        Returns:
+            Dictionary mapping parameter names to secret values
+        """
+        secrets = {}
+        for req in requirements:
+            try:
+                secret_value = self.get_secret(req.name, req.version)
+                secrets[req.parameter_name] = secret_value
+            except Exception as e:
+                if req.optional:
+                    logger.warning(f"Optional secret {req.name} not found: {e}")
+                    continue
+                else:
+                    raise
+        return secrets
+
+
+class EnvironmentSecretProvider(SecretProvider):
+    """Secret provider that fetches secrets from environment variables.
+
+    This provider maintains backward compatibility by reading secrets from
+    environment variables, but provides a secure interface for runtime injection.
+    """
+
+    def __init__(self, prefix: str = "KAILASH_SECRET_"):
+        """Initialize environment secret provider.
+
+        Args:
+            prefix: Prefix for environment variables containing secrets
+        """
+        self.prefix = prefix
+
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Get secret from environment variable.
+
+        Args:
+            name: Secret name (will be prefixed and uppercased)
+            version: Ignored for environment provider
+
+        Returns:
+            Secret value from environment
+
+        Raises:
+            SecretNotFoundError: If environment variable not found
+        """
+        # Convert name to environment variable format
+        env_name = f"{self.prefix}{name.upper().replace('-', '_')}"
+
+        secret_value = os.environ.get(env_name)
+        if secret_value is None:
+            # Try without prefix for backward compatibility
+            secret_value = os.environ.get(name.upper().replace("-", "_"))
+
+        if secret_value is None:
+            raise SecretNotFoundError(
+                f"Secret '{name}' not found in environment variables"
+            )
+
+        return secret_value
+
+    def list_secrets(self) -> List[str]:
+        """List all secrets available in environment.
+
+        Returns:
+            List of secret names (without prefix)
+        """
+        secrets = []
+        for key in os.environ:
+            if key.startswith(self.prefix):
+                # Remove prefix and convert back to secret name format
+                secret_name = key[len(self.prefix) :].lower().replace("_", "-")
+                secrets.append(secret_name)
+        return secrets
+
+
+class VaultSecretProvider(SecretProvider):
+    """Secret provider for HashiCorp Vault.
+
+    This provider integrates with HashiCorp Vault for enterprise secret management.
+    """
+
+    def __init__(self, vault_url: str, vault_token: str, mount_path: str = "secret"):
+        """Initialize Vault secret provider.
+
+        Args:
+            vault_url: Vault server URL
+            vault_token: Vault authentication token
+            mount_path: Vault mount path for secrets
+        """
+        self.vault_url = vault_url
+        self.vault_token = vault_token
+        self.mount_path = mount_path
+        self._client = None
+
+    @property
+    def client(self):
+        """Lazy initialization of Vault client."""
+        if self._client is None:
+            try:
+                import hvac
+
+                self._client = hvac.Client(url=self.vault_url, token=self.vault_token)
+            except ImportError:
+                raise RuntimeError(
+                    "hvac library not installed. Install with: pip install hvac"
+                )
+        return self._client
+
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Get secret from Vault.
+
+        Args:
+            name: Secret path in Vault
+            version: Optional version (for KV v2)
+
+        Returns:
+            Secret value
+        """
+        try:
+            # Try KV v2 first
+            response = self.client.secrets.kv.v2.read_secret_version(
+                path=name, version=version, mount_point=self.mount_path
+            )
+            return response["data"]["data"]["value"]
+        except Exception:
+            # Fall back to KV v1
+            response = self.client.secrets.kv.v1.read_secret(
+                path=name, mount_point=self.mount_path
+            )
+            return response["data"]["value"]
+
+    def list_secrets(self) -> List[str]:
+        """List all secrets in Vault.
+
+        Returns:
+            List of secret paths
+        """
+        try:
+            response = self.client.secrets.kv.v2.list_secrets(
+                path="", mount_point=self.mount_path
+            )
+            return response["data"]["keys"]
+        except Exception:
+            # Fall back to KV v1
+            response = self.client.secrets.kv.v1.list_secrets(
+                path="", mount_point=self.mount_path
+            )
+            return response["data"]["keys"]
+
+
+class AWSSecretProvider(SecretProvider):
+    """Secret provider for AWS Secrets Manager.
+
+    This provider integrates with AWS Secrets Manager for cloud-native secret management.
+    """
+
+    def __init__(self, region_name: str = "us-east-1"):
+        """Initialize AWS secret provider.
+
+        Args:
+            region_name: AWS region
+        """
+        self.region_name = region_name
+        self._client = None
+
+    @property
+    def client(self):
+        """Lazy initialization of AWS client."""
+        if self._client is None:
+            try:
+                import boto3
+
+                self._client = boto3.client(
+                    "secretsmanager", region_name=self.region_name
+                )
+            except ImportError:
+                raise RuntimeError(
+                    "boto3 library not installed. Install with: pip install boto3"
+                )
+        return self._client
+
+    def get_secret(self, name: str, version: Optional[str] = None) -> str:
+        """Get secret from AWS Secrets Manager.
+
+        Args:
+            name: Secret name in AWS
+            version: Optional version ID
+
+        Returns:
+            Secret value
+        """
+        kwargs = {"SecretId": name}
+        if version:
+            kwargs["VersionId"] = version
+
+        response = self.client.get_secret_value(**kwargs)
+        return response["SecretString"]
+
+    def list_secrets(self) -> List[str]:
+        """List all secrets in AWS Secrets Manager.
+
+        Returns:
+            List of secret names
+        """
+        response = self.client.list_secrets()
+        return [secret["Name"] for secret in response["SecretList"]]
+
+
+class SecretNotFoundError(Exception):
+    """Raised when a secret cannot be found."""
+
+    pass
+
+
+class SecretProviderError(Exception):
+    """Raised when a secret provider operation fails."""
+
+    pass

kailash/workflow/builder.py

@@ -380,18 +380,89 @@ class WorkflowBuilder:
             WorkflowValidationError: If nodes don't exist
             ConnectionError: If connection is invalid
         """
+        # Enhanced error messages with helpful suggestions
         if from_node not in self.nodes:
-            raise WorkflowValidationError(
-                f"Source node '{from_node}' not found in workflow"
-            )
+            available_nodes = list(self.nodes.keys())
+            similar_nodes = [
+                n
+                for n in available_nodes
+                if from_node.lower() in n.lower() or n.lower() in from_node.lower()
+            ]
+
+            error_msg = f"Source node '{from_node}' not found in workflow."
+            if available_nodes:
+                error_msg += f"\nAvailable nodes: {available_nodes}"
+            if similar_nodes:
+                error_msg += f"\nDid you mean: {similar_nodes}?"
+            error_msg += "\n\nTip: Use workflow.add_node() to create nodes before connecting them."
+            error_msg += f"\nExample: workflow.add_node('CSVReaderNode', '{from_node}', {{'file_path': 'data.csv'}})"
+
+            raise WorkflowValidationError(error_msg)
+
         if to_node not in self.nodes:
-            raise WorkflowValidationError(
-                f"Target node '{to_node}' not found in workflow"
+            available_nodes = list(self.nodes.keys())
+            similar_nodes = [
+                n
+                for n in available_nodes
+                if to_node.lower() in n.lower() or n.lower() in to_node.lower()
+            ]
+
+            error_msg = f"Target node '{to_node}' not found in workflow."
+            if available_nodes:
+                error_msg += f"\nAvailable nodes: {available_nodes}"
+            if similar_nodes:
+                error_msg += f"\nDid you mean: {similar_nodes}?"
+            error_msg += "\n\nTip: Use workflow.add_node() to create nodes before connecting them."
+            error_msg += f"\nExample: workflow.add_node('PythonCodeNode', '{to_node}', {{'code': 'result = data'}})"
+
+            raise WorkflowValidationError(error_msg)
+
+        # Self-connection check with helpful message
+        if from_node == to_node:
+            raise ConnectionError(
+                f"Cannot connect node '{from_node}' to itself.\n"
+                f"Tip: Consider using intermediate nodes or different port names.\n"
+                f"Example: Create a separate processing node between input and output."
             )
 
-        # Self-connection check
-        if from_node == to_node:
-            raise ConnectionError(f"Cannot connect node '{from_node}' to itself")
+        # REFINED: Enhanced duplicate connection detection
+        for existing_conn in self.connections:
+            if (
+                existing_conn["from_node"] == from_node
+                and existing_conn["from_output"] == from_output
+                and existing_conn["to_node"] == to_node
+                and existing_conn["to_input"] == to_input
+            ):
+                raise ConnectionError(
+                    f"Duplicate connection detected: {from_node}.{from_output} -> {to_node}.{to_input}\n"
+                    f"This connection already exists in the workflow.\n"
+                    f"Tip: Remove the duplicate add_connection() call or use different port names.\n"
+                    f"Current connections: {len(self.connections)} total"
+                )
+
+        # Enhanced port validation with suggestions
+        common_output_ports = [
+            "data",
+            "result",
+            "output",
+            "response",
+            "content",
+            "value",
+        ]
+        common_input_ports = ["data", "input", "input_data", "content", "value"]
+
+        # Log port usage patterns for debugging
+        if from_output not in common_output_ports:
+            logger.debug(
+                f"Using non-standard output port '{from_output}' on node '{from_node}'"
+            )
+            logger.debug(f"Common output ports: {common_output_ports}")
+
+        if to_input not in common_input_ports:
+            logger.debug(
+                f"Using non-standard input port '{to_input}' on node '{to_node}'"
+            )
+            logger.debug(f"Common input ports: {common_input_ports}")
 
         # Add connection to list
         connection = {
@@ -402,7 +473,15 @@ class WorkflowBuilder:
         }
         self.connections.append(connection)
 
-        logger.info(f"Connected '{from_node}.{from_output}' to '{to_node}.{to_input}'")
+        logger.info(f"Connected '{from_node}.{from_output}' -> '{to_node}.{to_input}'")
+
+        # Provide helpful tips for common connection patterns
+        if from_output == to_input == "data":
+            logger.debug("Using standard data flow connection pattern")
+        elif from_output in ["result", "output"] and to_input in ["data", "input"]:
+            logger.debug("Using result-to-input connection pattern")
+        else:
+            logger.debug(f"Using custom port mapping: {from_output} -> {to_input}")
         return self
 
     def connect(