kailash 0.2.2__py3-none-any.whl → 0.3.1__py3-none-any.whl

kailash/runtime/docker.py

@@ -21,7 +21,7 @@ import subprocess
 import sys
 import tempfile
 from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Any
 
 from kailash.nodes.base import Node
 
@@ -50,8 +50,8 @@ class DockerNodeWrapper:
         node: Node,
         node_id: str,
         base_image: str = "python:3.11-slim",
-        work_dir: Optional[Path] = None,
-        sdk_path: Optional[Path] = None,
+        work_dir: Path | None = None,
+        sdk_path: Path | None = None,
     ):
         """
         Initialize a Docker node wrapper.
@@ -174,7 +174,7 @@ def main():
     logger.info(f"Loaded configuration for {node_data['class']} node")
 
     # Load runtime inputs if available
-    input_path = Path("/examples/data/input/inputs.json")
+    input_path = Path("/data/inputs/json/inputs.json")
     runtime_inputs = {}
     if input_path.exists():
         logger.info(f"Loading inputs from {input_path}")
@@ -206,7 +206,7 @@ def main():
     except Exception as e:
         logger.error(f"Node execution failed: {e}")
         # Save error information
-        with open("/examples/data/output/error.json", 'w') as f:
+        with open("/data/outputs/json/error.json", 'w') as f:
             json.dump({
                 "error": str(e),
                 "type": e.__class__.__name__
@@ -216,7 +216,7 @@ def main():
     # Save results
     logger.info("Saving execution results")
     try:
-        result_path = Path("/examples/data/output/result.json")
+        result_path = Path("/data/outputs/json/result.json")
         with open(result_path, 'w') as f:
             # Handle non-serializable objects with basic conversion
             try:
@@ -335,7 +335,7 @@ ENTRYPOINT ["/app/entrypoint.py"]
             logger.error(error_msg)
             raise RuntimeError(error_msg)
 
-    def prepare_inputs(self, inputs: Dict[str, Any]):
+    def prepare_inputs(self, inputs: dict[str, Any]):
         """
         Prepare inputs for node execution.
 
@@ -349,8 +349,8 @@ ENTRYPOINT ["/app/entrypoint.py"]
     def run_container(
         self,
         network: str = None,
-        env_vars: Dict[str, str] = None,
-        resource_limits: Dict[str, str] = None,
+        env_vars: dict[str, str] = None,
+        resource_limits: dict[str, str] = None,
     ) -> bool:
         """
         Run the node in a Docker container.
@@ -416,13 +416,13 @@ ENTRYPOINT ["/app/entrypoint.py"]
             # Check if there's an error file
             error_file = self.output_dir / "error.json"
             if error_file.exists():
-                with open(error_file, "r") as f:
+                with open(error_file) as f:
                     error_data = json.load(f)
                     error_msg = f"Node execution error: {error_data.get('error', 'Unknown error')}"
 
             raise NodeExecutionError(error_msg)
 
-    def get_results(self) -> Dict[str, Any]:
+    def get_results(self) -> dict[str, Any]:
         """
         Get the results of node execution.
 
@@ -431,12 +431,12 @@ ENTRYPOINT ["/app/entrypoint.py"]
         """
         result_file = self.output_dir / "result.json"
         if result_file.exists():
-            with open(result_file, "r") as f:
+            with open(result_file) as f:
                 return json.load(f)
 
         error_file = self.output_dir / "error.json"
         if error_file.exists():
-            with open(error_file, "r") as f:
+            with open(error_file) as f:
                 error_data = json.load(f)
                 raise NodeExecutionError(
                     f"Node {self.node_id} execution failed: {error_data.get('error', 'Unknown error')}"
@@ -464,10 +464,10 @@ class DockerRuntime:
         self,
         base_image: str = "python:3.11-slim",
         network_name: str = "kailash-network",
-        work_dir: Optional[str] = None,
-        sdk_path: Optional[str] = None,
-        resource_limits: Optional[Dict[str, str]] = None,
-        task_manager: Optional[TaskManager] = None,
+        work_dir: str | None = None,
+        sdk_path: str | None = None,
+        resource_limits: dict[str, str] | None = None,
+        task_manager: TaskManager | None = None,
     ):
         """
         Initialize the Docker runtime.
@@ -516,14 +516,14 @@ class DockerRuntime:
         # Track node wrappers
         self.node_wrappers = {}
 
-    def _create_task_run(self, workflow: Workflow) -> Optional[str]:
+    def _create_task_run(self, workflow: Workflow) -> str | None:
         """Create a task run if task manager is available."""
         if self.task_manager:
             return self.task_manager.create_run(workflow.name)
         return None
 
     def _update_task_status(
-        self, run_id: Optional[str], node_id: str, status: str, output: Any = None
+        self, run_id: str | None, node_id: str, status: str, output: Any = None
     ):
         """Update task status if task manager is available."""
         if self.task_manager and run_id:
@@ -534,9 +534,7 @@ class DockerRuntime:
                 )
                 self.task_manager.update_run_status(run_id, "failed", error_msg)
 
-    def _complete_task_run(
-        self, run_id: Optional[str], status: str, result: Any = None
-    ):
+    def _complete_task_run(self, run_id: str | None, status: str, result: Any = None):
         """Complete task run if task manager is available."""
         if self.task_manager and run_id:
             if status == "completed":
@@ -567,9 +565,9 @@ class DockerRuntime:
     def execute(
         self,
         workflow: Workflow,
-        inputs: Dict[str, Dict[str, Any]] = None,
-        node_resource_limits: Dict[str, Dict[str, str]] = None,
-    ) -> Tuple[Dict[str, Dict[str, Any]], str]:
+        inputs: dict[str, dict[str, Any]] = None,
+        node_resource_limits: dict[str, dict[str, str]] = None,
+    ) -> tuple[dict[str, dict[str, Any]], str]:
         """
         Execute a workflow using Docker containers.
 
@@ -590,7 +588,7 @@ class DockerRuntime:
 
         try:
             # Validate workflow
-            workflow.validate()
+            workflow.validate(runtime_parameters=inputs)
 
             # Get execution order
             execution_order = workflow.get_execution_order()

kailash/runtime/local.py

@@ -40,8 +40,8 @@ Examples:
 """
 
 import logging
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Tuple
+from datetime import UTC, datetime
+from typing import Any
 
 import networkx as nx
 
@@ -90,9 +90,9 @@ class LocalRuntime:
     def execute(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager] = None,
-        parameters: Optional[Dict[str, Dict[str, Any]]] = None,
-    ) -> Tuple[Dict[str, Any], Optional[str]]:
+        task_manager: TaskManager | None = None,
+        parameters: dict[str, dict[str, Any]] | None = None,
+    ) -> tuple[dict[str, Any], str | None]:
         """Execute a workflow locally.
 
         Args:
@@ -113,8 +113,8 @@ class LocalRuntime:
         run_id = None
 
         try:
-            # Validate workflow
-            workflow.validate()
+            # Validate workflow with runtime parameters (Session 061)
+            workflow.validate(runtime_parameters=parameters)
 
             # Initialize tracking
             if task_manager:
@@ -197,10 +197,10 @@ class LocalRuntime:
     def _execute_workflow(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager],
-        run_id: Optional[str],
-        parameters: Dict[str, Dict[str, Any]],
-    ) -> Dict[str, Any]:
+        task_manager: TaskManager | None,
+        run_id: str | None,
+        parameters: dict[str, dict[str, Any]],
+    ) -> dict[str, Any]:
         """Execute the workflow nodes in topological order.
 
         Args:
@@ -273,7 +273,7 @@ class LocalRuntime:
                         run_id=run_id,
                         node_id=node_id,
                         node_type=node_instance.__class__.__name__,
-                        started_at=datetime.now(timezone.utc),
+                        started_at=datetime.now(UTC),
                         metadata=node_metadata,
                     )
                     # Start the task
@@ -296,6 +296,10 @@ class LocalRuntime:
                     parameters=parameters.get(node_id, {}),
                 )
 
+                # Update node config with parameters (Session 061: direct config update)
+                {**node_instance.config, **parameters.get(node_id, {})}
+                node_instance.config.update(parameters.get(node_id, {}))
+
                 if self.debug:
                     self.logger.debug(f"Node {node_id} inputs: {inputs}")
 
@@ -325,7 +329,7 @@ class LocalRuntime:
                         task.task_id,
                         TaskStatus.COMPLETED,
                         result=outputs,
-                        ended_at=datetime.now(timezone.utc),
+                        ended_at=datetime.now(UTC),
                         metadata={"execution_time": performance_metrics.duration},
                     )
 
@@ -346,7 +350,7 @@ class LocalRuntime:
                         task.task_id,
                         TaskStatus.FAILED,
                         error=str(e),
-                        ended_at=datetime.now(timezone.utc),
+                        ended_at=datetime.now(UTC),
                     )
 
                 # Determine if we should continue
@@ -371,9 +375,9 @@ class LocalRuntime:
         workflow: Workflow,
         node_id: str,
         node_instance: Node,
-        node_outputs: Dict[str, Dict[str, Any]],
-        parameters: Dict[str, Any],
-    ) -> Dict[str, Any]:
+        node_outputs: dict[str, dict[str, Any]],
+        parameters: dict[str, Any],
+    ) -> dict[str, Any]:
         """Prepare inputs for a node execution.
 
         Args:
@@ -391,21 +395,36 @@ class LocalRuntime:
         """
         inputs = {}
 
-        # Start with node configuration
-        inputs.update(node_instance.config)
+        # NOTE: Node configuration is handled separately in configure() call
+        # Only add runtime inputs and data from connected nodes here
+
+        # Add runtime parameters (those not used for node configuration)
+        # Map specific runtime parameters for known node types
+        if "consumer_timeout_ms" in parameters:
+            inputs["timeout_ms"] = parameters["consumer_timeout_ms"]
+
+        # Add other potential runtime parameters that are not configuration
+        runtime_param_names = {"max_messages", "timeout_ms", "limit", "offset"}
+        for param_name, param_value in parameters.items():
+            if param_name in runtime_param_names:
+                inputs[param_name] = param_value
 
         # Add connected inputs from other nodes
         for edge in workflow.graph.in_edges(node_id, data=True):
             source_node_id = edge[0]
             mapping = edge[2].get("mapping", {})
 
-            print(f"LOCAL RUNTIME DEBUG: Processing edge {source_node_id} -> {node_id}")
-            print(f"  Edge data: {edge[2]}")
-            print(f"  Mapping: {mapping}")
+            if self.debug:
+                self.logger.debug(f"Processing edge {source_node_id} -> {node_id}")
+                self.logger.debug(f"  Edge data: {edge[2]}")
+                self.logger.debug(f"  Mapping: {mapping}")
 
             if source_node_id in node_outputs:
                 source_outputs = node_outputs[source_node_id]
-                print(f"  Source outputs: {list(source_outputs.keys())}")
+                if self.debug:
+                    self.logger.debug(
+                        f"  Source outputs: {list(source_outputs.keys())}"
+                    )
 
                 # Check if the source node failed
                 if isinstance(source_outputs, dict) and source_outputs.get("failed"):
@@ -416,19 +435,24 @@ class LocalRuntime:
                 for source_key, target_key in mapping.items():
                     if source_key in source_outputs:
                         inputs[target_key] = source_outputs[source_key]
-                        print(
-                            f"  MAPPED: {source_key} -> {target_key} (type: {type(source_outputs[source_key])})"
-                        )
+                        if self.debug:
+                            self.logger.debug(
+                                f"  MAPPED: {source_key} -> {target_key} (type: {type(source_outputs[source_key])})"
+                            )
                     else:
-                        print(
-                            f"  MISSING: {source_key} not in {list(source_outputs.keys())}"
-                        )
+                        if self.debug:
+                            self.logger.debug(
+                                f"  MISSING: {source_key} not in {list(source_outputs.keys())}"
+                            )
                         self.logger.warning(
                             f"Source output '{source_key}' not found in node '{source_node_id}'. "
                             f"Available outputs: {list(source_outputs.keys())}"
                         )
             else:
-                print(f"  No outputs found for source node {source_node_id}")
+                if self.debug:
+                    self.logger.debug(
+                        f"  No outputs found for source node {source_node_id}"
+                    )
 
         # Apply parameter overrides
         inputs.update(parameters)
@@ -452,7 +476,7 @@ class LocalRuntime:
         # Future: implement configurable error handling policies
         return has_dependents
 
-    def validate_workflow(self, workflow: Workflow) -> List[str]:
+    def validate_workflow(self, workflow: Workflow) -> list[str]:
         """Validate a workflow before execution.
 
         Args:

kailash/runtime/parallel.py

@@ -8,8 +8,8 @@ import asyncio
 import logging
 import time
 from collections import deque
-from datetime import datetime, timezone
-from typing import Any, Deque, Dict, Optional, Set, Tuple
+from datetime import UTC, datetime
+from typing import Any
 
 import networkx as nx
 
@@ -66,9 +66,9 @@ class ParallelRuntime:
     async def execute(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager] = None,
-        parameters: Optional[Dict[str, Dict[str, Any]]] = None,
-    ) -> Tuple[Dict[str, Any], Optional[str]]:
+        task_manager: TaskManager | None = None,
+        parameters: dict[str, dict[str, Any]] | None = None,
+    ) -> tuple[dict[str, Any], str | None]:
         """Execute a workflow with parallel node execution.
 
         Args:
@@ -91,7 +91,7 @@ class ParallelRuntime:
 
         try:
             # Validate workflow
-            workflow.validate()
+            workflow.validate(runtime_parameters=parameters)
 
             # Initialize semaphore for concurrent execution control
             self.semaphore = asyncio.Semaphore(self.max_workers)
@@ -159,10 +159,10 @@ class ParallelRuntime:
     async def _execute_workflow_parallel(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager],
-        run_id: Optional[str],
-        parameters: Dict[str, Dict[str, Any]],
-    ) -> Dict[str, Any]:
+        task_manager: TaskManager | None,
+        run_id: str | None,
+        parameters: dict[str, dict[str, Any]],
+    ) -> dict[str, Any]:
         """Execute the workflow nodes in parallel where possible.
 
         This method uses a dynamic scheduling approach to run independent nodes
@@ -323,11 +323,11 @@ class ParallelRuntime:
         self,
         workflow: Workflow,
         node_id: str,
-        node_outputs: Dict[str, Dict[str, Any]],
-        parameters: Dict[str, Any],
-        task_manager: Optional[TaskManager],
-        run_id: Optional[str],
-    ) -> Tuple[Dict[str, Any], bool]:
+        node_outputs: dict[str, dict[str, Any]],
+        parameters: dict[str, Any],
+        task_manager: TaskManager | None,
+        run_id: str | None,
+    ) -> tuple[dict[str, Any], bool]:
         """Execute a single node asynchronously.
 
         Args:
@@ -359,7 +359,7 @@ class ParallelRuntime:
                     run_id=run_id,
                     node_id=node_id,
                     node_type=node_instance.__class__.__name__,
-                    started_at=datetime.now(timezone.utc),
+                    started_at=datetime.now(UTC),
                 )
         except Exception as e:
             self.logger.warning(f"Failed to create task for node '{node_id}': {e}")
@@ -398,7 +398,7 @@ class ParallelRuntime:
                     async def execute_with_metrics():
                         with collector.collect(node_id=node_id) as context:
                             result = await loop.run_in_executor(
-                                None, lambda: node_instance.execute(**inputs)
+                                None, lambda: node_instance.run(**inputs)
                             )
                             return result, context.result()
 
@@ -409,7 +409,7 @@ class ParallelRuntime:
                     task.update_status(
                         TaskStatus.COMPLETED,
                         result=outputs,
-                        ended_at=datetime.now(timezone.utc),
+                        ended_at=datetime.now(UTC),
                         metadata={"execution_time": performance_metrics.duration},
                     )
 
@@ -431,7 +431,7 @@ class ParallelRuntime:
             # Update task status
             if task:
                 task.update_status(
-                    TaskStatus.FAILED, error=str(e), ended_at=datetime.now(timezone.utc)
+                    TaskStatus.FAILED, error=str(e), ended_at=datetime.now(UTC)
                 )
 
             # Return error result
@@ -448,9 +448,9 @@ class ParallelRuntime:
         workflow: Workflow,
         node_id: str,
         node_instance: Any,
-        node_outputs: Dict[str, Dict[str, Any]],
-        parameters: Dict[str, Any],
-    ) -> Dict[str, Any]:
+        node_outputs: dict[str, dict[str, Any]],
+        parameters: dict[str, Any],
+    ) -> dict[str, Any]:
         """Prepare inputs for a node execution.
 
         Args:
@@ -520,9 +520,9 @@ class ParallelRuntime:
         self,
         workflow: Workflow,
         failed_node: str,
-        failed_nodes: Set[str],
-        pending_nodes: Set[str],
-        ready_nodes: Deque[str],
+        failed_nodes: set[str],
+        pending_nodes: set[str],
+        ready_nodes: deque[str],
     ) -> None:
         """Mark all dependent nodes as failed.
 

kailash/runtime/parallel_cyclic.py

@@ -2,8 +2,8 @@
 
 import logging
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Set, Tuple
+from datetime import UTC, datetime
+from typing import Any
 
 import networkx as nx
 
@@ -56,10 +56,10 @@ class ParallelCyclicRuntime:
     def execute(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager] = None,
-        parameters: Optional[Dict[str, Dict[str, Any]]] = None,
-        parallel_nodes: Optional[Set[str]] = None,
-    ) -> Tuple[Dict[str, Any], Optional[str]]:
+        task_manager: TaskManager | None = None,
+        parameters: dict[str, dict[str, Any]] | None = None,
+        parallel_nodes: set[str] | None = None,
+    ) -> tuple[dict[str, Any], str | None]:
         """Execute a workflow with parallel and cyclic support.
 
         Args:
@@ -80,7 +80,7 @@ class ParallelCyclicRuntime:
 
         try:
             # Validate workflow
-            workflow.validate()
+            workflow.validate(runtime_parameters=parameters)
 
             # Check for cycles first
             if self.enable_cycles and workflow.has_cycles():
@@ -108,9 +108,9 @@ class ParallelCyclicRuntime:
     def _execute_cyclic_workflow(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager],
-        parameters: Optional[Dict[str, Dict[str, Any]]],
-    ) -> Tuple[Dict[str, Any], str]:
+        task_manager: TaskManager | None,
+        parameters: dict[str, dict[str, Any]] | None,
+    ) -> tuple[dict[str, Any], str]:
         """Execute a cyclic workflow with potential parallel optimizations.
 
         Args:
@@ -141,10 +141,10 @@ class ParallelCyclicRuntime:
     def _execute_parallel_dag(
         self,
         workflow: Workflow,
-        task_manager: Optional[TaskManager],
-        parameters: Optional[Dict[str, Dict[str, Any]]],
-        parallel_nodes: Optional[Set[str]],
-    ) -> Tuple[Dict[str, Any], Optional[str]]:
+        task_manager: TaskManager | None,
+        parameters: dict[str, dict[str, Any]] | None,
+        parallel_nodes: set[str] | None,
+    ) -> tuple[dict[str, Any], str | None]:
         """Execute a DAG workflow with parallel node execution.
 
         Args:
@@ -252,8 +252,8 @@ class ParallelCyclicRuntime:
             raise
 
     def _analyze_parallel_groups(
-        self, workflow: Workflow, parallel_nodes: Optional[Set[str]]
-    ) -> List[List[str]]:
+        self, workflow: Workflow, parallel_nodes: set[str] | None
+    ) -> list[list[str]]:
         """Analyze workflow to identify groups of nodes that can be executed in parallel.
 
         Args:
@@ -320,11 +320,11 @@ class ParallelCyclicRuntime:
         self,
         workflow: Workflow,
         node_id: str,
-        previous_results: Dict[str, Any],
-        parameters: Optional[Dict[str, Dict[str, Any]]],
-        task_manager: Optional[TaskManager],
-        run_id: Optional[str],
-    ) -> Dict[str, Any]:
+        previous_results: dict[str, Any],
+        parameters: dict[str, dict[str, Any]] | None,
+        task_manager: TaskManager | None,
+        run_id: str | None,
+    ) -> dict[str, Any]:
         """Execute a single node in isolation.
 
         Args:
@@ -356,7 +356,7 @@ class ParallelCyclicRuntime:
                     run_id=run_id,
                     node_id=node_id,
                     node_type=node_instance.__class__.__name__,
-                    started_at=datetime.now(timezone.utc),
+                    started_at=datetime.now(UTC),
                     metadata={},
                 )
                 if task:
@@ -380,7 +380,7 @@ class ParallelCyclicRuntime:
             # Execute node with metrics collection
             collector = MetricsCollector()
             with collector.collect(node_id=node_id) as metrics_context:
-                outputs = node_instance.execute(**inputs)
+                outputs = node_instance.run(**inputs)
 
             # Get performance metrics
             performance_metrics = metrics_context.result()
@@ -397,7 +397,7 @@ class ParallelCyclicRuntime:
                     task.task_id,
                     TaskStatus.COMPLETED,
                     result=outputs,
-                    ended_at=datetime.now(timezone.utc),
+                    ended_at=datetime.now(UTC),
                     metadata={"execution_time": performance_metrics.duration},
                 )
                 task_manager.update_task_metrics(task.task_id, task_metrics)
@@ -415,7 +415,7 @@ class ParallelCyclicRuntime:
                     task.task_id,
                     TaskStatus.FAILED,
                     error=str(e),
-                    ended_at=datetime.now(timezone.utc),
+                    ended_at=datetime.now(UTC),
                 )
 
             self.logger.error(f"Node {node_id} failed: {e}", exc_info=self.debug)
@@ -428,9 +428,9 @@ class ParallelCyclicRuntime:
         workflow: Workflow,
         node_id: str,
         node_instance: Node,
-        previous_results: Dict[str, Any],
-        parameters: Dict[str, Any],
-    ) -> Dict[str, Any]:
+        previous_results: dict[str, Any],
+        parameters: dict[str, Any],
+    ) -> dict[str, Any]:
         """Prepare inputs for a node execution in parallel context.
 
         Args:
@@ -509,7 +509,7 @@ class ParallelCyclicRuntime:
             return False
 
     def _should_stop_on_group_error(
-        self, workflow: Workflow, failed_node: str, node_group: List[str]
+        self, workflow: Workflow, failed_node: str, node_group: list[str]
     ) -> bool:
         """Determine if execution should stop when a node in a parallel group fails.
 

kailash/runtime/runner.py

@@ -1,7 +1,7 @@
 """Main runner for workflow execution."""
 
 import logging
-from typing import Any, Dict, Optional, Tuple
+from typing import Any
 
 from kailash.runtime.local import LocalRuntime
 from kailash.tracking import TaskManager
@@ -11,7 +11,7 @@ from kailash.workflow import Workflow
 class WorkflowRunner:
     """High-level interface for running workflows."""
 
-    def __init__(self, debug: bool = False, task_manager: Optional[TaskManager] = None):
+    def __init__(self, debug: bool = False, task_manager: TaskManager | None = None):
         """Initialize the workflow runner.
 
         Args:
@@ -36,9 +36,9 @@ class WorkflowRunner:
     def run(
         self,
         workflow: Workflow,
-        parameters: Optional[Dict[str, Dict[str, Any]]] = None,
+        parameters: dict[str, dict[str, Any]] | None = None,
         runtime_type: str = "local",
-    ) -> Tuple[Dict[str, Any], str]:
+    ) -> tuple[dict[str, Any], str]:
         """Run a workflow.
 
         Args:
@@ -82,7 +82,7 @@ class WorkflowRunner:
         runtime = LocalRuntime(debug=self.debug)
         return runtime.validate_workflow(workflow)
 
-    def get_run_status(self, run_id: str) -> Dict[str, Any]:
+    def get_run_status(self, run_id: str) -> dict[str, Any]:
         """Get status of a workflow run.
 
         Args:
@@ -94,7 +94,7 @@ class WorkflowRunner:
         return self.task_manager.get_run_status(run_id)
 
     def get_run_history(
-        self, workflow_name: Optional[str] = None, limit: int = 10
+        self, workflow_name: str | None = None, limit: int = 10
     ) -> list:
         """Get run history.