kailash 0.7.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

kailash/workflow/builder.py

@@ -4,12 +4,10 @@ import logging
 import uuid
 from typing import TYPE_CHECKING, Any
 
+from kailash.nodes.base import Node
 from kailash.sdk_exceptions import ConnectionError, WorkflowValidationError
 from kailash.workflow.graph import Workflow
 
-if TYPE_CHECKING:
-    from kailash.nodes.base import Node
-
 logger = logging.getLogger(__name__)
 
 
@@ -21,6 +19,9 @@ class WorkflowBuilder:
         self.nodes: dict[str, dict[str, Any]] = {}
         self.connections: list[dict[str, str]] = []
         self._metadata: dict[str, Any] = {}
+        # Parameter injection capabilities
+        self.workflow_parameters: dict[str, Any] = {}
+        self.parameter_mappings: dict[str, dict[str, str]] = {}
 
     def add_node(self, *args, **kwargs) -> str:
         """
@@ -74,8 +75,6 @@ class WorkflowBuilder:
                 # Pattern: add_node(NodeClass)
                 return self._add_node_alternative(args[0], None, **kwargs)
             else:
-                from kailash.nodes.base import Node
-
                 if isinstance(args[0], Node):
                     # Pattern: add_node(node_instance)
                     return self._add_node_instance(args[0], None)
@@ -111,8 +110,6 @@ class WorkflowBuilder:
 
         elif len(args) >= 2:
             # Check if first arg is a Node instance
-            from kailash.nodes.base import Node
-
             if isinstance(args[0], Node):
                 # Pattern 4: Instance - add_node(node_instance, "node_id") or add_node(node_instance, "node_id", config)
                 # Config is ignored for instances
@@ -149,8 +146,6 @@ class WorkflowBuilder:
         """Handle legacy fluent API pattern: add_node('node_id', NodeClass, param=value)"""
         import warnings
 
-        from kailash.nodes.base import Node
-
         # If it's a class, validate it's a Node subclass
         if isinstance(node_class_or_type, type) and not issubclass(
             node_class_or_type, Node
@@ -183,8 +178,6 @@ class WorkflowBuilder:
         """Handle alternative pattern: add_node(NodeClass, 'node_id', param=value)"""
         import warnings
 
-        from kailash.nodes.base import Node
-
         # Validate that node_class is actually a Node subclass
         if not isinstance(node_class, type) or not issubclass(node_class, Node):
             raise WorkflowValidationError(
@@ -267,9 +260,6 @@ class WorkflowBuilder:
                 f"Node ID '{node_id}' already exists in workflow"
             )
 
-        # Import Node here to avoid circular imports
-        from kailash.nodes.base import Node
-
         # Handle different input types
         if isinstance(node_type, str):
             # String node type name
@@ -606,12 +596,108 @@ class WorkflowBuilder:
                     f"Failed to connect '{from_node}' to '{to_node}': {e}"
                 ) from e
 
+        # Parameter injection: Find nodes without incoming connections and inject parameters
+        if self.workflow_parameters:
+            nodes_with_inputs = set()
+            for conn in self.connections:
+                if not conn.get("is_workflow_input"):
+                    nodes_with_inputs.add(conn["to_node"])
+
+            nodes_without_inputs = set(self.nodes.keys()) - nodes_with_inputs
+
+            # For each node without inputs, check if it needs workflow parameters
+            for node_id in nodes_without_inputs:
+                node_info = self.nodes[node_id]
+                node_instance = workflow.get_node(node_id)
+
+                if hasattr(node_instance, "get_parameters"):
+                    params = node_instance.get_parameters()
+
+                    # Check which required parameters are missing from config
+                    for param_name, param_def in params.items():
+                        if param_def.required and param_name not in node_info["config"]:
+                            # Check if this parameter should come from workflow parameters
+                            if param_name in self.workflow_parameters:
+                                # Add to node config
+                                node_info["config"][param_name] = (
+                                    self.workflow_parameters[param_name]
+                                )
+                            elif node_id in self.parameter_mappings:
+                                # Check parameter mappings
+                                mapping = self.parameter_mappings[node_id]
+                                if param_name in mapping:
+                                    workflow_param = mapping[param_name]
+                                    if workflow_param in self.workflow_parameters:
+                                        node_info["config"][param_name] = (
+                                            self.workflow_parameters[workflow_param]
+                                        )
+
+        # Store workflow parameters in metadata for runtime reference
+        workflow.metadata["workflow_parameters"] = self.workflow_parameters
+        workflow.metadata["parameter_mappings"] = self.parameter_mappings
+
         logger.info(
             f"Built workflow '{workflow_id}' with "
             f"{len(self.nodes)} nodes and {len(self.connections)} connections"
         )
         return workflow
 
+    def set_workflow_parameters(self, **parameters) -> "WorkflowBuilder":
+        """
+        Set default parameters that will be passed to all nodes.
+
+        Args:
+            **parameters: Key-value pairs of workflow-level parameters
+
+        Returns:
+            Self for chaining
+        """
+        self.workflow_parameters.update(parameters)
+        return self
+
+    def add_parameter_mapping(
+        self, node_id: str, mappings: dict[str, str]
+    ) -> "WorkflowBuilder":
+        """
+        Add parameter mappings for a specific node.
+
+        Args:
+            node_id: Node to configure
+            mappings: Dict mapping workflow param names to node param names
+
+        Returns:
+            Self for chaining
+        """
+        if node_id not in self.parameter_mappings:
+            self.parameter_mappings[node_id] = {}
+        self.parameter_mappings[node_id].update(mappings)
+        return self
+
+    def add_input_connection(
+        self, to_node: str, to_input: str, from_workflow_param: str
+    ) -> "WorkflowBuilder":
+        """
+        Connect a workflow parameter directly to a node input.
+
+        Args:
+            to_node: Target node ID
+            to_input: Input parameter name on the node
+            from_workflow_param: Workflow parameter name
+
+        Returns:
+            Self for chaining
+        """
+        # Add a special connection type for workflow inputs
+        connection = {
+            "from_node": "__workflow_input__",
+            "from_output": from_workflow_param,
+            "to_node": to_node,
+            "to_input": to_input,
+            "is_workflow_input": True,
+        }
+        self.connections.append(connection)
+        return self
+
     def clear(self) -> "WorkflowBuilder":
         """
         Clear builder state.
@@ -622,6 +708,8 @@ class WorkflowBuilder:
         self.nodes = {}
         self.connections = []
         self._metadata = {}
+        self.workflow_parameters = {}
+        self.parameter_mappings = {}
         return self
 
     @classmethod

kailash/workflow/cyclic_runner.py

@@ -184,7 +184,7 @@ class CyclicWorkflowExecutor:
         if not workflow.has_cycles():
             # No cycles, use standard DAG execution
             logger.info("No cycles detected, using standard DAG execution")
-            return self.dag_runner.run(workflow, parameters), run_id
+            return self.dag_runner.execute(workflow, parameters), run_id
 
         # Execute with cycle support
         try:
@@ -370,18 +370,110 @@ class CyclicWorkflowExecutor:
                 )
                 results.update(cycle_results)
             else:
-                # Execute DAG nodes
-                for node_id in stage.nodes:
-                    if node_id not in state.node_outputs:
-                        logger.info(f"Executing DAG node: {node_id}")
-                        node_result = self._execute_node(
-                            workflow, node_id, state, task_manager=task_manager
-                        )
-                        results[node_id] = node_result
-                        state.node_outputs[node_id] = node_result
+                # Execute DAG nodes using extracted method
+                dag_results = self._execute_dag_portion(
+                    workflow, stage.nodes, state, task_manager
+                )
+                results.update(dag_results)
+
+        return results
+
+    def _execute_dag_portion(
+        self,
+        workflow: Workflow,
+        dag_nodes: list[str],
+        state: WorkflowState,
+        task_manager: TaskManager | None = None,
+    ) -> dict[str, Any]:
+        """Execute DAG (non-cyclic) portion of the workflow.
+
+        Args:
+            workflow: Workflow instance
+            dag_nodes: List of DAG node IDs to execute
+            state: Workflow state
+            task_manager: Optional task manager for tracking
+
+        Returns:
+            Dictionary with node IDs as keys and their results as values
+        """
+        results = {}
+
+        for node_id in dag_nodes:
+            if node_id not in state.node_outputs:
+                logger.info(f"Executing DAG node: {node_id}")
+                node_result = self._execute_node(
+                    workflow, node_id, state, task_manager=task_manager
+                )
+                results[node_id] = node_result
+                state.node_outputs[node_id] = node_result
+
+        return results
+
+    def _execute_cycle_groups(
+        self,
+        workflow: Workflow,
+        cycle_groups: list["CycleGroup"],
+        state: WorkflowState,
+        task_manager: TaskManager | None = None,
+    ) -> dict[str, Any]:
+        """Execute cycle groups portion of the workflow.
+
+        Args:
+            workflow: Workflow instance
+            cycle_groups: List of cycle groups to execute
+            state: Workflow state
+            task_manager: Optional task manager for tracking
+
+        Returns:
+            Dictionary with node IDs as keys and their results as values
+        """
+        results = {}
+
+        for cycle_group in cycle_groups:
+            logger.info(f"Executing cycle group: {cycle_group.cycle_id}")
+            cycle_results = self._execute_cycle_group(
+                workflow, cycle_group, state, task_manager
+            )
+            results.update(cycle_results)
 
         return results
 
+    def _propagate_parameters(
+        self,
+        current_params: dict[str, Any],
+        current_results: dict[str, Any],
+        cycle_config: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Handle parameter propagation between cycle iterations.
+
+        Args:
+            current_params: Current iteration parameters
+            current_results: Results from current iteration
+            cycle_config: Cycle configuration (optional)
+
+        Returns:
+            Updated parameters for the next iteration
+        """
+        # Base propagation: copy current results for next iteration
+        next_params = current_results.copy() if current_results else {}
+
+        # Apply any cycle-specific parameter mappings if provided
+        if cycle_config and "parameter_mappings" in cycle_config:
+            mappings = cycle_config["parameter_mappings"]
+            for src_key, dst_key in mappings.items():
+                if src_key in current_results:
+                    next_params[dst_key] = current_results[src_key]
+
+        # Preserve any initial parameters that aren't overridden
+        for key, value in current_params.items():
+            if key not in next_params:
+                next_params[key] = value
+
+        # Filter out None values to avoid validation errors
+        next_params = self._filter_none_values(next_params)
+
+        return next_params
+
     def _execute_cycle_group(
         self,
         workflow: Workflow,

kailash/workflow/visualization.py

@@ -18,7 +18,7 @@ class WorkflowVisualizer:
 
     def __init__(
         self,
-        workflow: Workflow,
+        workflow: Workflow | None = None,
         node_colors: dict[str, str] | None = None,
         edge_colors: dict[str, str] | None = None,
         layout: str = "hierarchical",
@@ -26,7 +26,7 @@ class WorkflowVisualizer:
         """Initialize visualizer.
 
         Args:
-            workflow: Workflow to visualize
+            workflow: Workflow to visualize (can be set later)
             node_colors: Custom node color map
             edge_colors: Custom edge color map
             layout: Layout algorithm to use
@@ -70,9 +70,12 @@ class WorkflowVisualizer:
         """Get colors for all nodes in workflow."""
         colors = []
         for node_id in self.workflow.graph.nodes():
-            node_instance = self.workflow.nodes[node_id]
-            node_type = node_instance.node_type
-            colors.append(self._get_node_color(node_type))
+            node_instance = self.workflow.nodes.get(node_id)
+            if node_instance:
+                node_type = node_instance.node_type
+                colors.append(self._get_node_color(node_type))
+            else:
+                colors.append(self.node_colors["default"])
         return colors
 
     def _get_node_labels(self) -> dict[str, str]:
@@ -119,11 +122,17 @@ class WorkflowVisualizer:
 
         return edge_labels
 
-    def _calculate_layout(self) -> dict[str, tuple[float, float]]:
+    def _calculate_layout(
+        self, workflow: "Workflow" = None
+    ) -> dict[str, tuple[float, float]]:
         """Calculate node positions for visualization."""
+        target_workflow = workflow or self.workflow
+        if not target_workflow:
+            return {}
+
         # Try to use stored positions first
         pos = {}
-        for node_id, node_instance in self.workflow.nodes.items():
+        for node_id, node_instance in target_workflow.nodes.items():
             if node_instance.position != (0, 0):
                 pos[node_id] = node_instance.position
 
@@ -133,32 +142,73 @@ class WorkflowVisualizer:
                 # Use hierarchical layout for DAGs
                 try:
                     # Create layers based on topological order
-                    layers = self._create_layers()
+                    layers = self._create_layers(target_workflow)
                     pos = self._hierarchical_layout(layers)
                 except Exception:
                     # Fallback to spring layout
-                    pos = nx.spring_layout(self.workflow.graph, k=3, iterations=50)
+                    pos = nx.spring_layout(target_workflow.graph, k=3, iterations=50)
             elif self.layout == "circular":
-                pos = nx.circular_layout(self.workflow.graph)
+                pos = nx.circular_layout(target_workflow.graph)
             elif self.layout == "spring":
-                pos = nx.spring_layout(self.workflow.graph, k=2, iterations=100)
+                pos = nx.spring_layout(target_workflow.graph, k=2, iterations=100)
             else:
                 # Default to spring layout
-                pos = nx.spring_layout(self.workflow.graph)
+                pos = nx.spring_layout(target_workflow.graph)
 
         return pos
 
-    def _create_layers(self) -> dict[int, list]:
+    def _get_layout_positions(
+        self, workflow: Workflow
+    ) -> dict[str, tuple[float, float]]:
+        """Get layout positions for workflow nodes."""
+        # Temporarily store workflow and calculate layout
+        original_workflow = self.workflow
+        self.workflow = workflow
+        try:
+            return self._calculate_layout()
+        finally:
+            self.workflow = original_workflow
+
+    def _get_node_colors(self, workflow: Workflow) -> list[str]:
+        """Get node colors for workflow."""
+        colors = []
+        for node_id in workflow.graph.nodes():
+            node_instance = workflow.get_node(node_id)
+            if node_instance:
+                node_type = node_instance.__class__.__name__.lower()
+                # Map node types to color categories
+                if "data" in node_type or "csv" in node_type or "json" in node_type:
+                    color_key = "data"
+                elif "transform" in node_type or "python" in node_type:
+                    color_key = "transform"
+                elif "switch" in node_type or "merge" in node_type:
+                    color_key = "logic"
+                elif "llm" in node_type or "ai" in node_type:
+                    color_key = "ai"
+                else:
+                    color_key = "default"
+                colors.append(
+                    self.node_colors.get(color_key, self.node_colors["default"])
+                )
+            else:
+                colors.append(self.node_colors["default"])
+        return colors
+
+    def _create_layers(self, workflow: "Workflow" = None) -> dict[int, list]:
         """Create layers of nodes for hierarchical layout."""
+        target_workflow = workflow or self.workflow
+        if not target_workflow:
+            return {}
+
         layers = {}
-        remaining = set(self.workflow.graph.nodes())
+        remaining = set(target_workflow.graph.nodes())
         layer = 0
 
         while remaining:
             # Find nodes with no dependencies in remaining set
             current_layer = []
             for node in remaining:
-                predecessors = set(self.workflow.graph.predecessors(node))
+                predecessors = set(target_workflow.graph.predecessors(node))
                 if not predecessors.intersection(remaining):
                     current_layer.append(node)
 
@@ -196,20 +246,38 @@ class WorkflowVisualizer:
 
     def _draw_graph(
         self,
-        pos: dict[str, tuple[float, float]],
-        node_colors: list[str],
-        show_labels: bool,
-        show_connections: bool,
+        workflow: Workflow | None = None,
+        pos: dict[str, tuple[float, float]] | None = None,
+        node_colors: list[str] | None = None,
+        show_labels: bool = True,
+        show_connections: bool = True,
     ) -> None:
         """Draw the graph with given positions and options."""
+        # Use provided workflow or fall back to instance workflow
+        target_workflow = workflow or self.workflow
+        if not target_workflow:
+            raise ValueError("No workflow provided to draw")
+
+        # Use default position if not provided
+        if pos is None:
+            pos = self._get_layout_positions(target_workflow)
+
+        # Use default colors if not provided
+        if node_colors is None:
+            node_colors = self._get_node_colors(target_workflow)
+
         # Draw nodes
         nx.draw_networkx_nodes(
-            self.workflow.graph, pos, node_color=node_colors, node_size=3000, alpha=0.9
+            target_workflow.graph,
+            pos,
+            node_color=node_colors,
+            node_size=3000,
+            alpha=0.9,
         )
 
         # Draw edges
         nx.draw_networkx_edges(
-            self.workflow.graph,
+            target_workflow.graph,
             pos,
             edge_color=self.edge_colors["default"],
             width=2,
@@ -221,16 +289,26 @@ class WorkflowVisualizer:
 
         # Draw labels
         if show_labels:
+            # Temporarily set workflow for label generation
+            old_workflow = self.workflow
+            self.workflow = target_workflow
             labels = self._get_node_labels()
+            self.workflow = old_workflow
+
             nx.draw_networkx_labels(
-                self.workflow.graph, pos, labels, font_size=10, font_weight="bold"
+                target_workflow.graph, pos, labels, font_size=10, font_weight="bold"
             )
 
         # Draw connection labels
         if show_connections:
+            # Temporarily set workflow for edge label generation
+            old_workflow = self.workflow
+            self.workflow = target_workflow
             edge_labels = self._get_edge_labels()
+            self.workflow = old_workflow
+
             nx.draw_networkx_edge_labels(
-                self.workflow.graph, pos, edge_labels, font_size=8
+                target_workflow.graph, pos, edge_labels, font_size=8
             )
 
     def visualize(
@@ -255,10 +333,16 @@ class WorkflowVisualizer:
             **kwargs: Additional options passed to plt.savefig
         """
         try:
+            # Check if workflow is available
+            if not self.workflow:
+                raise ValueError(
+                    "No workflow to visualize. Set workflow property or create visualizer with workflow."
+                )
+
             plt.figure(figsize=figsize)
 
             # Calculate node positions
-            pos = self._calculate_layout()
+            pos = self._calculate_layout(self.workflow)
 
             # Handle empty workflow case
             if not self.workflow.graph.nodes():
@@ -266,11 +350,17 @@ class WorkflowVisualizer:
                 node_colors = []
             else:
                 # Draw the graph with colors
-                node_colors = self._get_node_colors()
+                node_colors = self._get_node_colors(self.workflow)
 
             # Draw the graph components
-            if pos and node_colors:
-                self._draw_graph(pos, node_colors, show_labels, show_connections)
+            if pos:
+                self._draw_graph(
+                    workflow=self.workflow,
+                    pos=pos,
+                    node_colors=node_colors,
+                    show_labels=show_labels,
+                    show_connections=show_connections,
+                )
 
             # Set title
             if title is None:

{kailash-0.7.0.dist-info → kailash-0.8.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kailash
-Version: 0.7.0
+Version: 0.8.1
 Summary: Python SDK for the Kailash container-node architecture
 Home-page: https://github.com/integrum/kailash-python-sdk
 Author: Integrum
@@ -21,7 +21,7 @@ Requires-Dist: matplotlib>=3.5
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: click>=8.0
 Requires-Dist: pytest>=8.3.5
-Requires-Dist: mcp[cli]>=1.9.2
+Requires-Dist: mcp[cli]==1.11.0
 Requires-Dist: pandas>=2.2.3
 Requires-Dist: numpy>=2.2.5
 Requires-Dist: scipy>=1.15.3
@@ -58,6 +58,7 @@ Requires-Dist: python-jose>=3.5.0
 Requires-Dist: pytest-xdist>=3.6.0
 Requires-Dist: pytest-timeout>=2.3.0
 Requires-Dist: pytest-split>=0.9.0
+Requires-Dist: pytest-forked>=1.6.0
 Requires-Dist: asyncpg>=0.30.0
 Requires-Dist: aiomysql>=0.2.0
 Requires-Dist: twilio>=9.6.3
@@ -78,6 +79,7 @@ Requires-Dist: passlib>=1.7.4
 Requires-Dist: pyotp>=2.9.0
 Requires-Dist: opentelemetry-instrumentation-fastapi>=0.55b1
 Requires-Dist: seaborn>=0.13.2
+Requires-Dist: sqlparse>=0.5.3
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: pytest-cov>=3.0; extra == "dev"