kailash 0.6.6__py3-none-any.whl → 0.8.0__py3-none-any.whl

kailash/workflow/builder.py

@@ -2,8 +2,9 @@
 
 import logging
 import uuid
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
+from kailash.nodes.base import Node
 from kailash.sdk_exceptions import ConnectionError, WorkflowValidationError
 from kailash.workflow.graph import Workflow
 
@@ -18,23 +19,234 @@ class WorkflowBuilder:
         self.nodes: dict[str, dict[str, Any]] = {}
         self.connections: list[dict[str, str]] = []
         self._metadata: dict[str, Any] = {}
+        # Parameter injection capabilities
+        self.workflow_parameters: dict[str, Any] = {}
+        self.parameter_mappings: dict[str, dict[str, str]] = {}
 
-    def add_node(
+    def add_node(self, *args, **kwargs) -> str:
+        """
+        Unified add_node method supporting multiple API patterns.
+
+        Supported patterns:
+        1. add_node("NodeType", "node_id", {"param": value})     # Current/Preferred
+        2. add_node("node_id", NodeClass, param=value)           # Legacy fluent
+        3. add_node(NodeClass, "node_id", param=value)           # Alternative
+
+        Args:
+            *args: Positional arguments (pattern-dependent)
+            **kwargs: Keyword arguments for configuration
+
+        Returns:
+            Node ID (useful for method chaining)
+
+        Raises:
+            WorkflowValidationError: If node_id is already used or invalid pattern
+        """
+        # Pattern detection and routing
+        if len(args) == 0 and kwargs:
+            # Keyword-only pattern: add_node(node_type="NodeType", node_id="id", config={})
+            node_type = kwargs.pop("node_type", None)
+            node_id = kwargs.pop("node_id", None)
+            config = kwargs.pop("config", {})
+            # Any remaining kwargs are treated as config
+            config.update(kwargs)
+
+            if node_type is None:
+                raise WorkflowValidationError(
+                    "node_type is required when using keyword arguments"
+                )
+
+            return self._add_node_current(node_type, node_id, config)
+
+        elif len(args) == 1:
+            # Single argument with possible keywords
+            if isinstance(args[0], str) and kwargs:
+                # Pattern: add_node("NodeType", node_id="id", config={})
+                node_type = args[0]
+                node_id = kwargs.pop("node_id", None)
+                config = kwargs.pop("config", {})
+                # Any remaining kwargs are treated as config
+                config.update(kwargs)
+                return self._add_node_current(node_type, node_id, config)
+            elif isinstance(args[0], str):
+                # Pattern: add_node("NodeType")
+                return self._add_node_current(args[0], None, {})
+            elif hasattr(args[0], "__name__"):
+                # Pattern: add_node(NodeClass)
+                return self._add_node_alternative(args[0], None, **kwargs)
+            else:
+                if isinstance(args[0], Node):
+                    # Pattern: add_node(node_instance)
+                    return self._add_node_instance(args[0], None)
+
+        elif len(args) == 3 and isinstance(args[0], str) and isinstance(args[2], dict):
+            # Pattern 1: Current API - add_node("NodeType", "node_id", {"param": value})
+            return self._add_node_current(args[0], args[1], args[2])
+
+        elif len(args) >= 2 and isinstance(args[0], str):
+            # Pattern 2: Legacy fluent API - add_node("node_id", NodeClass, param=value)
+            if hasattr(args[1], "__name__") or isinstance(args[1], type):
+                return self._add_node_legacy_fluent(args[0], args[1], **kwargs)
+            elif isinstance(args[1], str):
+                # Two strings - assume current API: add_node("NodeType", "node_id")
+                config = kwargs if kwargs else (args[2] if len(args) > 2 else {})
+                return self._add_node_current(args[0], args[1], config)
+            else:
+                # Invalid second argument
+                raise WorkflowValidationError(
+                    f"Invalid node type: {type(args[1]).__name__}. "
+                    "Expected: str (node type name), Node class, or Node instance"
+                )
+
+        elif len(args) >= 2 and hasattr(args[0], "__name__"):
+            # Pattern 3: Alternative - add_node(NodeClass, "node_id", param=value)
+            # Handle both dict config and keyword args
+            if len(args) == 3 and isinstance(args[2], dict):
+                # Config provided as dict
+                return self._add_node_alternative(args[0], args[1], **args[2])
+            else:
+                # Config provided as kwargs
+                return self._add_node_alternative(args[0], args[1], **kwargs)
+
+        elif len(args) >= 2:
+            # Check if first arg is a Node instance
+            if isinstance(args[0], Node):
+                # Pattern 4: Instance - add_node(node_instance, "node_id") or add_node(node_instance, "node_id", config)
+                # Config is ignored for instances
+                return self._add_node_instance(args[0], args[1])
+            elif len(args) == 2:
+                # Invalid arguments for 2-arg call
+                raise WorkflowValidationError(
+                    f"Invalid node type: {type(args[0]).__name__}. "
+                    "Expected: str (node type name), Node class, or Node instance"
+                )
+
+            # For 3 or more args that don't match other patterns
+            # Error with helpful message
+            raise WorkflowValidationError(
+                f"Invalid add_node signature. Received {len(args)} args: {[type(arg).__name__ for arg in args]}\n"
+                f"Supported patterns:\n"
+                f"  add_node('NodeType', 'node_id', {{'param': value}})  # Preferred\n"
+                f"  add_node('node_id', NodeClass, param=value)          # Legacy\n"
+                f"  add_node(NodeClass, 'node_id', param=value)          # Alternative\n"
+                f"Examples:\n"
+                f"  add_node('HTTPRequestNode', 'api_call', {{'url': 'https://api.com'}})\n"
+                f"  add_node('csv_reader', CSVReaderNode, file_path='data.csv')"
+            )
+
+    def _add_node_current(
+        self, node_type: str, node_id: str | None, config: dict[str, Any]
+    ) -> str:
+        """Handle current API pattern: add_node('NodeType', 'node_id', {'param': value})"""
+        return self._add_node_unified(node_type, node_id, config)
+
+    def _add_node_legacy_fluent(
+        self, node_id: str, node_class_or_type: Any, **config
+    ) -> "WorkflowBuilder":
+        """Handle legacy fluent API pattern: add_node('node_id', NodeClass, param=value)"""
+        import warnings
+
+        # If it's a class, validate it's a Node subclass
+        if isinstance(node_class_or_type, type) and not issubclass(
+            node_class_or_type, Node
+        ):
+            raise WorkflowValidationError(
+                f"Invalid node type: {node_class_or_type}. Expected a Node subclass or string."
+            )
+
+        warnings.warn(
+            f"Legacy fluent API usage detected. "
+            f"Migration guide:\n"
+            f"  OLD: add_node('{node_id}', {getattr(node_class_or_type, '__name__', str(node_class_or_type))}, {list(config.keys())})\n"
+            f"  NEW: add_node('{getattr(node_class_or_type, '__name__', str(node_class_or_type))}', '{node_id}', {config})\n"
+            f"Legacy support will be removed in v0.8.0",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+
+        if hasattr(node_class_or_type, "__name__"):
+            node_type = node_class_or_type.__name__
+        else:
+            node_type = str(node_class_or_type)
+
+        self._add_node_unified(node_type, node_id, config)
+        return self  # Return self for fluent chaining
+
+    def _add_node_alternative(
+        self, node_class: type, node_id: str | None, **config
+    ) -> str:
+        """Handle alternative pattern: add_node(NodeClass, 'node_id', param=value)"""
+        import warnings
+
+        # Validate that node_class is actually a Node subclass
+        if not isinstance(node_class, type) or not issubclass(node_class, Node):
+            raise WorkflowValidationError(
+                f"Invalid node type: {node_class}. Expected a Node subclass."
+            )
+
+        # Generate ID if not provided
+        if node_id is None:
+            node_id = f"node_{uuid.uuid4().hex[:8]}"
+
+        warnings.warn(
+            f"Alternative API usage detected. Consider using preferred pattern:\n"
+            f"  CURRENT: add_node({node_class.__name__}, '{node_id}', {list(config.keys())})\n"
+            f"  PREFERRED: add_node('{node_class.__name__}', '{node_id}', {config})",
+            UserWarning,
+            stacklevel=3,
+        )
+
+        # Store the class reference along with the type name
+        self.nodes[node_id] = {
+            "type": node_class.__name__,
+            "config": config,
+            "class": node_class,
+        }
+        logger.info(f"Added node '{node_id}' of type '{node_class.__name__}'")
+        return node_id
+
+    def _add_node_instance(self, node_instance: "Node", node_id: str | None) -> str:
+        """Handle instance pattern: add_node(node_instance, 'node_id')"""
+        import warnings
+
+        # Generate ID if not provided
+        if node_id is None:
+            node_id = f"node_{uuid.uuid4().hex[:8]}"
+
+        warnings.warn(
+            f"Instance-based API usage detected. Consider using preferred pattern:\n"
+            f"  CURRENT: add_node(<instance>, '{node_id}')\n"
+            f"  PREFERRED: add_node('{node_instance.__class__.__name__}', '{node_id}', {{'param': value}})",
+            UserWarning,
+            stacklevel=3,
+        )
+
+        # Store the instance
+        self.nodes[node_id] = {
+            "instance": node_instance,
+            "type": node_instance.__class__.__name__,
+        }
+        logger.info(
+            f"Added node '{node_id}' with instance of type '{node_instance.__class__.__name__}'"
+        )
+        return node_id
+
+    def _add_node_unified(
         self,
-        node_type: str | type | Any,
+        node_type: str,
         node_id: str | None = None,
         config: dict[str, Any] | None = None,
     ) -> str:
         """
-        Add a node to the workflow.
+        Unified implementation for all add_node patterns.
 
         Args:
-            node_type: Node type name (string), Node class, or Node instance
+            node_type: Node type name (string)
             node_id: Unique identifier for this node (auto-generated if not provided)
-            config: Configuration for the node (ignored if node_type is an instance)
+            config: Configuration for the node
 
         Returns:
-            Node ID (useful for method chaining)
+            Node ID
 
         Raises:
             WorkflowValidationError: If node_id is already used
@@ -48,9 +260,6 @@ class WorkflowBuilder:
                 f"Node ID '{node_id}' already exists in workflow"
             )
 
-        # Import Node here to avoid circular imports
-        from kailash.nodes.base import Node
-
         # Handle different input types
         if isinstance(node_type, str):
             # String node type name
@@ -80,6 +289,39 @@ class WorkflowBuilder:
         logger.info(f"Added node '{node_id}' of type '{type_name}'")
         return node_id
 
+    # Fluent API methods for backward compatibility
+    def add_node_fluent(
+        self, node_id: str, node_class_or_type: Any, **config
+    ) -> "WorkflowBuilder":
+        """
+        DEPRECATED: Fluent API for backward compatibility.
+        Use add_node(node_type, node_id, config) instead.
+
+        Args:
+            node_id: Node identifier
+            node_class_or_type: Node class or type
+            **config: Node configuration as keyword arguments
+
+        Returns:
+            Self for method chaining
+        """
+        import warnings
+
+        warnings.warn(
+            "Fluent API is deprecated. Use add_node(node_type, node_id, config) instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        if hasattr(node_class_or_type, "__name__"):
+            # Node class
+            self.add_node(node_class_or_type.__name__, node_id, config)
+        else:
+            # Assume string type
+            self.add_node(str(node_class_or_type), node_id, config)
+
+        return self
+
     def add_node_instance(self, node_instance: Any, node_id: str | None = None) -> str:
         """
         Add a node instance to the workflow.
@@ -124,7 +366,7 @@ class WorkflowBuilder:
 
     def add_connection(
         self, from_node: str, from_output: str, to_node: str, to_input: str
-    ) -> None:
+    ) -> "WorkflowBuilder":
         """
         Connect two nodes in the workflow.
 
@@ -161,6 +403,7 @@ class WorkflowBuilder:
         self.connections.append(connection)
 
         logger.info(f"Connected '{from_node}.{from_output}' to '{to_node}.{to_input}'")
+        return self
 
     def connect(
         self,
@@ -353,12 +596,108 @@ class WorkflowBuilder:
                     f"Failed to connect '{from_node}' to '{to_node}': {e}"
                 ) from e
 
+        # Parameter injection: Find nodes without incoming connections and inject parameters
+        if self.workflow_parameters:
+            nodes_with_inputs = set()
+            for conn in self.connections:
+                if not conn.get("is_workflow_input"):
+                    nodes_with_inputs.add(conn["to_node"])
+
+            nodes_without_inputs = set(self.nodes.keys()) - nodes_with_inputs
+
+            # For each node without inputs, check if it needs workflow parameters
+            for node_id in nodes_without_inputs:
+                node_info = self.nodes[node_id]
+                node_instance = workflow.get_node(node_id)
+
+                if hasattr(node_instance, "get_parameters"):
+                    params = node_instance.get_parameters()
+
+                    # Check which required parameters are missing from config
+                    for param_name, param_def in params.items():
+                        if param_def.required and param_name not in node_info["config"]:
+                            # Check if this parameter should come from workflow parameters
+                            if param_name in self.workflow_parameters:
+                                # Add to node config
+                                node_info["config"][param_name] = (
+                                    self.workflow_parameters[param_name]
+                                )
+                            elif node_id in self.parameter_mappings:
+                                # Check parameter mappings
+                                mapping = self.parameter_mappings[node_id]
+                                if param_name in mapping:
+                                    workflow_param = mapping[param_name]
+                                    if workflow_param in self.workflow_parameters:
+                                        node_info["config"][param_name] = (
+                                            self.workflow_parameters[workflow_param]
+                                        )
+
+        # Store workflow parameters in metadata for runtime reference
+        workflow.metadata["workflow_parameters"] = self.workflow_parameters
+        workflow.metadata["parameter_mappings"] = self.parameter_mappings
+
         logger.info(
             f"Built workflow '{workflow_id}' with "
             f"{len(self.nodes)} nodes and {len(self.connections)} connections"
         )
         return workflow
 
+    def set_workflow_parameters(self, **parameters) -> "WorkflowBuilder":
+        """
+        Set default parameters that will be passed to all nodes.
+
+        Args:
+            **parameters: Key-value pairs of workflow-level parameters
+
+        Returns:
+            Self for chaining
+        """
+        self.workflow_parameters.update(parameters)
+        return self
+
+    def add_parameter_mapping(
+        self, node_id: str, mappings: dict[str, str]
+    ) -> "WorkflowBuilder":
+        """
+        Add parameter mappings for a specific node.
+
+        Args:
+            node_id: Node to configure
+            mappings: Dict mapping workflow param names to node param names
+
+        Returns:
+            Self for chaining
+        """
+        if node_id not in self.parameter_mappings:
+            self.parameter_mappings[node_id] = {}
+        self.parameter_mappings[node_id].update(mappings)
+        return self
+
+    def add_input_connection(
+        self, to_node: str, to_input: str, from_workflow_param: str
+    ) -> "WorkflowBuilder":
+        """
+        Connect a workflow parameter directly to a node input.
+
+        Args:
+            to_node: Target node ID
+            to_input: Input parameter name on the node
+            from_workflow_param: Workflow parameter name
+
+        Returns:
+            Self for chaining
+        """
+        # Add a special connection type for workflow inputs
+        connection = {
+            "from_node": "__workflow_input__",
+            "from_output": from_workflow_param,
+            "to_node": to_node,
+            "to_input": to_input,
+            "is_workflow_input": True,
+        }
+        self.connections.append(connection)
+        return self
+
     def clear(self) -> "WorkflowBuilder":
         """
         Clear builder state.
@@ -369,6 +708,8 @@ class WorkflowBuilder:
         self.nodes = {}
         self.connections = []
         self._metadata = {}
+        self.workflow_parameters = {}
+        self.parameter_mappings = {}
         return self
 
     @classmethod
@@ -399,9 +740,21 @@ class WorkflowBuilder:
             # Dict format: {node_id: {type: "...", parameters: {...}}}
             for node_id, node_config in nodes_config.items():
                 node_type = node_config.get("type")
-                node_params = node_config.get(
-                    "parameters", node_config.get("config", {})
-                )
+
+                # Handle parameter naming inconsistencies - prefer 'parameters' over 'config'
+                if "parameters" in node_config:
+                    node_params = node_config["parameters"]
+                elif "config" in node_config:
+                    node_params = node_config["config"]
+                else:
+                    node_params = {}
+
+                # Ensure node_params is a dictionary
+                if not isinstance(node_params, dict):
+                    logger.warning(
+                        f"Node '{node_id}' parameters must be a dict, got {type(node_params)}. Using empty dict."
+                    )
+                    node_params = {}
 
                 if not node_type:
                     raise WorkflowValidationError(
@@ -414,7 +767,21 @@ class WorkflowBuilder:
             for node_config in nodes_config:
                 node_id = node_config.get("id")
                 node_type = node_config.get("type")
-                node_params = node_config.get("config", {})
+
+                # Handle parameter naming inconsistencies - prefer 'parameters' over 'config'
+                if "parameters" in node_config:
+                    node_params = node_config["parameters"]
+                elif "config" in node_config:
+                    node_params = node_config["config"]
+                else:
+                    node_params = {}
+
+                # Ensure node_params is a dictionary
+                if not isinstance(node_params, dict):
+                    logger.warning(
+                        f"Node '{node_id}' parameters must be a dict, got {type(node_params)}. Using empty dict."
+                    )
+                    node_params = {}
 
                 if not node_id:
                     raise WorkflowValidationError("Node ID is required")

kailash/workflow/cyclic_runner.py

@@ -184,7 +184,7 @@ class CyclicWorkflowExecutor:
         if not workflow.has_cycles():
             # No cycles, use standard DAG execution
             logger.info("No cycles detected, using standard DAG execution")
-            return self.dag_runner.run(workflow, parameters), run_id
+            return self.dag_runner.execute(workflow, parameters), run_id
 
         # Execute with cycle support
         try:
@@ -370,18 +370,110 @@ class CyclicWorkflowExecutor:
                 )
                 results.update(cycle_results)
             else:
-                # Execute DAG nodes
-                for node_id in stage.nodes:
-                    if node_id not in state.node_outputs:
-                        logger.info(f"Executing DAG node: {node_id}")
-                        node_result = self._execute_node(
-                            workflow, node_id, state, task_manager=task_manager
-                        )
-                        results[node_id] = node_result
-                        state.node_outputs[node_id] = node_result
+                # Execute DAG nodes using extracted method
+                dag_results = self._execute_dag_portion(
+                    workflow, stage.nodes, state, task_manager
+                )
+                results.update(dag_results)
+
+        return results
+
+    def _execute_dag_portion(
+        self,
+        workflow: Workflow,
+        dag_nodes: list[str],
+        state: WorkflowState,
+        task_manager: TaskManager | None = None,
+    ) -> dict[str, Any]:
+        """Execute DAG (non-cyclic) portion of the workflow.
+
+        Args:
+            workflow: Workflow instance
+            dag_nodes: List of DAG node IDs to execute
+            state: Workflow state
+            task_manager: Optional task manager for tracking
+
+        Returns:
+            Dictionary with node IDs as keys and their results as values
+        """
+        results = {}
+
+        for node_id in dag_nodes:
+            if node_id not in state.node_outputs:
+                logger.info(f"Executing DAG node: {node_id}")
+                node_result = self._execute_node(
+                    workflow, node_id, state, task_manager=task_manager
+                )
+                results[node_id] = node_result
+                state.node_outputs[node_id] = node_result
+
+        return results
+
+    def _execute_cycle_groups(
+        self,
+        workflow: Workflow,
+        cycle_groups: list["CycleGroup"],
+        state: WorkflowState,
+        task_manager: TaskManager | None = None,
+    ) -> dict[str, Any]:
+        """Execute cycle groups portion of the workflow.
+
+        Args:
+            workflow: Workflow instance
+            cycle_groups: List of cycle groups to execute
+            state: Workflow state
+            task_manager: Optional task manager for tracking
+
+        Returns:
+            Dictionary with node IDs as keys and their results as values
+        """
+        results = {}
+
+        for cycle_group in cycle_groups:
+            logger.info(f"Executing cycle group: {cycle_group.cycle_id}")
+            cycle_results = self._execute_cycle_group(
+                workflow, cycle_group, state, task_manager
+            )
+            results.update(cycle_results)
 
         return results
 
+    def _propagate_parameters(
+        self,
+        current_params: dict[str, Any],
+        current_results: dict[str, Any],
+        cycle_config: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Handle parameter propagation between cycle iterations.
+
+        Args:
+            current_params: Current iteration parameters
+            current_results: Results from current iteration
+            cycle_config: Cycle configuration (optional)
+
+        Returns:
+            Updated parameters for the next iteration
+        """
+        # Base propagation: copy current results for next iteration
+        next_params = current_results.copy() if current_results else {}
+
+        # Apply any cycle-specific parameter mappings if provided
+        if cycle_config and "parameter_mappings" in cycle_config:
+            mappings = cycle_config["parameter_mappings"]
+            for src_key, dst_key in mappings.items():
+                if src_key in current_results:
+                    next_params[dst_key] = current_results[src_key]
+
+        # Preserve any initial parameters that aren't overridden
+        for key, value in current_params.items():
+            if key not in next_params:
+                next_params[key] = value
+
+        # Filter out None values to avoid validation errors
+        next_params = self._filter_none_values(next_params)
+
+        return next_params
+
     def _execute_cycle_group(
         self,
         workflow: Workflow,