griptape-nodes 0.64.11__py3-none-any.whl → 0.65.1__py3-none-any.whl

griptape_nodes/machines/control_flow.py

@@ -5,9 +5,8 @@ import logging
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-from griptape_nodes.exe_types.core_types import Parameter, ParameterTypeBuiltin
+from griptape_nodes.exe_types.base_iterative_nodes import BaseIterativeStartNode
 from griptape_nodes.exe_types.node_types import (
-    CONTROL_INPUT_PARAMETER,
     LOCAL_EXECUTION,
     BaseNode,
     NodeGroupNode,
@@ -28,6 +27,7 @@ from griptape_nodes.retained_mode.managers.node_manager import NodeManager
 from griptape_nodes.retained_mode.managers.settings import WorkflowExecutionMode
 
 if TYPE_CHECKING:
+    from griptape_nodes.exe_types.core_types import Parameter
     from griptape_nodes.exe_types.flow import ControlFlow
 
 
@@ -52,6 +52,7 @@ class ControlFlowContext:
     flow_name: str
     pickle_control_flow_result: bool
     end_node: BaseNode | None = None
+    is_isolated: bool
 
     def __init__(
         self,
@@ -60,12 +61,21 @@ class ControlFlowContext:
         *,
         execution_type: WorkflowExecutionMode = WorkflowExecutionMode.SEQUENTIAL,
         pickle_control_flow_result: bool = False,
+        is_isolated: bool = False,
     ) -> None:
         self.flow_name = flow_name
         if execution_type == WorkflowExecutionMode.PARALLEL:
             # Get the global DagBuilder from FlowManager
+            from griptape_nodes.machines.dag_builder import DagBuilder
+            from griptape_nodes.retained_mode.griptape_nodes import GriptapeNodes
+
+            # Create isolated DagBuilder for independent subflows
+            if is_isolated:
+                dag_builder = DagBuilder()
+                logger.debug("Created isolated DagBuilder for flow '%s'", flow_name)
+            else:
+                dag_builder = GriptapeNodes.FlowManager().global_dag_builder
 
-            dag_builder = GriptapeNodes.FlowManager().global_dag_builder
             self.resolution_machine = ParallelResolutionMachine(
                 flow_name, max_nodes_in_parallel, dag_builder=dag_builder
             )
@@ -73,6 +83,7 @@ class ControlFlowContext:
             self.resolution_machine = SequentialResolutionMachine()
         self.current_nodes = []
         self.pickle_control_flow_result = pickle_control_flow_result
+        self.is_isolated = is_isolated
 
     def get_next_nodes(self, output_parameter: Parameter | None = None) -> list[NextNodeInfo]:
         """Get all next nodes from the current nodes.
@@ -85,24 +96,27 @@ class ControlFlowContext:
             if output_parameter is not None:
                 # Get connected node from control flow
                 node_connection = (
-                    GriptapeNodes.FlowManager().get_connections().get_connected_node(current_node, output_parameter)
+                    GriptapeNodes.FlowManager()
+                    .get_connections()
+                    .get_connected_node(current_node, output_parameter, include_internal=False)
                 )
                 if node_connection is not None:
                     node, entry_parameter = node_connection
                     next_nodes.append(NextNodeInfo(node=node, entry_parameter=entry_parameter))
+            # Get next control output for this node
             else:
-                # Get next control output for this node
-
-                if (
-                    isinstance(current_node, NodeGroupNode)
-                    and current_node.get_parameter_value(current_node.execution_environment.name) != LOCAL_EXECUTION
-                ):
-                    next_output = self.get_next_control_output_for_non_local_execution(current_node)
-                else:
-                    next_output = current_node.get_next_control_output()
+                next_output = current_node.get_next_control_output()
                 if next_output is not None:
+                    if isinstance(current_node, BaseIterativeStartNode):
+                        if current_node.end_node is None:
+                            msg = "Iterative start node has no end node"
+                            raise ValueError(msg)
+                        next_nodes.append(NextNodeInfo(node=current_node.end_node, entry_parameter=None))
+                        continue
                     node_connection = (
-                        GriptapeNodes.FlowManager().get_connections().get_connected_node(current_node, next_output)
+                        GriptapeNodes.FlowManager()
+                        .get_connections()
+                        .get_connected_node(current_node, next_output, include_internal=False)
                     )
                     if node_connection is not None:
                         node, entry_parameter = node_connection
@@ -111,27 +125,13 @@ class ControlFlowContext:
                     logger.debug("Control Flow: Node '%s' has no control output", current_node.name)
 
         # If no connections found, check execution queue
-        if not next_nodes:
+        if not next_nodes and not self.is_isolated:
             node = GriptapeNodes.FlowManager().get_next_node_from_execution_queue()
             if node is not None:
                 next_nodes.append(NextNodeInfo(node=node, entry_parameter=None))
 
         return next_nodes
 
-    # Mirrored in @parallel_resolution.py. if you update one, update the other.
-    def get_next_control_output_for_non_local_execution(self, node: BaseNode) -> Parameter | None:
-        for param_name, value in node.parameter_output_values.items():
-            parameter = node.get_parameter_by_name(param_name)
-            if (
-                parameter is not None
-                and parameter.type == ParameterTypeBuiltin.CONTROL_TYPE
-                and value == CONTROL_INPUT_PARAMETER
-            ):
-                # This is the parameter
-                logger.debug("Control Flow: Found control output parameter '%s' for non-local execution", param_name)
-                return parameter
-        return None
-
     def reset(self, *, cancel: bool = False) -> None:
         if self.current_nodes is not None:
             for node in self.current_nodes:
@@ -317,7 +317,13 @@ class CompleteState(State):
 
 # MACHINE TIME!!!
 class ControlFlowMachine(FSM[ControlFlowContext]):
-    def __init__(self, flow_name: str, *, pickle_control_flow_result: bool = False) -> None:
+    def __init__(
+        self,
+        flow_name: str,
+        *,
+        pickle_control_flow_result: bool = False,
+        is_isolated: bool = False,
+    ) -> None:
         execution_type = GriptapeNodes.ConfigManager().get_config_value(
             "workflow_execution_mode", default=WorkflowExecutionMode.SEQUENTIAL
         )
@@ -327,6 +333,7 @@ class ControlFlowMachine(FSM[ControlFlowContext]):
             max_nodes_in_parallel,
             execution_type=execution_type,
             pickle_control_flow_result=pickle_control_flow_result,
+            is_isolated=is_isolated,
         )
         super().__init__(context)
 
@@ -404,48 +411,87 @@ class ControlFlowMachine(FSM[ControlFlowContext]):
         ):
             await self.update()
 
-    async def _process_nodes_for_dag(self, start_node: BaseNode) -> list[BaseNode]:
+    async def _process_nodes_for_dag(self, start_node: BaseNode) -> list[BaseNode]:  # noqa: C901, PLR0912
         """Process data_nodes from the global queue to build unified DAG.
 
         This method identifies data_nodes in the execution queue and processes
         their dependencies into the DAG resolution machine.
+
+        For isolated subflows, this skips the global queue entirely and just
+        processes the start node, as subflows are self-contained.
         """
         if not isinstance(self._context.resolution_machine, ParallelResolutionMachine):
             return []
-        # Get the global flow queue
-        flow_manager = GriptapeNodes.FlowManager()
-        dag_builder = flow_manager.global_dag_builder
+
+        # Use the DagBuilder from the resolution machine context (may be isolated or global)
+        dag_builder = self._context.resolution_machine.context.dag_builder
         if dag_builder is None:
             msg = "DAG builder is not initialized."
             raise ValueError(msg)
 
         # Build with the first node (it should already be the proxy if it's part of a group)
         dag_builder.add_node_with_dependencies(start_node, start_node.name)
-        queue_items = list(flow_manager.global_flow_queue.queue)
+
+        # Check if we're using an isolated DagBuilder (for subflows)
+        flow_manager = GriptapeNodes.FlowManager()
+        node_manager = GriptapeNodes.NodeManager()
+        is_isolated = dag_builder is not flow_manager.global_dag_builder
+
+        if is_isolated:
+            # For isolated subflows, we don't process the global queue
+            # Just return the start node - the subflow is self-contained
+            logger.debug(
+                "Using isolated DagBuilder for flow '%s' - skipping global queue processing", self._context.flow_name
+            )
+            return [start_node]
+
+        # For main flows using the global DagBuilder, process the global queue
         start_nodes = [start_node]
         from griptape_nodes.retained_mode.managers.flow_manager import DagExecutionType
 
-        # Find data_nodes and remove them from queue
+        # PASS 1: Process all control/start nodes first to build control flow graphs
+        queue_items = list(flow_manager.global_flow_queue.queue)
         for item in queue_items:
             if item.dag_execution_type in (DagExecutionType.CONTROL_NODE, DagExecutionType.START_NODE):
                 node = item.node
                 node.state = NodeResolutionState.UNRESOLVED
                 # Use proxy node if this node is part of a group, otherwise use original node
                 node_to_add = node
+
                 # Only add if not already added (proxy might already be in DAG)
                 if node_to_add.name not in dag_builder.node_to_reference:
                     dag_builder.add_node_with_dependencies(node_to_add, node_to_add.name)
                     if node_to_add not in start_nodes:
                         start_nodes.append(node_to_add)
                 flow_manager.global_flow_queue.queue.remove(item)
-            elif item.dag_execution_type == DagExecutionType.DATA_NODE:
+
+        # PASS 2: Process all data nodes after control graphs are built
+        queue_items = list(flow_manager.global_flow_queue.queue)
+        for item in queue_items:
+            if item.dag_execution_type == DagExecutionType.DATA_NODE:
                 node = item.node
                 node.state = NodeResolutionState.UNRESOLVED
                 # Use proxy node if this node is part of a group, otherwise use original node
                 node_to_add = node
+                disconnected = True
                 # Only add if not already added (proxy might already be in DAG)
                 if node_to_add.name not in dag_builder.node_to_reference:
-                    dag_builder.add_node_with_dependencies(node_to_add, node_to_add.name)
+                    # Now, we need to create the DAG, but it can't be queued or used until it's dependencies have been resolved.
+                    # Figure out which graph the data node belongs to, if it belongs to a graph.
+                    for graph_start_node_name in dag_builder.graphs:
+                        graph_start_node = node_manager.get_node_by_name(graph_start_node_name)
+                        correct_graph = flow_manager.is_node_connected(graph_start_node, node)
+                        # This means this node is in the downstream connection of one of this graph.
+                        if correct_graph:
+                            # Is the node connected to a graph?
+                            disconnected = False
+                            if node.name not in dag_builder.start_node_candidates:
+                                dag_builder.start_node_candidates[node.name] = set()
+                            dag_builder.start_node_candidates[node.name].add(graph_start_node_name)
+                    if disconnected:
+                        # If the node is not connected to any graph, we can add it as it's own graph here.
+                        # It will not cause any overlapping confusion with existing graphs.
+                        dag_builder.add_node_with_dependencies(node_to_add, node_to_add.name)
                 flow_manager.global_flow_queue.queue.remove(item)
 
         return start_nodes

griptape_nodes/machines/dag_builder.py

@@ -6,8 +6,9 @@ from enum import StrEnum
 from typing import TYPE_CHECKING
 
 from griptape_nodes.common.directed_graph import DirectedGraph
+from griptape_nodes.exe_types.base_iterative_nodes import BaseIterativeStartNode
 from griptape_nodes.exe_types.core_types import ParameterTypeBuiltin
-from griptape_nodes.exe_types.node_types import LOCAL_EXECUTION, NodeGroupNode, NodeResolutionState
+from griptape_nodes.exe_types.node_types import NodeResolutionState
 
 if TYPE_CHECKING:
     import asyncio
@@ -44,58 +45,16 @@ class DagBuilder:
     graphs: dict[str, DirectedGraph]  # Str is the name of the start node associated here.
     node_to_reference: dict[str, DagNode]
     graph_to_nodes: dict[str, set[str]]  # Track which nodes belong to which graph
+    start_node_candidates: dict[str, set[str]]
 
     def __init__(self) -> None:
         self.graphs = {}
         self.node_to_reference: dict[str, DagNode] = {}
         self.graph_to_nodes = {}
-
-    @staticmethod
-    def _should_use_parent_group(node: BaseNode) -> bool:
-        """Check if a node's parent group should be used for DAG edges.
-
-        Returns True if the node has a parent NodeGroupNode that is NOT in LOCAL_EXECUTION mode.
-        In LOCAL_EXECUTION mode, groups are transparent and children are treated as separate nodes.
-        """
-        if not isinstance(node.parent_group, NodeGroupNode):
-            return False
-        parent_execution_env = node.parent_group.get_parameter_value(node.parent_group.execution_environment.name)
-        return parent_execution_env != LOCAL_EXECUTION
-
-    def _get_node_for_dag_edge(self, node: BaseNode, graph: DirectedGraph, graph_name: str) -> BaseNode:
-        """Get the node to use for DAG edges - either the node itself or its parent group.
-
-        Args:
-            node: The original node
-            graph: The graph being built
-            graph_name: Name of the graph for tracking
-
-        Returns:
-            The node or parent group to use in DAG edges
-        """
-        if self._should_use_parent_group(node):
-            parent_group = node.parent_group
-            if isinstance(parent_group, NodeGroupNode):
-                self._ensure_group_node_in_dag(parent_group, graph, graph_name)
-                return parent_group
-        return node
-
-    def _ensure_group_node_in_dag(self, group_node: NodeGroupNode, graph: DirectedGraph, graph_name: str) -> None:
-        """Ensure a NodeGroupNode is added to the DAG if not already present.
-
-        Args:
-            group_node: The NodeGroupNode to add
-            graph: The graph to add it to
-            graph_name: Name of the graph for tracking
-        """
-        if group_node.name not in self.node_to_reference:
-            dag_node = DagNode(node_reference=group_node, node_state=NodeState.WAITING)
-            self.node_to_reference[group_node.name] = dag_node
-            graph.add_node(node_for_adding=group_node.name)
-            self.graph_to_nodes[graph_name].add(group_node.name)
+        self.start_node_candidates = {}
 
     # Complex with the inner recursive method, but it needs connections and added_nodes.
-    def add_node_with_dependencies(self, node: BaseNode, graph_name: str = "default") -> list[BaseNode]:  # noqa: C901
+    def add_node_with_dependencies(self, node: BaseNode, graph_name: str = "default") -> list[BaseNode]:
         """Add node and all its dependencies to DAG. Returns list of added nodes."""
         from griptape_nodes.retained_mode.griptape_nodes import GriptapeNodes
 
@@ -115,6 +74,14 @@ class DagBuilder:
 
             if current_node.name in self.node_to_reference:
                 return
+            # Add current node to tracking
+            dag_node = DagNode(node_reference=current_node, node_state=NodeState.WAITING)
+            self.node_to_reference[current_node.name] = dag_node
+            added_nodes.append(current_node)
+
+            # Add to graph
+            graph.add_node(node_for_adding=current_node.name)
+            self.graph_to_nodes[graph_name].add(current_node.name)
 
             # Check if we should ignore dependencies (for special nodes like output_selector)
             ignore_data_dependencies = hasattr(current_node, "ignore_dependencies")
@@ -129,7 +96,7 @@ class DagBuilder:
                 if ignore_data_dependencies:
                     continue
 
-                upstream_connection = connections.get_connected_node(current_node, param)
+                upstream_connection = connections.get_connected_node(current_node, param, include_internal=False)
                 if not upstream_connection:
                     continue
 
@@ -139,30 +106,11 @@ class DagBuilder:
                 if upstream_node.state == NodeResolutionState.RESOLVED:
                     continue
 
-                # Check for internal group connections - traverse but don't add edge
-                is_internal_connection = (
-                    self._should_use_parent_group(current_node)
-                    and upstream_node.parent_group == current_node.parent_group
-                )
-
                 # Recursively add upstream node
                 _add_node_recursive(upstream_node, visited, graph)
 
-                # Add edge unless it's an internal group connection
-                if not is_internal_connection:
-                    upstream_for_edge = self._get_node_for_dag_edge(upstream_node, graph, graph_name)
-                    current_for_edge = self._get_node_for_dag_edge(current_node, graph, graph_name)
-                    graph.add_edge(upstream_for_edge.name, current_for_edge.name)
-
-            # Always add current node to tracking (even if parent group is used for edges)
-            dag_node = DagNode(node_reference=current_node, node_state=NodeState.WAITING)
-            self.node_to_reference[current_node.name] = dag_node
-            added_nodes.append(current_node)
-
-            # Add to graph if not using parent group
-            if not self._should_use_parent_group(current_node):
-                graph.add_node(node_for_adding=current_node.name)
-                self.graph_to_nodes[graph_name].add(current_node.name)
+                # Add edge from upstream to current
+                graph.add_edge(upstream_node.name, current_node.name)
 
         _add_node_recursive(node, set(), graph)
 
@@ -193,6 +141,7 @@ class DagBuilder:
         self.graphs.clear()
         self.node_to_reference.clear()
         self.graph_to_nodes.clear()
+        self.start_node_candidates.clear()
 
     def can_queue_control_node(self, node: DagNode) -> bool:
         if len(self.graphs) == 1:
@@ -203,7 +152,8 @@ class DagBuilder:
         connections = GriptapeNodes.FlowManager().get_connections()
 
         control_connections = self.get_number_incoming_control_connections(node.node_reference, connections)
-        if control_connections <= 1:
+        # If no control connections, we can queue this! Don't worry about this.
+        if control_connections == 0:
             return True
 
         for graph in self.graphs.values():
@@ -239,10 +189,119 @@ class DagBuilder:
             # Find the parameter to check if it's a control type
             param = node.get_parameter_by_name(param_name)
             if param and ParameterTypeBuiltin.CONTROL_TYPE.value in param.input_types:
-                control_connection_count += len(connection_ids)
+                # Skip connections from end node or itself if this is a BaseIterativeStartNode
+                if isinstance(node, BaseIterativeStartNode):
+                    for connection_id in connection_ids:
+                        if connection_id in connections.connections:
+                            connection = connections.connections[connection_id]
+                            source_node = connection.source_node
+                            # Skip if connection is from end node or itself
+                            if source_node in (node.end_node, node):
+                                continue
+                            control_connection_count += 1
+                else:
+                    control_connection_count += len(connection_ids)
 
         return control_connection_count
 
+    @staticmethod
+    def collect_nodes_in_forward_control_path(
+        start_node: BaseNode, end_node: BaseNode, connections: Connections
+    ) -> set[str]:
+        """Collect all nodes in the forward control path from start_node to end_node.
+
+        Args:
+            start_node: The node to start traversal from
+            end_node: The node to stop traversal at (inclusive)
+            connections: The connections manager
+
+        Returns:
+            Set of node names in the control path from start to end (inclusive)
+        """
+        nodes_in_path: set[str] = set()
+        to_visit = [start_node]
+        visited = set()
+
+        while to_visit:
+            current_node = to_visit.pop(0)
+
+            if current_node.name in visited:
+                continue
+            visited.add(current_node.name)
+
+            # Add to our collection
+            nodes_in_path.add(current_node.name)
+
+            # Stop if we've reached the end node
+            if current_node == end_node:
+                continue
+
+            # Find all outgoing control connections
+            if current_node.name in connections.outgoing_index:
+                for param_name, connection_ids in connections.outgoing_index[current_node.name].items():
+                    param = current_node.get_parameter_by_name(param_name)
+                    if param and param.output_type == ParameterTypeBuiltin.CONTROL_TYPE.value:
+                        for connection_id in connection_ids:
+                            if connection_id in connections.connections:
+                                connection = connections.connections[connection_id]
+                                next_node = connection.target_node
+                                if next_node.name not in visited and not connection.is_node_group_internal:
+                                    to_visit.append(next_node)
+
+        return nodes_in_path
+
+    @staticmethod
+    def collect_data_dependencies_for_node(
+        node: BaseNode, connections: Connections, nodes_to_exclude: set[str], visited: set[str]
+    ) -> set[str]:
+        """Collect data dependencies for a node recursively.
+
+        Args:
+            node: The node to collect dependencies for
+            connections: The connections manager
+            nodes_to_exclude: Set of nodes to exclude (e.g., nodes already in control flow)
+            visited: Set of already visited dependency nodes (modified in place)
+
+        Returns:
+            Set of dependency node names
+        """
+        if node.name in visited:
+            return set()
+
+        visited.add(node.name)
+        dependencies: set[str] = set()
+
+        # Check for ignore_dependencies attribute (like output_selector)
+        ignore_data_dependencies = hasattr(node, "ignore_dependencies")
+        if ignore_data_dependencies:
+            return dependencies
+
+        # Process each parameter looking for data dependencies
+        for param in node.parameters:
+            # Skip control parameters
+            if param.type == ParameterTypeBuiltin.CONTROL_TYPE:
+                continue
+
+            # Get upstream data connection
+            upstream_connection = connections.get_connected_node(node, param, include_internal=False)
+            if upstream_connection:
+                upstream_node, _ = upstream_connection
+
+                # Skip if already resolved
+                if upstream_node.state == NodeResolutionState.RESOLVED:
+                    continue
+
+                # Only add if it's not in the exclusion set
+                if upstream_node.name not in nodes_to_exclude:
+                    dependencies.add(upstream_node.name)
+                    # Recursively collect dependencies of this dependency
+                    sub_deps = DagBuilder.collect_data_dependencies_for_node(
+                        upstream_node, connections, nodes_to_exclude, visited
+                    )
+                    dependencies.update(sub_deps)
+
+        return dependencies
+
     def _is_node_in_forward_path(
         self, start_node: BaseNode, target_node: BaseNode, connections: Connections, visited: set[str] | None = None
     ) -> bool:
@@ -282,3 +341,22 @@ class DagBuilder:
             for node_name in self.graph_to_nodes[graph_name]:
                 self.node_to_reference.pop(node_name, None)
             self.graph_to_nodes.pop(graph_name, None)
+
+    def remove_graph_from_dependencies(self) -> list[str]:
+        # Check all start node candidates and return those whose dependent graphs are all empty
+        start_nodes = []
+        # copy because we will be removing as iterating.
+        for start_node_name, graph_deps in self.start_node_candidates.copy().items():
+            # Check if all graphs this start node depends on are now empty
+            all_deps_empty = True
+            for graph_deps_name in graph_deps:
+                # Check if this graph exists and has nodes
+                if graph_deps_name in self.graphs and len(self.graphs[graph_deps_name].nodes()) > 0:
+                    all_deps_empty = False
+                    break
+
+            # If all dependent graphs are empty, this start node can be queued
+            if all_deps_empty:
+                del self.start_node_candidates[start_node_name]
+                start_nodes.append(start_node_name)
+        return start_nodes