Flowfile 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

flowfile_core/flowfile/flow_graph.py

@@ -1,27 +1,30 @@
 import datetime
 import pickle
+
+import os
+
 import polars as pl
+
 import fastexcel
-import re
 from fastapi.exceptions import HTTPException
 from time import time
 from functools import partial
-from typing import List, Dict, Union, Callable, Any, Optional, Tuple
+from typing import List, Dict, Union, Callable, Any, Optional, Tuple, Literal
 from uuid import uuid1
 from copy import deepcopy
 from pyarrow.parquet import ParquetFile
 from flowfile_core.configs import logger
-from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 from flowfile_core.configs.flow_logger import FlowLogger
 from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import cast_str_to_polars_type, FlowfileColumn
-from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.settings_validator import (calculate_fuzzy_match_schema,
-                                                                                       pre_calculate_pivot_schema)
+
 from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import CloudStorageReader
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine, execute_polars_code
-from flowfile_core.flowfile.flow_data_engine.read_excel_tables import get_open_xlsx_datatypes, \
-    get_calamine_xlsx_data_types
+from flowfile_core.flowfile.flow_data_engine.read_excel_tables import (get_open_xlsx_datatypes,
+                                                                       get_calamine_xlsx_data_types)
+
+from flowfile_core.flowfile.schema_callbacks import (calculate_fuzzy_match_schema, pre_calculate_pivot_schema)
 from flowfile_core.flowfile.sources import external_sources
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 from flowfile_core.schemas.output_model import NodeData, NodeResult, RunInformation
@@ -32,7 +35,11 @@ from flowfile_core.schemas.cloud_storage_schemas import (CloudStorageReadSetting
 from flowfile_core.flowfile.utils import snake_case_to_camel_case
 from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
-from flowfile_core.flowfile.util.execution_orderer import determine_execution_order
+from flowfile_core.flowfile.util.execution_orderer import compute_execution_plan
+from flowfile_core.flowfile.graph_tree.graph_tree import (add_un_drawn_nodes, build_flow_paths,
+                                                          build_node_info, calculate_depth,
+                                                          define_node_connections, draw_merged_paths,
+                                                          draw_standalone_paths, group_nodes_by_depth)
 from flowfile_core.flowfile.flow_data_engine.polars_code_parser import polars_code_parser
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import (ExternalDatabaseFetcher,
                                                                                                  ExternalDatabaseWriter,
@@ -172,12 +179,10 @@ class FlowGraph:
     schema: Optional[List[FlowfileColumn]] = None
     has_over_row_function: bool = False
     _flow_starts: List[Union[int, str]] = None
-    node_results: List[NodeResult] = None
     latest_run_info: Optional[RunInformation] = None
     start_datetime: datetime = None
     end_datetime: datetime = None
-    nodes_completed: int = 0
-    flow_settings: schemas.FlowSettings = None
+    _flow_settings: schemas.FlowSettings = None
     flow_logger: FlowLogger
 
     def __init__(self,
@@ -201,13 +206,11 @@ class FlowGraph:
         if isinstance(flow_settings, schemas.FlowGraphConfig):
             flow_settings = schemas.FlowSettings.from_flow_settings_input(flow_settings)
 
-        self.flow_settings = flow_settings
+        self._flow_settings = flow_settings
         self.uuid = str(uuid1())
-        self.nodes_completed = 0
         self.start_datetime = None
         self.end_datetime = None
         self.latest_run_info = None
-        self.node_results = []
         self._flow_id = flow_settings.flow_id
         self.flow_logger = FlowLogger(flow_settings.flow_id)
         self._flow_starts: List[FlowNode] = []
@@ -226,6 +229,19 @@ class FlowGraph:
         elif input_flow is not None:
             self.add_datasource(input_file=input_flow)
 
+    @property
+    def flow_settings(self) -> schemas.FlowSettings:
+        return self._flow_settings
+
+    @flow_settings.setter
+    def flow_settings(self, flow_settings: schemas.FlowSettings):
+        if (
+                (self._flow_settings.execution_location != flow_settings.execution_location) or
+                (self._flow_settings.execution_mode != flow_settings.execution_mode)
+        ):
+            self.reset()
+        self._flow_settings = flow_settings
+
     def add_node_promise(self, node_promise: input_schema.NodePromise):
         """Adds a placeholder node to the graph that is not yet fully configured.
 
@@ -242,64 +258,6 @@ class FlowGraph:
         self.add_node_step(node_id=node_promise.node_id, node_type=node_promise.node_type, function=placeholder,
                            setting_input=node_promise)
 
-    def print_tree(self, show_schema=False, show_descriptions=False):
-        """
-        Print flow_graph as a tree.
-        """
-        max_node_id = max(self._node_db.keys())
-
-        tree = ""
-        tabs = 0
-        tab_counter = 0
-        for node in self.nodes:
-            tab_counter += 1
-            node_input = node.setting_input
-            operation = str(self._node_db[node_input.node_id]).split("(")[1][:-1].replace("_", " ").title()
-
-            if operation == "Formula":
-                operation = "With Columns"
-
-            tree += str(operation) + " (id=" + str(node_input.node_id) + ")"
-
-            if show_descriptions & show_schema:
-                raise ValueError('show_descriptions and show_schema cannot be True simultaneously')
-            if show_descriptions:
-                tree += ": " + str(node_input.description)
-            elif show_schema:
-                tree += " -> ["
-                if operation == "Manual Input":
-                    schema = ", ".join([str(i.name) + ": " + str(i.data_type) for i in node_input.raw_data_format.columns])
-                    tree += schema
-                elif operation == "With Columns":
-                    tree_with_col_schema = ", " + node_input.function.field.name + ": " + node_input.function.field.data_type
-                    tree += schema + tree_with_col_schema
-                elif operation == "Filter":
-                    index = node_input.filter_input.advanced_filter.find("]")
-                    filtered_column = str(node_input.filter_input.advanced_filter[1:index])
-                    schema = re.sub('({str(filtered_column)}: [A-Za-z0-9]+\,\s)', "", schema)
-                    tree += schema
-                elif operation == "Group By":
-                    for col in node_input.groupby_input.agg_cols:
-                        schema = re.sub(str(col.old_name) + ': [a-z0-9]+\, ', "", schema)
-                    tree += schema
-                tree += "]"
-            else:
-                if operation == "Manual Input":
-                    tree += ": " + str(node_input.raw_data_format.data)
-                elif operation == "With Columns":
-                    tree += ": " + str(node_input.function)
-                elif operation == "Filter":
-                    tree += ": " + str(node_input.filter_input.advanced_filter)
-                elif operation == "Group By":
-                    tree += ": groupby=[" + ", ".join([col.old_name for col in node_input.groupby_input.agg_cols if col.agg == "groupby"]) + "], "
-                    tree += "agg=[" + ", ".join([str(col.agg) + "(" + str(col.old_name) + ")" for col in node_input.groupby_input.agg_cols if col.agg != "groupby"]) + "]"
-
-            if node_input.node_id < max_node_id:
-                tree += "\n" + "# " + " "*3*(tabs-1) + "|___ "
-            print("\n"*2)
-
-        return print(tree)
-
     def apply_layout(self, y_spacing: int = 150, x_spacing: int = 200, initial_y: int = 100):
         """Calculates and applies a layered layout to all nodes in the graph.
 
@@ -368,6 +326,86 @@ class FlowGraph:
         settings_str = "  -" + '\n  -'.join(f"{k}: {v}" for k, v in self.flow_settings)
         return f"FlowGraph(\nNodes: {self._node_db}\n\nSettings:\n{settings_str}"
 
+    def print_tree(self):
+        """Print flow_graph as a visual tree structure, showing the DAG relationships with ASCII art."""
+        if not self._node_db:
+            self.flow_logger.info("Empty flow graph")
+            return
+
+        # Build node information
+        node_info = build_node_info(self.nodes)
+
+        # Calculate depths for all nodes
+        for node_id in node_info:
+            calculate_depth(node_id, node_info)
+
+        # Group nodes by depth
+        depth_groups, max_depth = group_nodes_by_depth(node_info)
+        
+        # Sort nodes within each depth group
+        for depth in depth_groups:
+            depth_groups[depth].sort()
+
+        # Create the main flow visualization
+        lines = ["=" * 80, "Flow Graph Visualization", "=" * 80, ""]
+
+        # Track which nodes connect to what
+        merge_points = define_node_connections(node_info)
+        
+        # Build the flow paths
+
+        # Find the maximum label length for each depth level
+        max_label_length = {}
+        for depth in range(max_depth + 1):
+            if depth in depth_groups:
+                max_len = max(len(node_info[nid].label) for nid in depth_groups[depth])
+                max_label_length[depth] = max_len
+        
+        # Draw the paths
+        drawn_nodes = set()
+        merge_drawn = set()
+        
+        # Group paths by their merge points
+        paths_by_merge = {}
+        standalone_paths = []
+        
+        # Build flow paths
+        paths = build_flow_paths(node_info, self._flow_starts, merge_points)
+
+        # Define paths to merge and standalone paths
+        for path in paths:
+            if len(path) > 1 and path[-1] in merge_points and len(merge_points[path[-1]]) > 1:
+                merge_id = path[-1]
+                if merge_id not in paths_by_merge:
+                    paths_by_merge[merge_id] = []
+                paths_by_merge[merge_id].append(path)
+            else:
+                standalone_paths.append(path)
+
+        # Draw merged paths
+        draw_merged_paths(node_info, merge_points, paths_by_merge, merge_drawn, drawn_nodes, lines)
+
+        # Draw standlone paths
+        draw_standalone_paths(drawn_nodes, standalone_paths, lines, node_info)
+
+        # Add undrawn nodes
+        add_un_drawn_nodes(drawn_nodes, node_info, lines)
+        
+        try:
+            skip_nodes, ordered_nodes = compute_execution_plan(
+                nodes=self.nodes,
+                flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+            if ordered_nodes:
+                for i, node in enumerate(ordered_nodes, 1):
+                    lines.append(f"  {i:3d}. {node_info[node.node_id].label}")
+        except Exception as e:
+            lines.append(f"  Could not determine execution order: {e}")
+        
+        # Print everything
+        output = "\n".join(lines)
+        
+        print(output)
+        
     def get_nodes_overview(self):
         """Gets a list of dictionary representations for all nodes in the graph."""
         output = []
@@ -774,26 +812,34 @@ class FlowGraph:
         """
 
         def _func(main: FlowDataEngine, right: FlowDataEngine) -> FlowDataEngine:
-            f = main.start_fuzzy_join(fuzzy_match_input=fuzzy_settings.join_input, other=right, file_ref=node.hash,
+            node = self.get_node(node_id=fuzzy_settings.node_id)
+            if self.execution_location == "local":
+                return main.fuzzy_join(fuzzy_match_input=deepcopy(fuzzy_settings.join_input),
+                                       other=right,
+                                       node_logger=self.flow_logger.get_node_logger(fuzzy_settings.node_id))
+
+            f = main.start_fuzzy_join(fuzzy_match_input=deepcopy(fuzzy_settings.join_input), other=right, file_ref=node.hash,
                                       flow_id=self.flow_id, node_id=fuzzy_settings.node_id)
             logger.info("Started the fuzzy match action")
-            node._fetch_cached_df = f
+            node._fetch_cached_df = f  # Add to the node so it can be cancelled and fetch later if needed
             return FlowDataEngine(f.get_result())
 
-        self.add_node_step(node_id=fuzzy_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='fuzzy_match',
-                           setting_input=fuzzy_settings)
-        node = self.get_node(node_id=fuzzy_settings.node_id)
-
         def schema_callback():
-            return calculate_fuzzy_match_schema(fuzzy_settings.join_input,
+            fm_input_copy = deepcopy(fuzzy_settings.join_input)  # Deepcopy create an unique object per func
+            node = self.get_node(node_id=fuzzy_settings.node_id)
+            return calculate_fuzzy_match_schema(fm_input_copy,
                                                 left_schema=node.node_inputs.main_inputs[0].schema,
                                                 right_schema=node.node_inputs.right_input.schema
                                                 )
 
-        node.schema_callback = schema_callback
+        self.add_node_step(node_id=fuzzy_settings.node_id,
+                           function=_func,
+                           input_columns=[],
+                           node_type='fuzzy_match',
+                           setting_input=fuzzy_settings,
+                           input_node_ids=fuzzy_settings.depending_on_ids,
+                           schema_callback=schema_callback)
+
         return self
 
     def add_text_to_rows(self, node_text_to_rows: input_schema.NodeTextToRows) -> "FlowGraph":
@@ -1549,8 +1595,70 @@ class FlowGraph:
         Args:
             execution_location: The execution location to set.
         """
+        if self.flow_settings.execution_location != execution_location:
+            self.reset()
         self.flow_settings.execution_location = execution_location
 
+    def validate_if_node_can_be_fetched(self, node_id: int) -> None:
+        flow_node = self._node_db.get(node_id)
+        if not flow_node:
+            raise Exception("Node not found found")
+        skip_nodes, execution_order = compute_execution_plan(
+            nodes=self.nodes, flow_starts=self._flow_starts+self.get_implicit_starter_nodes()
+        )
+        if flow_node.node_id in [skip_node.node_id for skip_node in skip_nodes]:
+            raise Exception("Node can not be executed because it does not have it's inputs")
+
+    def create_initial_run_information(self, number_of_nodes: int,
+                                       run_type: Literal["fetch_one", "full_run"]):
+        return RunInformation(
+            flow_id=self.flow_id, start_time=datetime.datetime.now(), end_time=None,
+            success=None, number_of_nodes=number_of_nodes, node_step_result=[],
+            run_type=run_type
+        )
+
+    def trigger_fetch_node(self, node_id: int) -> RunInformation | None:
+        """Executes a specific node in the graph by its ID."""
+        if self.flow_settings.is_running:
+            raise Exception("Flow is already running")
+        flow_node = self.get_node(node_id)
+        self.flow_settings.is_running = True
+        self.flow_settings.is_canceled = False
+        self.flow_logger.clear_log_file()
+        self.latest_run_info = self.create_initial_run_information(1, "fetch_one")
+        node_logger = self.flow_logger.get_node_logger(flow_node.node_id)
+        node_result = NodeResult(node_id=flow_node.node_id, node_name=flow_node.name)
+        logger.info(f'Starting to run: node {flow_node.node_id}, start time: {node_result.start_timestamp}')
+        try:
+            self.latest_run_info.node_step_result.append(node_result)
+            flow_node.execute_node(run_location=self.flow_settings.execution_location,
+                                   performance_mode=False,
+                                   node_logger=node_logger,
+                                   optimize_for_downstream=False,
+                                   reset_cache=True)
+            node_result.error = str(flow_node.results.errors)
+            if self.flow_settings.is_canceled:
+                node_result.success = None
+                node_result.success = None
+                node_result.is_running = False
+            node_result.success = flow_node.results.errors is None
+            node_result.end_timestamp = time()
+            node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
+            node_result.is_running = False
+            self.latest_run_info.nodes_completed += 1
+            self.latest_run_info.end_time = datetime.datetime.now()
+            self.flow_settings.is_running = False
+            return self.get_run_info()
+        except Exception as e:
+            node_result.error = 'Node did not run'
+            node_result.success = False
+            node_result.end_timestamp = time()
+            node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
+            node_result.is_running = False
+            node_logger.error(f'Error in node {flow_node.node_id}: {e}')
+        finally:
+            self.flow_settings.is_running = False
+
     def run_graph(self) -> RunInformation | None:
         """Executes the entire data flow graph from start to finish.
 
@@ -1566,27 +1674,23 @@ class FlowGraph:
         if self.flow_settings.is_running:
             raise Exception('Flow is already running')
         try:
+
             self.flow_settings.is_running = True
             self.flow_settings.is_canceled = False
             self.flow_logger.clear_log_file()
-            self.nodes_completed = 0
-            self.node_results = []
-            self.start_datetime = datetime.datetime.now()
-            self.end_datetime = None
-            self.latest_run_info = None
             self.flow_logger.info('Starting to run flowfile flow...')
-            skip_nodes = [node for node in self.nodes if not node.is_correct]
-            skip_nodes.extend([lead_to_node for node in skip_nodes for lead_to_node in node.leads_to_nodes])
-            execution_order = determine_execution_order(all_nodes=[node for node in self.nodes if
-                                                                   node not in skip_nodes],
-                                                        flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+
+            skip_nodes, execution_order = compute_execution_plan(
+                nodes=self.nodes,
+                flow_starts=self._flow_starts+self.get_implicit_starter_nodes()
+            )
+
+            self.latest_run_info = self.create_initial_run_information(len(execution_order), "full_run")
+
             skip_node_message(self.flow_logger, skip_nodes)
             execution_order_message(self.flow_logger, execution_order)
             performance_mode = self.flow_settings.execution_mode == 'Performance'
-            if self.flow_settings.execution_location == 'local':
-                OFFLOAD_TO_WORKER.value = False
-            elif self.flow_settings.execution_location == 'remote':
-                OFFLOAD_TO_WORKER.value = True
+
             for node in execution_order:
                 node_logger = self.flow_logger.get_node_logger(node.node_id)
                 if self.flow_settings.is_canceled:
@@ -1596,7 +1700,7 @@ class FlowGraph:
                     node_logger.info(f'Skipping node {node.node_id}')
                     continue
                 node_result = NodeResult(node_id=node.node_id, node_name=node.name)
-                self.node_results.append(node_result)
+                self.latest_run_info.node_step_result.append(node_result)
                 logger.info(f'Starting to run: node {node.node_id}, start time: {node_result.start_timestamp}')
                 node.execute_node(run_location=self.flow_settings.execution_location,
                                   performance_mode=performance_mode,
@@ -1622,7 +1726,7 @@ class FlowGraph:
                 if not node_result.success:
                     skip_nodes.extend(list(node.get_all_dependent_nodes()))
                 node_logger.info(f'Completed node with success: {node_result.success}')
-                self.nodes_completed += 1
+                self.latest_run_info.nodes_completed += 1
             self.flow_logger.info('Flow completed!')
             self.end_datetime = datetime.datetime.now()
             self.flow_settings.is_running = False
@@ -1634,28 +1738,23 @@ class FlowGraph:
         finally:
             self.flow_settings.is_running = False
 
-    def get_run_info(self) -> RunInformation:
+    def get_run_info(self) -> RunInformation | None:
         """Gets a summary of the most recent graph execution.
 
         Returns:
             A RunInformation object with details about the last run.
         """
+        is_running = self.flow_settings.is_running
         if self.latest_run_info is None:
-            node_results = self.node_results
-            success = all(nr.success for nr in node_results)
-            self.latest_run_info = RunInformation(start_time=self.start_datetime, end_time=self.end_datetime,
-                                                  success=success,
-                                                  node_step_result=node_results, flow_id=self.flow_id,
-                                                  nodes_completed=self.nodes_completed,
-                                                  number_of_nodes=len(self.nodes))
-        elif self.latest_run_info.nodes_completed != self.nodes_completed:
-            node_results = self.node_results
-            self.latest_run_info = RunInformation(start_time=self.start_datetime, end_time=self.end_datetime,
-                                                  success=all(nr.success for nr in node_results),
-                                                  node_step_result=node_results, flow_id=self.flow_id,
-                                                  nodes_completed=self.nodes_completed,
-                                                  number_of_nodes=len(self.nodes))
-        return self.latest_run_info
+            return
+
+        elif not is_running and self.latest_run_info.success is not None:
+            return self.latest_run_info
+
+        run_info = self.latest_run_info
+        if not is_running:
+            run_info.success = all(nr.success for nr in run_info.node_step_result)
+        return run_info
 
     @property
     def node_connections(self) -> List[Tuple[int, int]]:
@@ -1726,8 +1825,14 @@ class FlowGraph:
         Args:
             flow_path: The path where the flow file will be saved.
         """
-        with open(flow_path, 'wb') as f:
-            pickle.dump(self.get_node_storage(), f)
+        logger.info("Saving flow to %s", flow_path)
+        os.makedirs(os.path.dirname(flow_path), exist_ok=True)
+        try:
+            with open(flow_path, 'wb') as f:
+                pickle.dump(self.get_node_storage(), f)
+        except Exception as e:
+            logger.error(f"Error saving flow: {e}")
+
         self.flow_settings.path = flow_path
 
     def get_frontend_data(self) -> dict:
@@ -1922,4 +2027,4 @@ def delete_connection(graph, node_connection: input_schema.NodeConnection):
         to_node.delete_input_node(
             node_connection.output_connection.node_id,
             connection_type=node_connection.input_connection.connection_class,
-        )
+        )

flowfile_core/flowfile/flow_node/flow_node.py

@@ -5,7 +5,6 @@ from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEng
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.schemas import input_schema, schemas
 from flowfile_core.configs.flow_logger import NodeLogger
-from flowfile_core.configs.settings import SINGLE_FILE_MODE, OFFLOAD_TO_WORKER
 
 from flowfile_core.schemas.output_model import TableExample, FileColumn, NodeData
 from flowfile_core.flowfile.utils import get_hash
@@ -13,7 +12,7 @@ from flowfile_core.configs.node_store import nodes as node_interface
 from flowfile_core.flowfile.setting_generator import setting_generator, setting_updator
 from time import sleep
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations import (
-    ExternalDfFetcher, ExternalSampler, results_exists, get_external_df_result,
+    ExternalDfFetcher, ExternalSampler, clear_task_from_worker, results_exists, get_external_df_result,
     ExternalDatabaseFetcher, ExternalDatabaseWriter, ExternalCloudWriter)
 from flowfile_core.flowfile.flow_node.models import (NodeStepSettings, NodeStepInputs, NodeSchemaInformation,
                                                      NodeStepStats, NodeResults)
@@ -679,9 +678,10 @@ class FlowNode:
 
         if results_exists(self.hash):
             logger.warning('Not implemented')
+            clear_task_from_worker(self.hash)
 
     def needs_run(self, performance_mode: bool, node_logger: NodeLogger = None,
-                  execution_location: schemas.ExecutionLocationsLiteral = "auto") -> bool:
+                  execution_location: schemas.ExecutionLocationsLiteral = "remote") -> bool:
         """Determines if the node needs to be executed.
 
         The decision is based on its run state, caching settings, and execution mode.
@@ -694,7 +694,7 @@ class FlowNode:
         Returns:
             True if the node should be run, False otherwise.
         """
-        if execution_location == "local" or SINGLE_FILE_MODE:
+        if execution_location == "local":
             return False
 
         flow_logger = logger if node_logger is None else node_logger
@@ -724,6 +724,8 @@ class FlowNode:
         Raises:
             Exception: Propagates exceptions from the execution.
         """
+        self.clear_table_example()
+
         def example_data_generator():
             example_data = None
 
@@ -736,6 +738,7 @@ class FlowNode:
         resulting_data = self.get_resulting_data()
 
         if not performance_mode:
+            self.node_stats.has_run_with_current_setup = True
             self.results.example_data_generator = example_data_generator()
             self.node_schema.result_schema = self.results.resulting_data.schema
             self.node_stats.has_completed_last_run = True
@@ -855,8 +858,12 @@ class FlowNode:
             logger.warning('No external process to cancel')
         self.node_stats.is_canceled = True
 
-    def execute_node(self, run_location: schemas.ExecutionLocationsLiteral, reset_cache: bool = False,
-                     performance_mode: bool = False, retry: bool = True, node_logger: NodeLogger = None):
+    def execute_node(self, run_location: schemas.ExecutionLocationsLiteral,
+                     reset_cache: bool = False,
+                     performance_mode: bool = False,
+                     retry: bool = True,
+                     node_logger: NodeLogger = None,
+                     optimize_for_downstream: bool = True):
         """Orchestrates the execution, handling location, caching, and retries.
 
         Args:
@@ -865,25 +872,33 @@ class FlowNode:
             performance_mode: If True, optimizes for speed over diagnostics.
             retry: If True, allows retrying execution on recoverable errors.
             node_logger: The logger for this node execution.
+            optimize_for_downstream: If true, operations that shuffle the order of rows are fully cached and provided as
+            input to downstream steps
 
         Raises:
             Exception: If the node_logger is not defined.
         """
         if node_logger is None:
             raise Exception('Flow logger is not defined')
-        # node_logger = flow_logger.get_node_logger(self.node_id)
+        #  TODO: Simplify which route is being picked there are many duplicate checks
+
         if reset_cache:
             self.remove_cache()
             self.node_stats.has_run_with_current_setup = False
             self.node_stats.has_completed_last_run = False
+
         if self.is_setup:
             node_logger.info(f'Starting to run {self.__name__}')
             if (self.needs_run(performance_mode, node_logger, run_location) or self.node_template.node_group == "output"
-                    and not (run_location == 'local' or SINGLE_FILE_MODE)):
+                    and not (run_location == 'local')):
+                self.clear_table_example()
                 self.prepare_before_run()
+                self.reset()
                 try:
-                    if ((run_location == 'remote' or (self.node_default.transform_type == 'wide')
-                            and not run_location == 'local')) or self.node_settings.cache_results:
+                    if (((run_location == 'remote' or
+                         (self.node_default.transform_type == 'wide' and optimize_for_downstream) and
+                         not run_location == 'local'))
+                            or self.node_settings.cache_results):
                         node_logger.info('Running the node remotely')
                         if self.node_settings.cache_results:
                             performance_mode = False
@@ -908,8 +923,14 @@ class FlowNode:
                                           node_logger=node_logger)
                     else:
                         self.results.errors = str(e)
-                        node_logger.error(f'Error with running the node: {e}')
-            elif ((run_location == 'local' or SINGLE_FILE_MODE) and
+                        if "Connection refused" in str(e) and "/submit_query/" in str(e):
+                            node_logger.warning("There was an issue connecting to the remote worker, "
+                                                "ensure the worker process is running, "
+                                                "or change the settings to, so it executes locally")
+                            node_logger.error("Could not execute in the remote worker. (Re)start the worker service, or change settings to local settings.")
+                        else:
+                            node_logger.error(f'Error with running the node: {e}')
+            elif ((run_location == 'local') and
                   (not self.node_stats.has_run_with_current_setup or self.node_template.node_group == "output")):
                 try:
                     node_logger.info('Executing fully locally')
@@ -919,7 +940,7 @@ class FlowNode:
                     node_logger.error(f'Error with running the node: {e}')
                     self.node_stats.error = str(e)
                     self.node_stats.has_completed_last_run = False
-                self.node_stats.has_run_with_current_setup = True
+
             else:
                 node_logger.info('Node has already run, not running the node')
         else:
@@ -959,16 +980,16 @@ class FlowNode:
             logger.info(f'{self.node_id}: Node needs reset')
             self.node_stats.has_run_with_current_setup = False
             self.results.reset()
-            if self.is_correct:
-                self._schema_callback = None  # Ensure the schema callback is reset
-                if self.schema_callback:
-                    logger.info(f'{self.node_id}: Resetting the schema callback')
-                    self.schema_callback.start()
             self.node_schema.result_schema = None
             self.node_schema.predicted_schema = None
             self._hash = None
             self.node_information.is_setup = None
             self.results.errors = None
+            if self.is_correct:
+                self._schema_callback = None  # Ensure the schema callback is reset
+                if self.schema_callback:
+                    logger.info(f'{self.node_id}: Resetting the schema callback')
+                    self.schema_callback.start()
             self.evaluate_nodes()
             _ = self.hash  # Recalculate the hash after reset
 
@@ -1103,6 +1124,17 @@ class FlowNode:
         if self.singular_input:
             return self.all_inputs[0]
 
+    def clear_table_example(self) -> None:
+        """
+        Clear the table example in the results so that it clears the existing results
+        Returns:
+            None
+        """
+
+        self.results.example_data = None
+        self.results.example_data_generator = None
+        self.results.example_data_path = None
+
     def get_table_example(self, include_data: bool = False) -> TableExample | None:
         """Generates a `TableExample` model summarizing the node's output.
 
@@ -1131,10 +1163,15 @@ class FlowNode:
                 data = []
             schema = [FileColumn.model_validate(c.get_column_repr()) for c in self.schema]
             fl = self.get_resulting_data()
+            has_example_data = self.results.example_data_generator is not None
+
             return TableExample(node_id=self.node_id,
                                 name=str(self.node_id), number_of_records=999,
                                 number_of_columns=fl.number_of_fields,
-                                table_schema=schema, columns=fl.columns, data=data)
+                                table_schema=schema, columns=fl.columns, data=data,
+                                has_example_data=has_example_data,
+                                has_run_with_current_setup=self.node_stats.has_run_with_current_setup
+                                )
         else:
             logger.warning('getting the table example but the node has not run')
             try: