Flowfile 0.3.8__py3-none-any.whl → 0.3.9__py3-none-any.whl

flowfile_core/flowfile/flow_graph.py

@@ -2,7 +2,6 @@ import datetime
 import pickle
 import polars as pl
 import fastexcel
-import re
 from fastapi.exceptions import HTTPException
 from time import time
 from functools import partial
@@ -11,17 +10,17 @@ from uuid import uuid1
 from copy import deepcopy
 from pyarrow.parquet import ParquetFile
 from flowfile_core.configs import logger
-from flowfile_core.configs.settings import OFFLOAD_TO_WORKER
 from flowfile_core.configs.flow_logger import FlowLogger
 from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
 from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import cast_str_to_polars_type, FlowfileColumn
-from flowfile_core.flowfile.flow_data_engine.fuzzy_matching.settings_validator import (calculate_fuzzy_match_schema,
-                                                                                       pre_calculate_pivot_schema)
+
 from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import CloudStorageReader
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine, execute_polars_code
-from flowfile_core.flowfile.flow_data_engine.read_excel_tables import get_open_xlsx_datatypes, \
-    get_calamine_xlsx_data_types
+from flowfile_core.flowfile.flow_data_engine.read_excel_tables import (get_open_xlsx_datatypes,
+                                                                       get_calamine_xlsx_data_types)
+
+from flowfile_core.flowfile.schema_callbacks import (calculate_fuzzy_match_schema, pre_calculate_pivot_schema)
 from flowfile_core.flowfile.sources import external_sources
 from flowfile_core.schemas import input_schema, schemas, transform_schema
 from flowfile_core.schemas.output_model import NodeData, NodeResult, RunInformation
@@ -32,7 +31,11 @@ from flowfile_core.schemas.cloud_storage_schemas import (CloudStorageReadSetting
 from flowfile_core.flowfile.utils import snake_case_to_camel_case
 from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
 from flowfile_core.flowfile.flow_node.flow_node import FlowNode
-from flowfile_core.flowfile.util.execution_orderer import determine_execution_order
+from flowfile_core.flowfile.util.execution_orderer import compute_execution_plan
+from flowfile_core.flowfile.graph_tree.graph_tree import (add_un_drawn_nodes, build_flow_paths,
+                                                          build_node_info, calculate_depth,
+                                                          define_node_connections, draw_merged_paths,
+                                                          draw_standalone_paths, group_nodes_by_depth)
 from flowfile_core.flowfile.flow_data_engine.polars_code_parser import polars_code_parser
 from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import (ExternalDatabaseFetcher,
                                                                                                  ExternalDatabaseWriter,
@@ -177,7 +180,7 @@ class FlowGraph:
     start_datetime: datetime = None
     end_datetime: datetime = None
     nodes_completed: int = 0
-    flow_settings: schemas.FlowSettings = None
+    _flow_settings: schemas.FlowSettings = None
     flow_logger: FlowLogger
 
     def __init__(self,
@@ -201,7 +204,7 @@ class FlowGraph:
         if isinstance(flow_settings, schemas.FlowGraphConfig):
             flow_settings = schemas.FlowSettings.from_flow_settings_input(flow_settings)
 
-        self.flow_settings = flow_settings
+        self._flow_settings = flow_settings
         self.uuid = str(uuid1())
         self.nodes_completed = 0
         self.start_datetime = None
@@ -226,6 +229,19 @@ class FlowGraph:
         elif input_flow is not None:
             self.add_datasource(input_file=input_flow)
 
+    @property
+    def flow_settings(self) -> schemas.FlowSettings:
+        return self._flow_settings
+
+    @flow_settings.setter
+    def flow_settings(self, flow_settings: schemas.FlowSettings):
+        if (
+                (self._flow_settings.execution_location != flow_settings.execution_location) or
+                (self._flow_settings.execution_mode != flow_settings.execution_mode)
+        ):
+            self.reset()
+        self._flow_settings = flow_settings
+
     def add_node_promise(self, node_promise: input_schema.NodePromise):
         """Adds a placeholder node to the graph that is not yet fully configured.
 
@@ -242,64 +258,6 @@ class FlowGraph:
         self.add_node_step(node_id=node_promise.node_id, node_type=node_promise.node_type, function=placeholder,
                            setting_input=node_promise)
 
-    def print_tree(self, show_schema=False, show_descriptions=False):
-        """
-        Print flow_graph as a tree.
-        """
-        max_node_id = max(self._node_db.keys())
-
-        tree = ""
-        tabs = 0
-        tab_counter = 0
-        for node in self.nodes:
-            tab_counter += 1
-            node_input = node.setting_input
-            operation = str(self._node_db[node_input.node_id]).split("(")[1][:-1].replace("_", " ").title()
-
-            if operation == "Formula":
-                operation = "With Columns"
-
-            tree += str(operation) + " (id=" + str(node_input.node_id) + ")"
-
-            if show_descriptions & show_schema:
-                raise ValueError('show_descriptions and show_schema cannot be True simultaneously')
-            if show_descriptions:
-                tree += ": " + str(node_input.description)
-            elif show_schema:
-                tree += " -> ["
-                if operation == "Manual Input":
-                    schema = ", ".join([str(i.name) + ": " + str(i.data_type) for i in node_input.raw_data_format.columns])
-                    tree += schema
-                elif operation == "With Columns":
-                    tree_with_col_schema = ", " + node_input.function.field.name + ": " + node_input.function.field.data_type
-                    tree += schema + tree_with_col_schema
-                elif operation == "Filter":
-                    index = node_input.filter_input.advanced_filter.find("]")
-                    filtered_column = str(node_input.filter_input.advanced_filter[1:index])
-                    schema = re.sub('({str(filtered_column)}: [A-Za-z0-9]+\,\s)', "", schema)
-                    tree += schema
-                elif operation == "Group By":
-                    for col in node_input.groupby_input.agg_cols:
-                        schema = re.sub(str(col.old_name) + ': [a-z0-9]+\, ', "", schema)
-                    tree += schema
-                tree += "]"
-            else:
-                if operation == "Manual Input":
-                    tree += ": " + str(node_input.raw_data_format.data)
-                elif operation == "With Columns":
-                    tree += ": " + str(node_input.function)
-                elif operation == "Filter":
-                    tree += ": " + str(node_input.filter_input.advanced_filter)
-                elif operation == "Group By":
-                    tree += ": groupby=[" + ", ".join([col.old_name for col in node_input.groupby_input.agg_cols if col.agg == "groupby"]) + "], "
-                    tree += "agg=[" + ", ".join([str(col.agg) + "(" + str(col.old_name) + ")" for col in node_input.groupby_input.agg_cols if col.agg != "groupby"]) + "]"
-
-            if node_input.node_id < max_node_id:
-                tree += "\n" + "# " + " "*3*(tabs-1) + "|___ "
-            print("\n"*2)
-
-        return print(tree)
-
     def apply_layout(self, y_spacing: int = 150, x_spacing: int = 200, initial_y: int = 100):
         """Calculates and applies a layered layout to all nodes in the graph.
 
@@ -368,6 +326,86 @@ class FlowGraph:
         settings_str = "  -" + '\n  -'.join(f"{k}: {v}" for k, v in self.flow_settings)
         return f"FlowGraph(\nNodes: {self._node_db}\n\nSettings:\n{settings_str}"
 
+    def print_tree(self):
+        """Print flow_graph as a visual tree structure, showing the DAG relationships with ASCII art."""
+        if not self._node_db:
+            self.flow_logger.info("Empty flow graph")
+            return
+
+        # Build node information
+        node_info = build_node_info(self.nodes)
+
+        # Calculate depths for all nodes
+        for node_id in node_info:
+            calculate_depth(node_id, node_info)
+
+        # Group nodes by depth
+        depth_groups, max_depth = group_nodes_by_depth(node_info)
+        
+        # Sort nodes within each depth group
+        for depth in depth_groups:
+            depth_groups[depth].sort()
+
+        # Create the main flow visualization
+        lines = ["=" * 80, "Flow Graph Visualization", "=" * 80, ""]
+
+        # Track which nodes connect to what
+        merge_points = define_node_connections(node_info)
+        
+        # Build the flow paths
+
+        # Find the maximum label length for each depth level
+        max_label_length = {}
+        for depth in range(max_depth + 1):
+            if depth in depth_groups:
+                max_len = max(len(node_info[nid].label) for nid in depth_groups[depth])
+                max_label_length[depth] = max_len
+        
+        # Draw the paths
+        drawn_nodes = set()
+        merge_drawn = set()
+        
+        # Group paths by their merge points
+        paths_by_merge = {}
+        standalone_paths = []
+        
+        # Build flow paths
+        paths = build_flow_paths(node_info, self._flow_starts, merge_points)
+
+        # Define paths to merge and standalone paths
+        for path in paths:
+            if len(path) > 1 and path[-1] in merge_points and len(merge_points[path[-1]]) > 1:
+                merge_id = path[-1]
+                if merge_id not in paths_by_merge:
+                    paths_by_merge[merge_id] = []
+                paths_by_merge[merge_id].append(path)
+            else:
+                standalone_paths.append(path)
+
+        # Draw merged paths
+        draw_merged_paths(node_info, merge_points, paths_by_merge, merge_drawn, drawn_nodes, lines)
+
+        # Draw standlone paths
+        draw_standalone_paths(drawn_nodes, standalone_paths, lines, node_info)
+
+        # Add undrawn nodes
+        add_un_drawn_nodes(drawn_nodes, node_info, lines)
+        
+        try:
+            skip_nodes, ordered_nodes = compute_execution_plan(
+                nodes=self.nodes,
+                flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+            if ordered_nodes:
+                for i, node in enumerate(ordered_nodes, 1):
+                    lines.append(f"  {i:3d}. {node_info[node.node_id].label}")
+        except Exception as e:
+            lines.append(f"  Could not determine execution order: {e}")
+        
+        # Print everything
+        output = "\n".join(lines)
+        
+        print(output)
+        
     def get_nodes_overview(self):
         """Gets a list of dictionary representations for all nodes in the graph."""
         output = []
@@ -774,26 +812,34 @@ class FlowGraph:
         """
 
         def _func(main: FlowDataEngine, right: FlowDataEngine) -> FlowDataEngine:
+            node = self.get_node(node_id=fuzzy_settings.node_id)
+            if self.execution_location == "local":
+                return main.fuzzy_join(fuzzy_match_input=fuzzy_settings.join_input,
+                                       other=right,
+                                       node_logger=self.flow_logger.get_node_logger(fuzzy_settings.node_id))
+
             f = main.start_fuzzy_join(fuzzy_match_input=fuzzy_settings.join_input, other=right, file_ref=node.hash,
                                       flow_id=self.flow_id, node_id=fuzzy_settings.node_id)
             logger.info("Started the fuzzy match action")
-            node._fetch_cached_df = f
+            node._fetch_cached_df = f  # Add to the node so it can be cancelled and fetch later if needed
             return FlowDataEngine(f.get_result())
 
-        self.add_node_step(node_id=fuzzy_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='fuzzy_match',
-                           setting_input=fuzzy_settings)
-        node = self.get_node(node_id=fuzzy_settings.node_id)
-
         def schema_callback():
-            return calculate_fuzzy_match_schema(fuzzy_settings.join_input,
+            fm_input_copy = deepcopy(fuzzy_settings.join_input)  # Deepcopy create an unique object per func
+            node = self.get_node(node_id=fuzzy_settings.node_id)
+            return calculate_fuzzy_match_schema(fm_input_copy,
                                                 left_schema=node.node_inputs.main_inputs[0].schema,
                                                 right_schema=node.node_inputs.right_input.schema
                                                 )
 
-        node.schema_callback = schema_callback
+        self.add_node_step(node_id=fuzzy_settings.node_id,
+                           function=_func,
+                           input_columns=[],
+                           node_type='fuzzy_match',
+                           setting_input=fuzzy_settings,
+                           input_node_ids=fuzzy_settings.depending_on_ids,
+                           schema_callback=schema_callback)
+
         return self
 
     def add_text_to_rows(self, node_text_to_rows: input_schema.NodeTextToRows) -> "FlowGraph":
@@ -1549,6 +1595,8 @@ class FlowGraph:
         Args:
             execution_location: The execution location to set.
         """
+        if self.flow_settings.execution_location != execution_location:
+            self.reset()
         self.flow_settings.execution_location = execution_location
 
     def run_graph(self) -> RunInformation | None:
@@ -1575,18 +1623,11 @@ class FlowGraph:
             self.end_datetime = None
             self.latest_run_info = None
             self.flow_logger.info('Starting to run flowfile flow...')
-            skip_nodes = [node for node in self.nodes if not node.is_correct]
-            skip_nodes.extend([lead_to_node for node in skip_nodes for lead_to_node in node.leads_to_nodes])
-            execution_order = determine_execution_order(all_nodes=[node for node in self.nodes if
-                                                                   node not in skip_nodes],
-                                                        flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+            skip_nodes, execution_order = compute_execution_plan(nodes=self.nodes, flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+
             skip_node_message(self.flow_logger, skip_nodes)
             execution_order_message(self.flow_logger, execution_order)
             performance_mode = self.flow_settings.execution_mode == 'Performance'
-            if self.flow_settings.execution_location == 'local':
-                OFFLOAD_TO_WORKER.value = False
-            elif self.flow_settings.execution_location == 'remote':
-                OFFLOAD_TO_WORKER.value = True
             for node in execution_order:
                 node_logger = self.flow_logger.get_node_logger(node.node_id)
                 if self.flow_settings.is_canceled:
@@ -1922,4 +1963,4 @@ def delete_connection(graph, node_connection: input_schema.NodeConnection):
         to_node.delete_input_node(
             node_connection.output_connection.node_id,
             connection_type=node_connection.input_connection.connection_class,
-        )
+        )

flowfile_core/flowfile/flow_node/flow_node.py

@@ -5,7 +5,6 @@ from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEng
 from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.schemas import input_schema, schemas
 from flowfile_core.configs.flow_logger import NodeLogger
-from flowfile_core.configs.settings import SINGLE_FILE_MODE, OFFLOAD_TO_WORKER
 
 from flowfile_core.schemas.output_model import TableExample, FileColumn, NodeData
 from flowfile_core.flowfile.utils import get_hash
@@ -681,7 +680,7 @@ class FlowNode:
             logger.warning('Not implemented')
 
     def needs_run(self, performance_mode: bool, node_logger: NodeLogger = None,
-                  execution_location: schemas.ExecutionLocationsLiteral = "auto") -> bool:
+                  execution_location: schemas.ExecutionLocationsLiteral = "worker") -> bool:
         """Determines if the node needs to be executed.
 
         The decision is based on its run state, caching settings, and execution mode.
@@ -694,7 +693,7 @@ class FlowNode:
         Returns:
             True if the node should be run, False otherwise.
         """
-        if execution_location == "local" or SINGLE_FILE_MODE:
+        if execution_location == "local":
             return False
 
         flow_logger = logger if node_logger is None else node_logger
@@ -879,7 +878,7 @@ class FlowNode:
         if self.is_setup:
             node_logger.info(f'Starting to run {self.__name__}')
             if (self.needs_run(performance_mode, node_logger, run_location) or self.node_template.node_group == "output"
-                    and not (run_location == 'local' or SINGLE_FILE_MODE)):
+                    and not (run_location == 'local')):
                 self.prepare_before_run()
                 try:
                     if ((run_location == 'remote' or (self.node_default.transform_type == 'wide')
@@ -908,8 +907,14 @@ class FlowNode:
                                           node_logger=node_logger)
                     else:
                         self.results.errors = str(e)
-                        node_logger.error(f'Error with running the node: {e}')
-            elif ((run_location == 'local' or SINGLE_FILE_MODE) and
+                        if "Connection refused" in str(e) and "/submit_query/" in str(e):
+                            node_logger.warning("There was an issue connecting to the remote worker, "
+                                                "ensure the worker process is running, "
+                                                "or change the settings to, so it executes locally")
+                            node_logger.error("Could not execute in the remote worker. (Re)start the worker service, or change settings to local settings.")
+                        else:
+                            node_logger.error(f'Error with running the node: {e}')
+            elif ((run_location == 'local') and
                   (not self.node_stats.has_run_with_current_setup or self.node_template.node_group == "output")):
                 try:
                     node_logger.info('Executing fully locally')
@@ -959,16 +964,16 @@ class FlowNode:
             logger.info(f'{self.node_id}: Node needs reset')
             self.node_stats.has_run_with_current_setup = False
             self.results.reset()
-            if self.is_correct:
-                self._schema_callback = None  # Ensure the schema callback is reset
-                if self.schema_callback:
-                    logger.info(f'{self.node_id}: Resetting the schema callback')
-                    self.schema_callback.start()
             self.node_schema.result_schema = None
             self.node_schema.predicted_schema = None
             self._hash = None
             self.node_information.is_setup = None
             self.results.errors = None
+            if self.is_correct:
+                self._schema_callback = None  # Ensure the schema callback is reset
+                if self.schema_callback:
+                    logger.info(f'{self.node_id}: Resetting the schema callback')
+                    self.schema_callback.start()
             self.evaluate_nodes()
             _ = self.hash  # Recalculate the hash after reset
 

flowfile_core/flowfile/flow_node/models.py

@@ -108,14 +108,12 @@ class NodeStepSettings:
         streamable: If True, the node can process data in a streaming fashion.
         setup_errors: If True, indicates a non-blocking error occurred during setup.
         breaking_setup_errors: If True, indicates an error occurred that prevents execution.
-        execute_location: The preferred location for execution ('auto', 'local', 'remote').
     """
     cache_results: bool = False
     renew_schema: bool = True
     streamable: bool = True
     setup_errors: bool = False
     breaking_setup_errors: bool = False
-    execute_location: schemas.ExecutionLocationsLiteral = 'auto'
 
 
 class NodeStepInputs:

flowfile_core/flowfile/flow_node/schema_callback.py

@@ -1,71 +1,166 @@
-
 from typing import Callable, Any, Optional, Generic, TypeVar
 from concurrent.futures import ThreadPoolExecutor, Future
+import threading
 from flowfile_core.configs import logger
 
-
 T = TypeVar('T')
 
 
 class SingleExecutionFuture(Generic[T]):
-    """Single execution of a function in a separate thread with caching of the result."""
-    executor: ThreadPoolExecutor
-    future: Optional[Future[T]]
+    """Thread-safe single execution of a function with result caching.
+
+    Ensures a function is executed at most once even when called from multiple threads.
+    Subsequent calls return the cached result.
+    """
+
     func: Callable[[], T]
     on_error: Optional[Callable[[Exception], Any]]
-    result_value: Optional[T]
-    has_run_at_least_once: bool = False  # Indicates if the function has been run at least once
+    _lock: threading.RLock
+    _executor: Optional[ThreadPoolExecutor]
+    _future: Optional[Future[T]]
+    _result_value: Optional[T]
+    _exception: Optional[Exception]
+    _has_completed: bool
+    _has_started: bool
 
     def __init__(
-        self,
-        func: Callable[[], T],
-        on_error: Optional[Callable[[Exception], Any]] = None
+            self,
+            func: Callable[[], T],
+            on_error: Optional[Callable[[Exception], Any]] = None
     ) -> None:
         """Initialize with function and optional error handler."""
-        self.executor = ThreadPoolExecutor(max_workers=1)
-        self.future = None
         self.func = func
         self.on_error = on_error
-        self.result_value = None
-        self.has_run_at_least_once = False
+
+        # Thread safety
+        self._lock = threading.RLock()  # RLock allows re-entrant locking
+
+        # Execution state
+        self._executor = None
+        self._future = None
+        self._result_value = None
+        self._exception = None
+        self._has_completed = False
+        self._has_started = False
+
+    def _ensure_executor(self) -> ThreadPoolExecutor:
+        """Ensure executor exists, creating if necessary."""
+        if self._executor is None or self._executor._shutdown:
+            self._executor = ThreadPoolExecutor(max_workers=1)
+        return self._executor
 
     def start(self) -> None:
         """Start the function execution if not already started."""
-        if not self.future:
-            logger.info("single executor function started")
-            self.future = self.executor.submit(self.func)
+        with self._lock:
+            if self._has_started:
+                logger.info("Function already started or completed")
+                return
+
+            logger.info("Starting single executor function")
+            executor: ThreadPoolExecutor = self._ensure_executor()
+            self._future = executor.submit(self._func_wrapper)
+            self._has_started = True
+
+    def _func_wrapper(self) -> T:
+        """Wrapper to capture the result or exception."""
+        try:
+            result: T = self.func()
+            with self._lock:
+                self._result_value = result
+                self._has_completed = True
+            return result
+        except Exception as e:
+            with self._lock:
+                self._exception = e
+                self._has_completed = True
+            raise
 
     def cleanup(self) -> None:
-        """Clean up resources by clearing the future and shutting down the executor."""
-        self.has_run_at_least_once = True
-        self.executor.shutdown(wait=False)
+        """Clean up resources by shutting down the executor."""
+        with self._lock:
+            if self._executor and not self._executor._shutdown:
+                self._executor.shutdown(wait=False)
 
     def __call__(self) -> Optional[T]:
         """Execute function if not running and return its result."""
-        if self.result_value:
-            return self.result_value
-        if not self.future:
-            self.start()
-        else:
-            logger.info("Function already running or did complete")
-        try:
-            self.result_value = self.future.result()
-            logger.info("Done with the function")
-            return self.result_value
-        except Exception as e:
-            if self.on_error:
-                return self.on_error(e)
-            else:
-                raise e
-        finally:
-            self.cleanup()
+        with self._lock:
+            # If already completed, return cached result or raise cached exception
+            if self._has_completed:
+                if self._exception:
+                    if self.on_error:
+                        return self.on_error(self._exception)
+                    else:
+                        raise self._exception
+                return self._result_value
+
+            # Start if not already started
+            if not self._has_started:
+                self.start()
+
+        # Wait for completion outside the lock to avoid blocking other threads
+        if self._future:
+            try:
+                result: T = self._future.result()
+                logger.info("Function completed successfully")
+                return result
+            except Exception as e:
+                logger.error(f"Function raised exception: {e}")
+                if self.on_error:
+                    return self.on_error(e)
+                else:
+                    raise
+
+        return None
+
+    def reset(self) -> None:
+        """Reset the execution state, allowing the function to be run again."""
+        with self._lock:
+            logger.info("Resetting single execution future")
+
+            # Cancel any pending execution
+            if self._future and not self._future.done():
+                self._future.cancel()
 
-    def reset(self):
-        """Reset the future and result value."""
-        logger.info("Resetting the future and result value")
-        self.result_value = None
-        self.future = None
+            # Clean up old executor
+            if self._executor and not self._executor._shutdown:
+                self._executor.shutdown(wait=False)
+
+            # Reset state
+            self._executor = None
+            self._future = None
+            self._result_value = None
+            self._exception = None
+            self._has_completed = False
+            self._has_started = False
+
+    def is_running(self) -> bool:
+        """Check if the function is currently executing."""
+        with self._lock:
+            return bool(
+                self._has_started and
+                not self._has_completed and
+                self._future is not None and
+                not self._future.done()
+            )
+
+    def is_completed(self) -> bool:
+        """Check if the function has completed execution."""
+        with self._lock:
+            return self._has_completed
+
+    def get_result(self) -> Optional[T]:
+        """Get the cached result without triggering execution."""
+        with self._lock:
+            if self._exception:
+                if self.on_error:
+                    return self.on_error(self._exception)
+                else:
+                    raise self._exception
+            return self._result_value
 
     def __del__(self) -> None:
         """Ensure executor is shut down on deletion."""
-        self.cleanup()
+        try:
+            self.cleanup()
+        except Exception:
+            pass