Flowfile 0.5.1__py3-none-any.whl → 0.5.4__py3-none-any.whl

flowfile_core/flowfile/flow_graph.py

@@ -1,80 +1,103 @@
 import datetime
-
-import os
-import yaml
 import json
-
-import polars as pl
+import os
+from collections.abc import Callable
+from copy import deepcopy
+from functools import partial
+from importlib.metadata import PackageNotFoundError, version
 from pathlib import Path
+from time import time
+from typing import Any, Literal, Union
+from uuid import uuid1
 
 import fastexcel
+import polars as pl
+import yaml
 from fastapi.exceptions import HTTPException
-from time import time
-from functools import partial
-from typing import List, Dict, Union, Callable, Any, Optional, Tuple, Literal
-from uuid import uuid1
-from copy import deepcopy
 from pyarrow.parquet import ParquetFile
+
 from flowfile_core.configs import logger
 from flowfile_core.configs.flow_logger import FlowLogger
-from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
-from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, cast_str_to_polars_type
-
+from flowfile_core.configs.node_store import CUSTOM_NODE_STORE
+from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
+from flowfile_core.flowfile.database_connection_manager.db_connections import (
+    get_local_cloud_connection,
+    get_local_database_connection,
+)
 from flowfile_core.flowfile.flow_data_engine.cloud_storage_reader import CloudStorageReader
-from flowfile_core.schemas.transform_schema import FuzzyMatchInputManager
-from flowfile_core.utils.arrow_reader import get_read_top_n
 from flowfile_core.flowfile.flow_data_engine.flow_data_engine import FlowDataEngine, execute_polars_code
-from flowfile_core.flowfile.flow_data_engine.read_excel_tables import (get_open_xlsx_datatypes,
-                                                                       get_calamine_xlsx_data_types)
-
-from flowfile_core.flowfile.schema_callbacks import (calculate_fuzzy_match_schema, pre_calculate_pivot_schema)
+from flowfile_core.flowfile.flow_data_engine.flow_file_column.main import FlowfileColumn, cast_str_to_polars_type
+from flowfile_core.flowfile.flow_data_engine.polars_code_parser import polars_code_parser
+from flowfile_core.flowfile.flow_data_engine.read_excel_tables import (
+    get_calamine_xlsx_data_types,
+    get_open_xlsx_datatypes,
+)
+from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import (
+    ExternalCloudWriter,
+    ExternalDatabaseFetcher,
+    ExternalDatabaseWriter,
+    ExternalDfFetcher,
+)
+from flowfile_core.flowfile.flow_node.flow_node import FlowNode
+from flowfile_core.flowfile.graph_tree.graph_tree import (
+    add_un_drawn_nodes,
+    build_flow_paths,
+    build_node_info,
+    calculate_depth,
+    define_node_connections,
+    draw_merged_paths,
+    draw_standalone_paths,
+    group_nodes_by_depth,
+)
+from flowfile_core.flowfile.node_designer.custom_node import CustomNodeBase
+from flowfile_core.flowfile.schema_callbacks import calculate_fuzzy_match_schema, pre_calculate_pivot_schema
 from flowfile_core.flowfile.sources import external_sources
+from flowfile_core.flowfile.sources.external_sources.factory import data_source_factory
+from flowfile_core.flowfile.sources.external_sources.sql_source import models as sql_models
+from flowfile_core.flowfile.sources.external_sources.sql_source import utils as sql_utils
+from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import BaseSqlSource, SqlSource
+from flowfile_core.flowfile.util.calculate_layout import calculate_layered_layout
+from flowfile_core.flowfile.util.execution_orderer import compute_execution_plan
+from flowfile_core.flowfile.utils import snake_case_to_camel_case
 from flowfile_core.schemas import input_schema, schemas, transform_schema
+from flowfile_core.schemas.cloud_storage_schemas import (
+    AuthMethod,
+    CloudStorageReadSettingsInternal,
+    CloudStorageWriteSettingsInternal,
+    FullCloudStorageConnection,
+    get_cloud_storage_write_settings_worker_interface,
+)
 from flowfile_core.schemas.output_model import NodeData, NodeResult, RunInformation
-from flowfile_core.schemas.cloud_storage_schemas import (CloudStorageReadSettingsInternal,
-                                                         CloudStorageWriteSettingsInternal,
-                                                         FullCloudStorageConnection,
-                                                         get_cloud_storage_write_settings_worker_interface, AuthMethod)
-from flowfile_core.flowfile.utils import snake_case_to_camel_case
-from flowfile_core.flowfile.analytics.utils import create_graphic_walker_node_from_node_promise
-from flowfile_core.flowfile.flow_node.flow_node import FlowNode
-from flowfile_core.flowfile.util.execution_orderer import compute_execution_plan
-from flowfile_core.flowfile.graph_tree.graph_tree import (add_un_drawn_nodes, build_flow_paths,
-                                                          build_node_info, calculate_depth,
-                                                          define_node_connections, draw_merged_paths,
-                                                          draw_standalone_paths, group_nodes_by_depth)
-from flowfile_core.flowfile.flow_data_engine.polars_code_parser import polars_code_parser
-from flowfile_core.flowfile.flow_data_engine.subprocess_operations.subprocess_operations import (ExternalDatabaseFetcher,
-                                                                                                 ExternalDatabaseWriter,
-                                                                                                 ExternalDfFetcher,
-                                                                                                 ExternalCloudWriter)
-from flowfile_core.secret_manager.secret_manager import get_encrypted_secret, decrypt_secret
-from flowfile_core.flowfile.sources.external_sources.sql_source import utils as sql_utils, models as sql_models
-from flowfile_core.flowfile.sources.external_sources.sql_source.sql_source import SqlSource, BaseSqlSource
-from flowfile_core.flowfile.database_connection_manager.db_connections import (get_local_database_connection,
-                                                                               get_local_cloud_connection)
-from flowfile_core.flowfile.util.calculate_layout import calculate_layered_layout
-from flowfile_core.flowfile.node_designer.custom_node import CustomNodeBase
-from importlib.metadata import version, PackageNotFoundError
+from flowfile_core.schemas.transform_schema import FuzzyMatchInputManager
+from flowfile_core.secret_manager.secret_manager import decrypt_secret, get_encrypted_secret
+from flowfile_core.utils.arrow_reader import get_read_top_n
 
 try:
     __version__ = version("Flowfile")
 except PackageNotFoundError:
-    __version__ = "0.0.0-dev"
+    __version__ = "0.5.0"
 
 
 def represent_list_json(dumper, data):
     """Use inline style for short simple lists, block style for complex ones."""
     if len(data) <= 10 and all(isinstance(item, (int, str, float, bool, type(None))) for item in data):
-        return dumper.represent_sequence('tag:yaml.org,2002:seq', data, flow_style=True)
-    return dumper.represent_sequence('tag:yaml.org,2002:seq', data, flow_style=False)
+        return dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=True)
+    return dumper.represent_sequence("tag:yaml.org,2002:seq", data, flow_style=False)
 
 
 yaml.add_representer(list, represent_list_json)
 
 
-def get_xlsx_schema(engine: str, file_path: str, sheet_name: str, start_row: int, start_column: int,
-                    end_row: int, end_column: int, has_headers: bool):
+def get_xlsx_schema(
+    engine: str,
+    file_path: str,
+    sheet_name: str,
+    start_row: int,
+    start_column: int,
+    end_row: int,
+    end_column: int,
+    has_headers: bool,
+):
     """Calculates the schema of an XLSX file by reading a sample of rows.
 
     Args:
@@ -91,27 +114,29 @@ def get_xlsx_schema(engine: str, file_path: str, sheet_name: str, start_row: int
         A list of FlowfileColumn objects representing the schema.
     """
     try:
-        logger.info('Starting to calculate the schema')
-        if engine == 'openpyxl':
+        logger.info("Starting to calculate the schema")
+        if engine == "openpyxl":
             max_col = end_column if end_column > 0 else None
-            return get_open_xlsx_datatypes(file_path=file_path,
-                                           sheet_name=sheet_name,
-                                           min_row=start_row + 1,
-                                           min_col=start_column + 1,
-                                           max_row=100,
-                                           max_col=max_col, has_headers=has_headers)
-        elif engine == 'calamine':
-            return get_calamine_xlsx_data_types(file_path=file_path,
-                                                sheet_name=sheet_name,
-                                                start_row=start_row,
-                                                end_row=end_row)
-        logger.info('done calculating the schema')
+            return get_open_xlsx_datatypes(
+                file_path=file_path,
+                sheet_name=sheet_name,
+                min_row=start_row + 1,
+                min_col=start_column + 1,
+                max_row=100,
+                max_col=max_col,
+                has_headers=has_headers,
+            )
+        elif engine == "calamine":
+            return get_calamine_xlsx_data_types(
+                file_path=file_path, sheet_name=sheet_name, start_row=start_row, end_row=end_row
+            )
+        logger.info("done calculating the schema")
     except Exception as e:
         logger.error(e)
         return []
 
 
-def skip_node_message(flow_logger: FlowLogger, nodes: List[FlowNode]) -> None:
+def skip_node_message(flow_logger: FlowLogger, nodes: list[FlowNode]) -> None:
     """Logs a warning message listing all nodes that will be skipped during execution.
 
     Args:
@@ -120,10 +145,10 @@ def skip_node_message(flow_logger: FlowLogger, nodes: List[FlowNode]) -> None:
     """
     if len(nodes) > 0:
         msg = "\n".join(str(node) for node in nodes)
-        flow_logger.warning(f'skipping nodes:\n{msg}')
+        flow_logger.warning(f"skipping nodes:\n{msg}")
 
 
-def execution_order_message(flow_logger: FlowLogger, nodes: List[FlowNode]) -> None:
+def execution_order_message(flow_logger: FlowLogger, nodes: list[FlowNode]) -> None:
     """Logs an informational message showing the determined execution order of nodes.
 
     Args:
@@ -131,11 +156,19 @@ def execution_order_message(flow_logger: FlowLogger, nodes: List[FlowNode]) -> N
         nodes: A list of FlowNode objects in the order they will be executed.
     """
     msg = "\n".join(str(node) for node in nodes)
-    flow_logger.info(f'execution order:\n{msg}')
-
-
-def get_xlsx_schema_callback(engine: str, file_path: str, sheet_name: str, start_row: int, start_column: int,
-                             end_row: int, end_column: int, has_headers: bool):
+    flow_logger.info(f"execution order:\n{msg}")
+
+
+def get_xlsx_schema_callback(
+    engine: str,
+    file_path: str,
+    sheet_name: str,
+    start_row: int,
+    start_column: int,
+    end_row: int,
+    end_column: int,
+    has_headers: bool,
+):
     """Creates a partially applied function for lazy calculation of an XLSX schema.
 
     Args:
@@ -151,12 +184,22 @@ def get_xlsx_schema_callback(engine: str, file_path: str, sheet_name: str, start
     Returns:
         A callable function that, when called, will execute `get_xlsx_schema`.
     """
-    return partial(get_xlsx_schema, engine=engine, file_path=file_path, sheet_name=sheet_name, start_row=start_row,
-                   start_column=start_column, end_row=end_row, end_column=end_column, has_headers=has_headers)
+    return partial(
+        get_xlsx_schema,
+        engine=engine,
+        file_path=file_path,
+        sheet_name=sheet_name,
+        start_row=start_row,
+        start_column=start_column,
+        end_row=end_row,
+        end_column=end_column,
+        has_headers=has_headers,
+    )
 
 
-def get_cloud_connection_settings(connection_name: str,
-                                  user_id: int, auth_mode: AuthMethod) -> FullCloudStorageConnection:
+def get_cloud_connection_settings(
+    connection_name: str, user_id: int, auth_mode: AuthMethod
+) -> FullCloudStorageConnection:
     """Retrieves cloud storage connection settings, falling back to environment variables if needed.
 
     Args:
@@ -186,32 +229,44 @@ class FlowGraph:
 
     It manages nodes, connections, and the execution of the entire flow.
     """
+
     uuid: str
-    depends_on: Dict[int, Union[ParquetFile, FlowDataEngine, "FlowGraph", pl.DataFrame,]]
+    depends_on: dict[
+        int,
+        Union[
+            ParquetFile,
+            FlowDataEngine,
+            "FlowGraph",
+            pl.DataFrame,
+        ],
+    ]
     _flow_id: int
     _input_data: Union[ParquetFile, FlowDataEngine, "FlowGraph"]
-    _input_cols: List[str]
-    _output_cols: List[str]
-    _node_db: Dict[Union[str, int], FlowNode]
-    _node_ids: List[Union[str, int]]
-    _results: Optional[FlowDataEngine] = None
+    _input_cols: list[str]
+    _output_cols: list[str]
+    _node_db: dict[str | int, FlowNode]
+    _node_ids: list[str | int]
+    _results: FlowDataEngine | None = None
     cache_results: bool = False
-    schema: Optional[List[FlowfileColumn]] = None
+    schema: list[FlowfileColumn] | None = None
     has_over_row_function: bool = False
-    _flow_starts: List[Union[int, str]] = None
-    latest_run_info: Optional[RunInformation] = None
+    _flow_starts: list[int | str] = None
+    latest_run_info: RunInformation | None = None
     start_datetime: datetime = None
     end_datetime: datetime = None
     _flow_settings: schemas.FlowSettings = None
     flow_logger: FlowLogger
 
-    def __init__(self,
-                 flow_settings: schemas.FlowSettings | schemas.FlowGraphConfig,
-                 name: str = None, input_cols: List[str] = None,
-                 output_cols: List[str] = None,
-                 path_ref: str = None,
-                 input_flow: Union[ParquetFile, FlowDataEngine, "FlowGraph"] = None,
-                 cache_results: bool = False):
+    def __init__(
+        self,
+        flow_settings: schemas.FlowSettings | schemas.FlowGraphConfig,
+        name: str = None,
+        input_cols: list[str] = None,
+        output_cols: list[str] = None,
+        path_ref: str = None,
+        input_flow: Union[ParquetFile, FlowDataEngine, "FlowGraph"] = None,
+        cache_results: bool = False,
+    ):
         """Initializes a new FlowGraph instance.
 
         Args:
@@ -233,7 +288,7 @@ class FlowGraph:
         self.latest_run_info = None
         self._flow_id = flow_settings.flow_id
         self.flow_logger = FlowLogger(flow_settings.flow_id)
-        self._flow_starts: List[FlowNode] = []
+        self._flow_starts: list[FlowNode] = []
         self._results = None
         self.schema = None
         self.has_over_row_function = False
@@ -255,13 +310,21 @@ class FlowGraph:
 
     @flow_settings.setter
     def flow_settings(self, flow_settings: schemas.FlowSettings):
-        if (
-                (self._flow_settings.execution_location != flow_settings.execution_location) or
-                (self._flow_settings.execution_mode != flow_settings.execution_mode)
+        if (self._flow_settings.execution_location != flow_settings.execution_location) or (
+            self._flow_settings.execution_mode != flow_settings.execution_mode
         ):
             self.reset()
         self._flow_settings = flow_settings
 
+    def add_node_to_starting_list(self, node: FlowNode) -> None:
+        """Adds a node to the list of starting nodes for the flow if not already present.
+
+        Args:
+            node: The FlowNode to add as a starting node.
+        """
+        if node.node_id not in {self_node.node_id for self_node in self._flow_starts}:
+            self._flow_starts.append(node)
+
     def add_node_promise(self, node_promise: input_schema.NodePromise):
         """Adds a placeholder node to the graph that is not yet fully configured.
 
@@ -270,13 +333,31 @@ class FlowGraph:
         Args:
             node_promise: A promise object containing basic node information.
         """
+
         def placeholder(n: FlowNode = None):
             if n is None:
                 return FlowDataEngine()
             return n
 
-        self.add_node_step(node_id=node_promise.node_id, node_type=node_promise.node_type, function=placeholder,
-                           setting_input=node_promise)
+        self.add_node_step(
+            node_id=node_promise.node_id,
+            node_type=node_promise.node_type,
+            function=placeholder,
+            setting_input=node_promise,
+        )
+        if node_promise.is_user_defined:
+            node_needs_settings: bool
+            custom_node = CUSTOM_NODE_STORE.get(node_promise.node_type)
+            if custom_node is None:
+                raise Exception(f"Custom node type '{node_promise.node_type}' not found in registry.")
+            settings_schema = custom_node.model_fields["settings_schema"].default
+            node_needs_settings = settings_schema is not None and not settings_schema.is_empty()
+            if not node_needs_settings:
+                user_defined_node_settings = input_schema.UserDefinedNode(settings={}, **node_promise.model_dump())
+                initialized_model = custom_node()
+                self.add_user_defined_node(
+                    custom_node=initialized_model, user_defined_node_settings=user_defined_node_settings
+                )
 
     def apply_layout(self, y_spacing: int = 150, x_spacing: int = 200, initial_y: int = 100):
         """Calculates and applies a layered layout to all nodes in the graph.
@@ -304,20 +385,24 @@ class FlowGraph:
             updated_count = 0
             for node_id, (pos_x, pos_y) in new_positions.items():
                 node = self.get_node(node_id)
-                if node and hasattr(node, 'setting_input'):
+                if node and hasattr(node, "setting_input"):
                     setting = node.setting_input
-                    if hasattr(setting, 'pos_x') and hasattr(setting, 'pos_y'):
+                    if hasattr(setting, "pos_x") and hasattr(setting, "pos_y"):
                         setting.pos_x = pos_x
                         setting.pos_y = pos_y
                         updated_count += 1
                     else:
-                        self.flow_logger.warning(f"Node {node_id} setting_input ({type(setting)}) lacks pos_x/pos_y attributes.")
+                        self.flow_logger.warning(
+                            f"Node {node_id} setting_input ({type(setting)}) lacks pos_x/pos_y attributes."
+                        )
                 elif node:
                     self.flow_logger.warning(f"Node {node_id} lacks setting_input attribute.")
                 # else: Node not found, already warned by calculate_layered_layout
 
             end_time = time()
-            self.flow_logger.info(f"Layout applied to {updated_count}/{len(self.nodes)} nodes in {end_time - start_time:.2f} seconds.")
+            self.flow_logger.info(
+                f"Layout applied to {updated_count}/{len(self.nodes)} nodes in {end_time - start_time:.2f} seconds."
+            )
 
         except Exception as e:
             self.flow_logger.error(f"Error applying layout: {e}")
@@ -337,13 +422,13 @@ class FlowGraph:
         """
         self._flow_id = new_id
         for node in self.nodes:
-            if hasattr(node.setting_input, 'flow_id'):
+            if hasattr(node.setting_input, "flow_id"):
                 node.setting_input.flow_id = new_id
         self.flow_settings.flow_id = new_id
 
     def __repr__(self):
         """Provides the official string representation of the FlowGraph instance."""
-        settings_str = "  -" + '\n  -'.join(f"{k}: {v}" for k, v in self.flow_settings)
+        settings_str = "  -" + "\n  -".join(f"{k}: {v}" for k, v in self.flow_settings)
         return f"FlowGraph(\nNodes: {self._node_db}\n\nSettings:\n{settings_str}"
 
     def print_tree(self):
@@ -361,7 +446,7 @@ class FlowGraph:
 
         # Group nodes by depth
         depth_groups, max_depth = group_nodes_by_depth(node_info)
-        
+
         # Sort nodes within each depth group
         for depth in depth_groups:
             depth_groups[depth].sort()
@@ -371,7 +456,7 @@ class FlowGraph:
 
         # Track which nodes connect to what
         merge_points = define_node_connections(node_info)
-        
+
         # Build the flow paths
 
         # Find the maximum label length for each depth level
@@ -380,15 +465,15 @@ class FlowGraph:
             if depth in depth_groups:
                 max_len = max(len(node_info[nid].label) for nid in depth_groups[depth])
                 max_label_length[depth] = max_len
-        
+
         # Draw the paths
         drawn_nodes = set()
         merge_drawn = set()
-        
+
         # Group paths by their merge points
         paths_by_merge = {}
         standalone_paths = []
-        
+
         # Build flow paths
         paths = build_flow_paths(node_info, self._flow_starts, merge_points)
 
@@ -410,22 +495,22 @@ class FlowGraph:
 
         # Add undrawn nodes
         add_un_drawn_nodes(drawn_nodes, node_info, lines)
-        
+
         try:
             skip_nodes, ordered_nodes = compute_execution_plan(
-                nodes=self.nodes,
-                flow_starts=self._flow_starts+self.get_implicit_starter_nodes())
+                nodes=self.nodes, flow_starts=self._flow_starts + self.get_implicit_starter_nodes()
+            )
             if ordered_nodes:
                 for i, node in enumerate(ordered_nodes, 1):
                     lines.append(f"  {i:3d}. {node_info[node.node_id].label}")
         except Exception as e:
             lines.append(f"  Could not determine execution order: {e}")
-        
+
         # Print everything
         output = "\n".join(lines)
-        
+
         print(output)
-        
+
     def get_nodes_overview(self):
         """Gets a list of dictionary representations for all nodes in the graph."""
         output = []
@@ -433,7 +518,7 @@ class FlowGraph:
             output.append(v.get_repr())
         return output
 
-    def remove_from_output_cols(self, columns: List[str]):
+    def remove_from_output_cols(self, columns: list[str]):
         """Removes specified columns from the list of expected output columns.
 
         Args:
@@ -442,7 +527,7 @@ class FlowGraph:
         cols = set(columns)
         self._output_cols = [c for c in self._output_cols if c not in cols]
 
-    def get_node(self, node_id: Union[int, str] = None) -> FlowNode | None:
+    def get_node(self, node_id: int | str = None) -> FlowNode | None:
         """Retrieves a node from the graph by its ID.
 
         Args:
@@ -456,24 +541,43 @@ class FlowGraph:
         node = self._node_db.get(node_id)
         if node is not None:
             return node
-        
-    def add_user_defined_node(self, *,
-                              custom_node: CustomNodeBase,
-                              user_defined_node_settings: input_schema.UserDefinedNode
-                              ):
-       
-        def _func(*fdes: FlowDataEngine) -> FlowDataEngine | None:
-            output = custom_node.process(*(fde.data_frame for fde in fdes))
-            if isinstance(output, pl.LazyFrame | pl.DataFrame):
+
+    def add_user_defined_node(
+        self, *, custom_node: CustomNodeBase, user_defined_node_settings: input_schema.UserDefinedNode
+    ):
+        """Adds a user-defined custom node to the graph.
+
+        Args:
+            custom_node: The custom node instance to add.
+            user_defined_node_settings: The settings for the user-defined node.
+        """
+
+        def _func(*flow_data_engine: FlowDataEngine) -> FlowDataEngine | None:
+            user_id = user_defined_node_settings.user_id
+            if user_id is not None:
+                custom_node.set_execution_context(user_id)
+                if custom_node.settings_schema:
+                    custom_node.settings_schema.set_secret_context(user_id, custom_node.accessed_secrets)
+
+            output = custom_node.process(*(fde.data_frame for fde in flow_data_engine))
+
+            accessed_secrets = custom_node.get_accessed_secrets()
+            if accessed_secrets:
+                logger.info(f"Node '{user_defined_node_settings.node_id}' accessed secrets: {accessed_secrets}")
+            if isinstance(output, (pl.LazyFrame, pl.DataFrame)):
                 return FlowDataEngine(output)
             return None
-        
-        self.add_node_step(node_id=user_defined_node_settings.node_id,
-                           function=_func,
-                           setting_input=user_defined_node_settings,
-                           input_node_ids=user_defined_node_settings.depending_on_ids,
-                           node_type=custom_node.item,
-                           )
+
+        self.add_node_step(
+            node_id=user_defined_node_settings.node_id,
+            function=_func,
+            setting_input=user_defined_node_settings,
+            input_node_ids=user_defined_node_settings.depending_on_ids,
+            node_type=custom_node.item,
+        )
+        if custom_node.number_of_inputs == 0:
+            node = self.get_node(user_defined_node_settings.node_id)
+            self.add_node_to_starting_list(node)
 
     def add_pivot(self, pivot_settings: input_schema.NodePivot):
         """Adds a pivot node to the graph.
@@ -485,11 +589,13 @@ class FlowGraph:
         def _func(fl: FlowDataEngine):
             return fl.do_pivot(pivot_settings.pivot_input, self.flow_logger.get_node_logger(pivot_settings.node_id))
 
-        self.add_node_step(node_id=pivot_settings.node_id,
-                           function=_func,
-                           node_type='pivot',
-                           setting_input=pivot_settings,
-                           input_node_ids=[pivot_settings.depending_on_id])
+        self.add_node_step(
+            node_id=pivot_settings.node_id,
+            function=_func,
+            node_type="pivot",
+            setting_input=pivot_settings,
+            input_node_ids=[pivot_settings.depending_on_id],
+        )
 
         node = self.get_node(pivot_settings.node_id)
 
@@ -498,6 +604,7 @@ class FlowGraph:
             input_data.lazy = True  # ensure the dataset is lazy
             input_lf = input_data.data_frame  # get the lazy frame
             return pre_calculate_pivot_schema(input_data.schema, pivot_settings.pivot_input, input_lf=input_lf)
+
         node.schema_callback = schema_callback
 
     def add_unpivot(self, unpivot_settings: input_schema.NodeUnpivot):
@@ -510,11 +617,13 @@ class FlowGraph:
         def _func(fl: FlowDataEngine) -> FlowDataEngine:
             return fl.unpivot(unpivot_settings.unpivot_input)
 
-        self.add_node_step(node_id=unpivot_settings.node_id,
-                           function=_func,
-                           node_type='unpivot',
-                           setting_input=unpivot_settings,
-                           input_node_ids=[unpivot_settings.depending_on_id])
+        self.add_node_step(
+            node_id=unpivot_settings.node_id,
+            function=_func,
+            node_type="unpivot",
+            setting_input=unpivot_settings,
+            input_node_ids=[unpivot_settings.depending_on_id],
+        )
 
     def add_union(self, union_settings: input_schema.NodeUnion):
         """Adds a union node to combine multiple data streams.
@@ -524,14 +633,16 @@ class FlowGraph:
         """
 
         def _func(*flowfile_tables: FlowDataEngine):
-            dfs: List[pl.LazyFrame] | List[pl.DataFrame] = [flt.data_frame for flt in flowfile_tables]
-            return FlowDataEngine(pl.concat(dfs, how='diagonal_relaxed'))
+            dfs: list[pl.LazyFrame] | list[pl.DataFrame] = [flt.data_frame for flt in flowfile_tables]
+            return FlowDataEngine(pl.concat(dfs, how="diagonal_relaxed"))
 
-        self.add_node_step(node_id=union_settings.node_id,
-                           function=_func,
-                           node_type=f'union',
-                           setting_input=union_settings,
-                           input_node_ids=union_settings.depending_on_ids)
+        self.add_node_step(
+            node_id=union_settings.node_id,
+            function=_func,
+            node_type="union",
+            setting_input=union_settings,
+            input_node_ids=union_settings.depending_on_ids,
+        )
 
     def add_initial_node_analysis(self, node_promise: input_schema.NodePromise):
         """Adds a data exploration/analysis node based on a node promise.
@@ -559,13 +670,14 @@ class FlowGraph:
                 flowfile_table = flowfile_table.get_sample(sample_size, random=True)
             external_sampler = ExternalDfFetcher(
                 lf=flowfile_table.data_frame,
-                file_ref="__gf_walker"+node.hash,
+                file_ref="__gf_walker" + node.hash,
                 wait_on_completion=True,
                 node_id=node.node_id,
                 flow_id=self.flow_id,
             )
-            node.results.analysis_data_generator = get_read_top_n(external_sampler.status.file_ref,
-                                                                  n=min(sample_size, number_of_records))
+            node.results.analysis_data_generator = get_read_top_n(
+                external_sampler.status.file_ref, n=min(sample_size, number_of_records)
+            )
             return flowfile_table
 
         def schema_callback():
@@ -574,11 +686,15 @@ class FlowGraph:
                 input_node = node.all_inputs[0]
                 return input_node.schema
             else:
-                return [FlowfileColumn.from_input('col_1', 'na')]
+                return [FlowfileColumn.from_input("col_1", "na")]
 
-        self.add_node_step(node_id=node_analysis.node_id, node_type='explore_data',
-                           function=analysis_preparation,
-                           setting_input=node_analysis, schema_callback=schema_callback)
+        self.add_node_step(
+            node_id=node_analysis.node_id,
+            node_type="explore_data",
+            function=analysis_preparation,
+            setting_input=node_analysis,
+            schema_callback=schema_callback,
+        )
         node = self.get_node(node_analysis.node_id)
 
     def add_group_by(self, group_by_settings: input_schema.NodeGroupBy):
@@ -591,19 +707,20 @@ class FlowGraph:
         def _func(fl: FlowDataEngine) -> FlowDataEngine:
             return fl.do_group_by(group_by_settings.groupby_input, False)
 
-        self.add_node_step(node_id=group_by_settings.node_id,
-                           function=_func,
-                           node_type=f'group_by',
-                           setting_input=group_by_settings,
-                           input_node_ids=[group_by_settings.depending_on_id])
+        self.add_node_step(
+            node_id=group_by_settings.node_id,
+            function=_func,
+            node_type="group_by",
+            setting_input=group_by_settings,
+            input_node_ids=[group_by_settings.depending_on_id],
+        )
 
         node = self.get_node(group_by_settings.node_id)
 
         def schema_callback():
-
             output_columns = [(c.old_name, c.new_name, c.output_type) for c in group_by_settings.groupby_input.agg_cols]
             depends_on = node.node_inputs.main_inputs[0]
-            input_schema_dict: Dict[str, str] = {s.name: s.data_type for s in depends_on.schema}
+            input_schema_dict: dict[str, str] = {s.name: s.data_type for s in depends_on.schema}
             output_schema = []
             for old_name, new_name, data_type in output_columns:
                 data_type = input_schema_dict[old_name] if data_type is None else data_type
@@ -618,38 +735,148 @@ class FlowGraph:
         Args:
             filter_settings: The settings for the filter operation.
         """
+        from flowfile_core.schemas.transform_schema import FilterOperator
+
+        def _build_basic_filter_expression(
+            basic_filter: transform_schema.BasicFilter, field_data_type: str | None = None
+        ) -> str:
+            """Build a filter expression string from a BasicFilter object.
+
+            Uses the Flowfile expression language that is compatible with polars_expr_transformer.
+
+            Args:
+                basic_filter: The basic filter configuration.
+                field_data_type: The data type of the field (optional, for smart quoting).
+
+            Returns:
+                A filter expression string compatible with polars_expr_transformer.
+            """
+            field = f"[{basic_filter.field}]"
+            value = basic_filter.value
+            value2 = basic_filter.value2
+
+            is_numeric_value = value.replace(".", "", 1).replace("-", "", 1).isnumeric() if value else False
+            should_quote = field_data_type == "str" or not is_numeric_value
+
+            try:
+                operator = basic_filter.get_operator()
+            except (ValueError, AttributeError):
+                operator = FilterOperator.from_symbol(str(basic_filter.operator))
+
+            if operator == FilterOperator.EQUALS:
+                if should_quote:
+                    return f'{field}="{value}"'
+                return f"{field}={value}"
+
+            elif operator == FilterOperator.NOT_EQUALS:
+                if should_quote:
+                    return f'{field}!="{value}"'
+                return f"{field}!={value}"
+
+            elif operator == FilterOperator.GREATER_THAN:
+                if should_quote:
+                    return f'{field}>"{value}"'
+                return f"{field}>{value}"
+
+            elif operator == FilterOperator.GREATER_THAN_OR_EQUALS:
+                if should_quote:
+                    return f'{field}>="{value}"'
+                return f"{field}>={value}"
+
+            elif operator == FilterOperator.LESS_THAN:
+                if should_quote:
+                    return f'{field}<"{value}"'
+                return f"{field}<{value}"
+
+            elif operator == FilterOperator.LESS_THAN_OR_EQUALS:
+                if should_quote:
+                    return f'{field}<="{value}"'
+                return f"{field}<={value}"
+
+            elif operator == FilterOperator.CONTAINS:
+                return f'contains({field}, "{value}")'
+
+            elif operator == FilterOperator.NOT_CONTAINS:
+                return f'contains({field}, "{value}") = false'
+
+            elif operator == FilterOperator.STARTS_WITH:
+                return f'left({field}, {len(value)}) = "{value}"'
+
+            elif operator == FilterOperator.ENDS_WITH:
+                return f'right({field}, {len(value)}) = "{value}"'
+
+            elif operator == FilterOperator.IS_NULL:
+                return f"is_empty({field})"
+
+            elif operator == FilterOperator.IS_NOT_NULL:
+                return f"is_not_empty({field})"
+
+            elif operator == FilterOperator.IN:
+                values = [v.strip() for v in value.split(",")]
+                if len(values) == 1:
+                    if should_quote:
+                        return f'{field}="{values[0]}"'
+                    return f"{field}={values[0]}"
+                if should_quote:
+                    conditions = [f'({field}="{v}")' for v in values]
+                else:
+                    conditions = [f"({field}={v})" for v in values]
+                return " | ".join(conditions)
+
+            elif operator == FilterOperator.NOT_IN:
+                values = [v.strip() for v in value.split(",")]
+                if len(values) == 1:
+                    if should_quote:
+                        return f'{field}!="{values[0]}"'
+                    return f"{field}!={values[0]}"
+                if should_quote:
+                    conditions = [f'({field}!="{v}")' for v in values]
+                else:
+                    conditions = [f"({field}!={v})" for v in values]
+                return " & ".join(conditions)
 
-        is_advanced = filter_settings.filter_input.filter_type == 'advanced'
-        if is_advanced:
-            predicate = filter_settings.filter_input.advanced_filter
-        else:
-            _basic_filter = filter_settings.filter_input.basic_filter
-            filter_settings.filter_input.advanced_filter = (f'[{_basic_filter.field}]{_basic_filter.filter_type}"'
-                                                            f'{_basic_filter.filter_value}"')
+            elif operator == FilterOperator.BETWEEN:
+                if value2 is None:
+                    raise ValueError("BETWEEN operator requires value2")
+                if should_quote:
+                    return f'({field}>="{value}") & ({field}<="{value2}")'
+                return f"({field}>={value}) & ({field}<={value2})"
+
+            else:
+                # Fallback for unknown operators - use legacy format
+                if should_quote:
+                    return f'{field}{operator.to_symbol()}"{value}"'
+                return f"{field}{operator.to_symbol()}{value}"
 
         def _func(fl: FlowDataEngine):
-            is_advanced = filter_settings.filter_input.filter_type == 'advanced'
+            is_advanced = filter_settings.filter_input.is_advanced()
+
             if is_advanced:
+                predicate = filter_settings.filter_input.advanced_filter
                 return fl.do_filter(predicate)
             else:
                 basic_filter = filter_settings.filter_input.basic_filter
-                if basic_filter.filter_value.isnumeric():
+                if basic_filter is None:
+                    logger.warning("Basic filter is None, returning unfiltered data")
+                    return fl
+
+                try:
                     field_data_type = fl.get_schema_column(basic_filter.field).generic_datatype()
-                    if field_data_type == 'str':
-                        _f = f'[{basic_filter.field}]{basic_filter.filter_type}"{basic_filter.filter_value}"'
-                    else:
-                        _f = f'[{basic_filter.field}]{basic_filter.filter_type}{basic_filter.filter_value}'
-                else:
-                    _f = f'[{basic_filter.field}]{basic_filter.filter_type}"{basic_filter.filter_value}"'
-                filter_settings.filter_input.advanced_filter = _f
-                return fl.do_filter(_f)
+                except Exception:
+                    field_data_type = None
 
-        self.add_node_step(filter_settings.node_id, _func,
-                           node_type='filter',
-                           renew_schema=False,
-                           setting_input=filter_settings,
-                           input_node_ids=[filter_settings.depending_on_id]
-                           )
+                expression = _build_basic_filter_expression(basic_filter, field_data_type)
+                filter_settings.filter_input.advanced_filter = expression
+                return fl.do_filter(expression)
+
+        self.add_node_step(
+            filter_settings.node_id,
+            _func,
+            node_type="filter",
+            renew_schema=False,
+            setting_input=filter_settings,
+            input_node_ids=[filter_settings.depending_on_id],
+        )
 
     def add_record_count(self, node_number_of_records: input_schema.NodeRecordCount):
         """Adds a filter node to the graph.
@@ -661,11 +888,13 @@ class FlowGraph:
         def _func(fl: FlowDataEngine) -> FlowDataEngine:
             return fl.get_record_count()
 
-        self.add_node_step(node_id=node_number_of_records.node_id,
-                           function=_func,
-                           node_type='record_count',
-                           setting_input=node_number_of_records,
-                           input_node_ids=[node_number_of_records.depending_on_id])
+        self.add_node_step(
+            node_id=node_number_of_records.node_id,
+            function=_func,
+            node_type="record_count",
+            setting_input=node_number_of_records,
+            input_node_ids=[node_number_of_records.depending_on_id],
+        )
 
     def add_polars_code(self, node_polars_code: input_schema.NodePolarsCode):
         """Adds a node that executes custom Polars code.
@@ -676,11 +905,14 @@ class FlowGraph:
 
         def _func(*flowfile_tables: FlowDataEngine) -> FlowDataEngine:
             return execute_polars_code(*flowfile_tables, code=node_polars_code.polars_code_input.polars_code)
-        self.add_node_step(node_id=node_polars_code.node_id,
-                           function=_func,
-                           node_type='polars_code',
-                           setting_input=node_polars_code,
-                           input_node_ids=node_polars_code.depending_on_ids)
+
+        self.add_node_step(
+            node_id=node_polars_code.node_id,
+            function=_func,
+            node_type="polars_code",
+            setting_input=node_polars_code,
+            input_node_ids=node_polars_code.depending_on_ids,
+        )
 
         try:
             polars_code_parser.validate_code(node_polars_code.polars_code_input.polars_code)
@@ -688,9 +920,7 @@ class FlowGraph:
             node = self.get_node(node_id=node_polars_code.node_id)
             node.results.errors = str(e)
 
-    def add_dependency_on_polars_lazy_frame(self,
-                                            lazy_frame: pl.LazyFrame,
-                                            node_id: int):
+    def add_dependency_on_polars_lazy_frame(self, lazy_frame: pl.LazyFrame, node_id: int):
         """Adds a special node that directly injects a Polars LazyFrame into the graph.
 
         Note: This is intended for backend use and will not work in the UI editor.
@@ -699,13 +929,16 @@ class FlowGraph:
             lazy_frame: The Polars LazyFrame to inject.
             node_id: The ID for the new node.
         """
+
         def _func():
             return FlowDataEngine(lazy_frame)
-        node_promise = input_schema.NodePromise(flow_id=self.flow_id,
-                                                node_id=node_id, node_type="polars_lazy_frame",
-                                                is_setup=True)
-        self.add_node_step(node_id=node_promise.node_id, node_type=node_promise.node_type, function=_func,
-                           setting_input=node_promise)
+
+        node_promise = input_schema.NodePromise(
+            flow_id=self.flow_id, node_id=node_id, node_type="polars_lazy_frame", is_setup=True
+        )
+        self.add_node_step(
+            node_id=node_promise.node_id, node_type=node_promise.node_type, function=_func, setting_input=node_promise
+        )
 
     def add_unique(self, unique_settings: input_schema.NodeUnique):
         """Adds a node to find and remove duplicate rows.
@@ -717,12 +950,14 @@ class FlowGraph:
         def _func(fl: FlowDataEngine) -> FlowDataEngine:
             return fl.make_unique(unique_settings.unique_input)
 
-        self.add_node_step(node_id=unique_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='unique',
-                           setting_input=unique_settings,
-                           input_node_ids=[unique_settings.depending_on_id])
+        self.add_node_step(
+            node_id=unique_settings.node_id,
+            function=_func,
+            input_columns=[],
+            node_type="unique",
+            setting_input=unique_settings,
+            input_node_ids=[unique_settings.depending_on_id],
+        )
 
     def add_graph_solver(self, graph_solver_settings: input_schema.NodeGraphSolver):
         """Adds a node that solves graph-like problems within the data.
@@ -735,14 +970,17 @@ class FlowGraph:
             graph_solver_settings: The settings object defining the graph inputs
                 and the specific algorithm to apply.
         """
+
         def _func(fl: FlowDataEngine) -> FlowDataEngine:
             return fl.solve_graph(graph_solver_settings.graph_solver_input)
 
-        self.add_node_step(node_id=graph_solver_settings.node_id,
-                           function=_func,
-                           node_type='graph_solver',
-                           setting_input=graph_solver_settings,
-                           input_node_ids=[graph_solver_settings.depending_on_id])
+        self.add_node_step(
+            node_id=graph_solver_settings.node_id,
+            function=_func,
+            node_type="graph_solver",
+            setting_input=graph_solver_settings,
+            input_node_ids=[graph_solver_settings.depending_on_id],
+        )
 
     def add_formula(self, function_settings: input_schema.NodeFormula):
         """Adds a node that applies a formula to create or modify a column.
@@ -757,24 +995,28 @@ class FlowGraph:
         else:
             output_type = None
         if output_type not in (None, transform_schema.AUTO_DATA_TYPE):
-            new_col = [FlowfileColumn.from_input(column_name=function_settings.function.field.name,
-                                                 data_type=str(output_type))]
+            new_col = [
+                FlowfileColumn.from_input(column_name=function_settings.function.field.name, data_type=str(output_type))
+            ]
         else:
-            new_col = [FlowfileColumn.from_input(function_settings.function.field.name, 'String')]
+            new_col = [FlowfileColumn.from_input(function_settings.function.field.name, "String")]
 
         def _func(fl: FlowDataEngine):
-            return fl.apply_sql_formula(func=function_settings.function.function,
-                                        col_name=function_settings.function.field.name,
-                                        output_data_type=output_type)
-
-        self.add_node_step(function_settings.node_id, _func,
-                           output_schema=new_col,
-                           node_type='formula',
-                           renew_schema=False,
-                           setting_input=function_settings,
-                           input_node_ids=[function_settings.depending_on_id]
-                           )
-        # TODO: Add validation here
+            return fl.apply_sql_formula(
+                func=function_settings.function.function,
+                col_name=function_settings.function.field.name,
+                output_data_type=output_type,
+            )
+
+        self.add_node_step(
+            function_settings.node_id,
+            _func,
+            output_schema=new_col,
+            node_type="formula",
+            renew_schema=False,
+            setting_input=function_settings,
+            input_node_ids=[function_settings.depending_on_id],
+        )
         if error != "":
             node = self.get_node(function_settings.node_id)
             node.results.errors = error
@@ -791,22 +1033,27 @@ class FlowGraph:
         Returns:
             The `FlowGraph` instance for method chaining.
         """
+
         def _func(main: FlowDataEngine, right: FlowDataEngine) -> FlowDataEngine:
             for left_select in cross_join_settings.cross_join_input.left_select.renames:
                 left_select.is_available = True if left_select.old_name in main.schema else False
             for right_select in cross_join_settings.cross_join_input.right_select.renames:
                 right_select.is_available = True if right_select.old_name in right.schema else False
-            return main.do_cross_join(cross_join_input=cross_join_settings.cross_join_input,
-                                      auto_generate_selection=cross_join_settings.auto_generate_selection,
-                                      verify_integrity=False,
-                                      other=right)
-
-        self.add_node_step(node_id=cross_join_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='cross_join',
-                           setting_input=cross_join_settings,
-                           input_node_ids=cross_join_settings.depending_on_ids)
+            return main.do_cross_join(
+                cross_join_input=cross_join_settings.cross_join_input,
+                auto_generate_selection=cross_join_settings.auto_generate_selection,
+                verify_integrity=False,
+                other=right,
+            )
+
+        self.add_node_step(
+            node_id=cross_join_settings.node_id,
+            function=_func,
+            input_columns=[],
+            node_type="cross_join",
+            setting_input=cross_join_settings,
+            input_node_ids=cross_join_settings.depending_on_ids,
+        )
         return self
 
     def add_join(self, join_settings: input_schema.NodeJoin) -> "FlowGraph":
@@ -818,22 +1065,27 @@ class FlowGraph:
         Returns:
             The `FlowGraph` instance for method chaining.
         """
+
         def _func(main: FlowDataEngine, right: FlowDataEngine) -> FlowDataEngine:
             for left_select in join_settings.join_input.left_select.renames:
                 left_select.is_available = True if left_select.old_name in main.schema else False
             for right_select in join_settings.join_input.right_select.renames:
                 right_select.is_available = True if right_select.old_name in right.schema else False
-            return main.join(join_input=join_settings.join_input,
-                             auto_generate_selection=join_settings.auto_generate_selection,
-                             verify_integrity=False,
-                             other=right)
-
-        self.add_node_step(node_id=join_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='join',
-                           setting_input=join_settings,
-                           input_node_ids=join_settings.depending_on_ids)
+            return main.join(
+                join_input=join_settings.join_input,
+                auto_generate_selection=join_settings.auto_generate_selection,
+                verify_integrity=False,
+                other=right,
+            )
+
+        self.add_node_step(
+            node_id=join_settings.node_id,
+            function=_func,
+            input_columns=[],
+            node_type="join",
+            setting_input=join_settings,
+            input_node_ids=join_settings.depending_on_ids,
+        )
         return self
 
     def add_fuzzy_match(self, fuzzy_settings: input_schema.NodeFuzzyMatch) -> "FlowGraph":
@@ -849,31 +1101,43 @@ class FlowGraph:
         def _func(main: FlowDataEngine, right: FlowDataEngine) -> FlowDataEngine:
             node = self.get_node(node_id=fuzzy_settings.node_id)
             if self.execution_location == "local":
-                return main.fuzzy_join(fuzzy_match_input=deepcopy(fuzzy_settings.join_input),
-                                       other=right,
-                                       node_logger=self.flow_logger.get_node_logger(fuzzy_settings.node_id))
+                return main.fuzzy_join(
+                    fuzzy_match_input=deepcopy(fuzzy_settings.join_input),
+                    other=right,
+                    node_logger=self.flow_logger.get_node_logger(fuzzy_settings.node_id),
+                )
 
-            f = main.start_fuzzy_join(fuzzy_match_input=deepcopy(fuzzy_settings.join_input), other=right, file_ref=node.hash,
-                                      flow_id=self.flow_id, node_id=fuzzy_settings.node_id)
+            f = main.start_fuzzy_join(
+                fuzzy_match_input=deepcopy(fuzzy_settings.join_input),
+                other=right,
+                file_ref=node.hash,
+                flow_id=self.flow_id,
+                node_id=fuzzy_settings.node_id,
+            )
             logger.info("Started the fuzzy match action")
             node._fetch_cached_df = f  # Add to the node so it can be cancelled and fetch later if needed
             return FlowDataEngine(f.get_result())
 
         def schema_callback():
-            fm_input_copy = FuzzyMatchInputManager(fuzzy_settings.join_input)  # Deepcopy create an unique object per func
+            fm_input_copy = FuzzyMatchInputManager(
+                fuzzy_settings.join_input
+            )  # Deepcopy create an unique object per func
             node = self.get_node(node_id=fuzzy_settings.node_id)
-            return calculate_fuzzy_match_schema(fm_input_copy,
-                                                left_schema=node.node_inputs.main_inputs[0].schema,
-                                                right_schema=node.node_inputs.right_input.schema
-                                                )
-
-        self.add_node_step(node_id=fuzzy_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='fuzzy_match',
-                           setting_input=fuzzy_settings,
-                           input_node_ids=fuzzy_settings.depending_on_ids,
-                           schema_callback=schema_callback)
+            return calculate_fuzzy_match_schema(
+                fm_input_copy,
+                left_schema=node.node_inputs.main_inputs[0].schema,
+                right_schema=node.node_inputs.right_input.schema,
+            )
+
+        self.add_node_step(
+            node_id=fuzzy_settings.node_id,
+            function=_func,
+            input_columns=[],
+            node_type="fuzzy_match",
+            setting_input=fuzzy_settings,
+            input_node_ids=fuzzy_settings.depending_on_ids,
+            schema_callback=schema_callback,
+        )
 
         return self
 
@@ -890,14 +1154,17 @@ class FlowGraph:
         Returns:
             The `FlowGraph` instance for method chaining.
         """
+
         def _func(table: FlowDataEngine) -> FlowDataEngine:
             return table.split(node_text_to_rows.text_to_rows_input)
 
-        self.add_node_step(node_id=node_text_to_rows.node_id,
-                           function=_func,
-                           node_type='text_to_rows',
-                           setting_input=node_text_to_rows,
-                           input_node_ids=[node_text_to_rows.depending_on_id])
+        self.add_node_step(
+            node_id=node_text_to_rows.node_id,
+            function=_func,
+            node_type="text_to_rows",
+            setting_input=node_text_to_rows,
+            input_node_ids=[node_text_to_rows.depending_on_id],
+        )
         return self
 
     def add_sort(self, sort_settings: input_schema.NodeSort) -> "FlowGraph":
@@ -913,11 +1180,13 @@ class FlowGraph:
         def _func(table: FlowDataEngine) -> FlowDataEngine:
             return table.do_sort(sort_settings.sort_input)
 
-        self.add_node_step(node_id=sort_settings.node_id,
-                           function=_func,
-                           node_type='sort',
-                           setting_input=sort_settings,
-                           input_node_ids=[sort_settings.depending_on_id])
+        self.add_node_step(
+            node_id=sort_settings.node_id,
+            function=_func,
+            node_type="sort",
+            setting_input=sort_settings,
+            input_node_ids=[sort_settings.depending_on_id],
+        )
         return self
 
     def add_sample(self, sample_settings: input_schema.NodeSample) -> "FlowGraph":
@@ -929,15 +1198,17 @@ class FlowGraph:
         Returns:
             The `FlowGraph` instance for method chaining.
         """
+
         def _func(table: FlowDataEngine) -> FlowDataEngine:
             return table.get_sample(sample_settings.sample_size)
 
-        self.add_node_step(node_id=sample_settings.node_id,
-                           function=_func,
-                           node_type='sample',
-                           setting_input=sample_settings,
-                           input_node_ids=[sample_settings.depending_on_id]
-                           )
+        self.add_node_step(
+            node_id=sample_settings.node_id,
+            function=_func,
+            node_type="sample",
+            setting_input=sample_settings,
+            input_node_ids=[sample_settings.depending_on_id],
+        )
         return self
 
     def add_record_id(self, record_id_settings: input_schema.NodeRecordId) -> "FlowGraph":
@@ -954,12 +1225,13 @@ class FlowGraph:
         def _func(table: FlowDataEngine) -> FlowDataEngine:
             return table.add_record_id(record_id_settings.record_id_input)
 
-        self.add_node_step(node_id=record_id_settings.node_id,
-                           function=_func,
-                           node_type='record_id',
-                           setting_input=record_id_settings,
-                           input_node_ids=[record_id_settings.depending_on_id]
-                           )
+        self.add_node_step(
+            node_id=record_id_settings.node_id,
+            function=_func,
+            node_type="record_id",
+            setting_input=record_id_settings,
+            input_node_ids=[record_id_settings.depending_on_id],
+        )
         return self
 
     def add_select(self, select_settings: input_schema.NodeSelect) -> "FlowGraph":
@@ -991,16 +1263,19 @@ class FlowGraph:
             for i in ids_to_remove:
                 v = select_cols.pop(i)
                 del v
-            return table.do_select(select_inputs=transform_schema.SelectInputs(select_cols),
-                                   keep_missing=select_settings.keep_missing)
-
-        self.add_node_step(node_id=select_settings.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='select',
-                           drop_columns=list(drop_cols),
-                           setting_input=select_settings,
-                           input_node_ids=[select_settings.depending_on_id])
+            return table.do_select(
+                select_inputs=transform_schema.SelectInputs(select_cols), keep_missing=select_settings.keep_missing
+            )
+
+        self.add_node_step(
+            node_id=select_settings.node_id,
+            function=_func,
+            input_columns=[],
+            node_type="select",
+            drop_columns=list(drop_cols),
+            setting_input=select_settings,
+            input_node_ids=[select_settings.depending_on_id],
+        )
         return self
 
     @property
@@ -1008,7 +1283,7 @@ class FlowGraph:
         """Checks if the graph has any nodes."""
         return len(self._node_ids) > 0
 
-    def delete_node(self, node_id: Union[int, str]):
+    def delete_node(self, node_id: int | str):
         """Deletes a node from the graph and updates all its connections.
 
         Args:
@@ -1023,7 +1298,7 @@ class FlowGraph:
         if node:
             logger.info(f"Found node: {node_id}, processing deletion")
 
-            lead_to_steps: List[FlowNode] = node.leads_to_nodes
+            lead_to_steps: list[FlowNode] = node.leads_to_nodes
             logger.debug(f"Node {node_id} leads to {len(lead_to_steps)} other nodes")
 
             if len(lead_to_steps) > 0:
@@ -1032,7 +1307,7 @@ class FlowGraph:
                     lead_to_step.delete_input_node(node_id, complete=True)
 
             if not node.is_start:
-                depends_on: List[FlowNode] = node.node_inputs.get_all_inputs()
+                depends_on: list[FlowNode] = node.node_inputs.get_all_inputs()
                 logger.debug(f"Node {node_id} depends on {len(depends_on)} other nodes")
 
                 for depend_on in depends_on:
@@ -1052,18 +1327,20 @@ class FlowGraph:
         """Checks if the graph has an initial input data source."""
         return self._input_data is not None
 
-    def add_node_step(self,
-                      node_id: Union[int, str],
-                      function: Callable,
-                      input_columns: List[str] = None,
-                      output_schema: List[FlowfileColumn] = None,
-                      node_type: str = None,
-                      drop_columns: List[str] = None,
-                      renew_schema: bool = True,
-                      setting_input: Any = None,
-                      cache_results: bool = None,
-                      schema_callback: Callable = None,
-                      input_node_ids: List[int] = None) -> FlowNode:
+    def add_node_step(
+        self,
+        node_id: int | str,
+        function: Callable,
+        input_columns: list[str] = None,
+        output_schema: list[FlowfileColumn] = None,
+        node_type: str = None,
+        drop_columns: list[str] = None,
+        renew_schema: bool = True,
+        setting_input: Any = None,
+        cache_results: bool = None,
+        schema_callback: Callable = None,
+        input_node_ids: list[int] = None,
+    ) -> FlowNode:
         """The core method for adding or updating a node in the graph.
 
         Args:
@@ -1096,29 +1373,33 @@ class FlowGraph:
         if isinstance(input_columns, str):
             input_columns = [input_columns]
         if (
-                input_nodes is not None or
-                function.__name__ in ('placeholder', 'analysis_preparation') or
-                node_type in ("cloud_storage_reader", "polars_lazy_frame", "input_data")
+            input_nodes is not None
+            or function.__name__ in ("placeholder", "analysis_preparation")
+            or node_type in ("cloud_storage_reader", "polars_lazy_frame", "input_data")
         ):
             if not existing_node:
-                node = FlowNode(node_id=node_id,
-                                function=function,
-                                output_schema=output_schema,
-                                input_columns=input_columns,
-                                drop_columns=drop_columns,
-                                renew_schema=renew_schema,
-                                setting_input=setting_input,
-                                node_type=node_type,
-                                name=function.__name__,
-                                schema_callback=schema_callback,
-                                parent_uuid=self.uuid)
+                node = FlowNode(
+                    node_id=node_id,
+                    function=function,
+                    output_schema=output_schema,
+                    input_columns=input_columns,
+                    drop_columns=drop_columns,
+                    renew_schema=renew_schema,
+                    setting_input=setting_input,
+                    node_type=node_type,
+                    name=function.__name__,
+                    schema_callback=schema_callback,
+                    parent_uuid=self.uuid,
+                )
             else:
-                existing_node.update_node(function=function,
-                                          output_schema=output_schema,
-                                          input_columns=input_columns,
-                                          drop_columns=drop_columns,
-                                          setting_input=setting_input,
-                                          schema_callback=schema_callback)
+                existing_node.update_node(
+                    function=function,
+                    output_schema=output_schema,
+                    input_columns=input_columns,
+                    drop_columns=drop_columns,
+                    setting_input=setting_input,
+                    schema_callback=schema_callback,
+                )
                 node = existing_node
         else:
             raise Exception("No data initialized")
@@ -1126,7 +1407,7 @@ class FlowGraph:
         self._node_ids.append(node_id)
         return node
 
-    def add_include_cols(self, include_columns: List[str]):
+    def add_include_cols(self, include_columns: list[str]):
         """Adds columns to both the input and output column lists.
 
         Args:
@@ -1147,23 +1428,30 @@ class FlowGraph:
         """
 
         def _func(df: FlowDataEngine):
-            execute_remote = self.execution_location != 'local'
-            df.output(output_fs=output_file.output_settings, flow_id=self.flow_id, node_id=output_file.node_id,
-                      execute_remote=execute_remote)
+            execute_remote = self.execution_location != "local"
+            df.output(
+                output_fs=output_file.output_settings,
+                flow_id=self.flow_id,
+                node_id=output_file.node_id,
+                execute_remote=execute_remote,
+            )
             return df
 
         def schema_callback():
             input_node: FlowNode = self.get_node(output_file.node_id).node_inputs.main_inputs[0]
 
             return input_node.schema
-        input_node_id = getattr(output_file, "depending_on_id") if hasattr(output_file, 'depending_on_id') else None
-        self.add_node_step(node_id=output_file.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type='output',
-                           setting_input=output_file,
-                           schema_callback=schema_callback,
-                           input_node_ids=[input_node_id])
+
+        input_node_id = output_file.depending_on_id if hasattr(output_file, "depending_on_id") else None
+        self.add_node_step(
+            node_id=output_file.node_id,
+            function=_func,
+            input_columns=[],
+            node_type="output",
+            setting_input=output_file,
+            schema_callback=schema_callback,
+            input_node_ids=[input_node_id],
+        )
 
     def add_database_writer(self, node_database_writer: input_schema.NodeDatabaseWriter):
         """Adds a node to write data to a database.
@@ -1172,18 +1460,20 @@ class FlowGraph:
             node_database_writer: The settings for the database writer node.
         """
 
-        node_type = 'database_writer'
+        node_type = "database_writer"
         database_settings: input_schema.DatabaseWriteSettings = node_database_writer.database_write_settings
-        database_connection: Optional[input_schema.DatabaseConnection | input_schema.FullDatabaseConnection]
-        if database_settings.connection_mode == 'inline':
+        database_connection: input_schema.DatabaseConnection | input_schema.FullDatabaseConnection | None
+        if database_settings.connection_mode == "inline":
             database_connection: input_schema.DatabaseConnection = database_settings.database_connection
-            encrypted_password = get_encrypted_secret(current_user_id=node_database_writer.user_id,
-                                                      secret_name=database_connection.password_ref)
+            encrypted_password = get_encrypted_secret(
+                current_user_id=node_database_writer.user_id, secret_name=database_connection.password_ref
+            )
             if encrypted_password is None:
                 raise HTTPException(status_code=400, detail="Password not found")
         else:
-            database_reference_settings = get_local_database_connection(database_settings.database_connection_name,
-                                                                        node_database_writer.user_id)
+            database_reference_settings = get_local_database_connection(
+                database_settings.database_connection_name, node_database_writer.user_id
+            )
             encrypted_password = database_reference_settings.password.get_secret_value()
 
         def _func(df: FlowDataEngine):
@@ -1192,14 +1482,20 @@ class FlowGraph:
                 sql_models.DatabaseExternalWriteSettings.create_from_from_node_database_writer(
                     node_database_writer=node_database_writer,
                     password=encrypted_password,
-                    table_name=(database_settings.schema_name+'.'+database_settings.table_name
-                                if database_settings.schema_name else database_settings.table_name),
-                    database_reference_settings=(database_reference_settings if database_settings.connection_mode == 'reference'
-                                                 else None),
-                    lf=df.data_frame
+                    table_name=(
+                        database_settings.schema_name + "." + database_settings.table_name
+                        if database_settings.schema_name
+                        else database_settings.table_name
+                    ),
+                    database_reference_settings=(
+                        database_reference_settings if database_settings.connection_mode == "reference" else None
+                    ),
+                    lf=df.data_frame,
                 )
             )
-            external_database_writer = ExternalDatabaseWriter(database_external_write_settings, wait_on_completion=False)
+            external_database_writer = ExternalDatabaseWriter(
+                database_external_write_settings, wait_on_completion=False
+            )
             node._fetch_cached_df = external_database_writer
             external_database_writer.get_result()
             return df
@@ -1226,56 +1522,64 @@ class FlowGraph:
         """
 
         logger.info("Adding database reader")
-        node_type = 'database_reader'
+        node_type = "database_reader"
         database_settings: input_schema.DatabaseSettings = node_database_reader.database_settings
-        database_connection: Optional[input_schema.DatabaseConnection | input_schema.FullDatabaseConnection]
-        if database_settings.connection_mode == 'inline':
+        database_connection: input_schema.DatabaseConnection | input_schema.FullDatabaseConnection | None
+        if database_settings.connection_mode == "inline":
             database_connection: input_schema.DatabaseConnection = database_settings.database_connection
-            encrypted_password = get_encrypted_secret(current_user_id=node_database_reader.user_id,
-                                                      secret_name=database_connection.password_ref)
+            encrypted_password = get_encrypted_secret(
+                current_user_id=node_database_reader.user_id, secret_name=database_connection.password_ref
+            )
             if encrypted_password is None:
                 raise HTTPException(status_code=400, detail="Password not found")
         else:
-            database_reference_settings = get_local_database_connection(database_settings.database_connection_name,
-                                                                        node_database_reader.user_id)
+            database_reference_settings = get_local_database_connection(
+                database_settings.database_connection_name, node_database_reader.user_id
+            )
             database_connection = database_reference_settings
             encrypted_password = database_reference_settings.password.get_secret_value()
 
         def _func():
-            sql_source = BaseSqlSource(query=None if database_settings.query_mode == 'table' else database_settings.query,
-                                       table_name=database_settings.table_name,
-                                       schema_name=database_settings.schema_name,
-                                       fields=node_database_reader.fields,
-                                       )
+            sql_source = BaseSqlSource(
+                query=None if database_settings.query_mode == "table" else database_settings.query,
+                table_name=database_settings.table_name,
+                schema_name=database_settings.schema_name,
+                fields=node_database_reader.fields,
+            )
             database_external_read_settings = (
                 sql_models.DatabaseExternalReadSettings.create_from_from_node_database_reader(
                     node_database_reader=node_database_reader,
                     password=encrypted_password,
                     query=sql_source.query,
-                    database_reference_settings=(database_reference_settings if database_settings.connection_mode == 'reference'
-                                                 else None),
+                    database_reference_settings=(
+                        database_reference_settings if database_settings.connection_mode == "reference" else None
+                    ),
                 )
             )
 
-            external_database_fetcher = ExternalDatabaseFetcher(database_external_read_settings, wait_on_completion=False)
+            external_database_fetcher = ExternalDatabaseFetcher(
+                database_external_read_settings, wait_on_completion=False
+            )
             node._fetch_cached_df = external_database_fetcher
             fl = FlowDataEngine(external_database_fetcher.get_result())
             node_database_reader.fields = [c.get_minimal_field_info() for c in fl.schema]
             return fl
 
         def schema_callback():
-            sql_source = SqlSource(connection_string=
-                                   sql_utils.construct_sql_uri(database_type=database_connection.database_type,
-                                                               host=database_connection.host,
-                                                               port=database_connection.port,
-                                                               database=database_connection.database,
-                                                               username=database_connection.username,
-                                                               password=decrypt_secret(encrypted_password)),
-                                   query=None if database_settings.query_mode == 'table' else database_settings.query,
-                                   table_name=database_settings.table_name,
-                                   schema_name=database_settings.schema_name,
-                                   fields=node_database_reader.fields,
-                                   )
+            sql_source = SqlSource(
+                connection_string=sql_utils.construct_sql_uri(
+                    database_type=database_connection.database_type,
+                    host=database_connection.host,
+                    port=database_connection.port,
+                    database=database_connection.database,
+                    username=database_connection.username,
+                    password=decrypt_secret(encrypted_password),
+                ),
+                query=None if database_settings.query_mode == "table" else database_settings.query,
+                table_name=database_settings.table_name,
+                schema_name=database_settings.schema_name,
+                fields=node_database_reader.fields,
+            )
             return sql_source.get_schema()
 
         node = self.get_node(node_database_reader.node_id)
@@ -1285,16 +1589,20 @@ class FlowGraph:
             node.function = _func
             node.setting_input = node_database_reader
             node.node_settings.cache_results = node_database_reader.cache_results
-            if node_database_reader.node_id not in set(start_node.node_id for start_node in self._flow_starts):
-                self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
             node.schema_callback = schema_callback
         else:
-            node = FlowNode(node_database_reader.node_id, function=_func,
-                            setting_input=node_database_reader,
-                            name=node_type, node_type=node_type, parent_uuid=self.uuid,
-                            schema_callback=schema_callback)
+            node = FlowNode(
+                node_database_reader.node_id,
+                function=_func,
+                setting_input=node_database_reader,
+                name=node_type,
+                node_type=node_type,
+                parent_uuid=self.uuid,
+                schema_callback=schema_callback,
+            )
             self._node_db[node_database_reader.node_id] = node
-            self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
             self._node_ids.append(node_database_reader.node_id)
 
     def add_sql_source(self, external_source_input: input_schema.NodeExternalSource):
@@ -1305,7 +1613,7 @@ class FlowGraph:
         Args:
             external_source_input: The settings for the external SQL source node.
         """
-        logger.info('Adding sql source')
+        logger.info("Adding sql source")
         self.add_external_source(external_source_input)
 
     def add_cloud_storage_writer(self, node_cloud_storage_writer: input_schema.NodeCloudStorageWriter) -> None:
@@ -1316,27 +1624,30 @@ class FlowGraph:
         """
 
         node_type = "cloud_storage_writer"
+
         def _func(df: FlowDataEngine):
             df.lazy = True
-            execute_remote = self.execution_location != 'local'
+            execute_remote = self.execution_location != "local"
             cloud_connection_settings = get_cloud_connection_settings(
                 connection_name=node_cloud_storage_writer.cloud_storage_settings.connection_name,
                 user_id=node_cloud_storage_writer.user_id,
-                auth_mode=node_cloud_storage_writer.cloud_storage_settings.auth_mode
+                auth_mode=node_cloud_storage_writer.cloud_storage_settings.auth_mode,
             )
             full_cloud_storage_connection = FullCloudStorageConnection(
                 storage_type=cloud_connection_settings.storage_type,
                 auth_method=cloud_connection_settings.auth_method,
                 aws_allow_unsafe_html=cloud_connection_settings.aws_allow_unsafe_html,
-                **CloudStorageReader.get_storage_options(cloud_connection_settings)
+                **CloudStorageReader.get_storage_options(cloud_connection_settings),
             )
             if execute_remote:
                 settings = get_cloud_storage_write_settings_worker_interface(
                     write_settings=node_cloud_storage_writer.cloud_storage_settings,
                     connection=full_cloud_storage_connection,
                     lf=df.data_frame,
+                    user_id=node_cloud_storage_writer.user_id,
                     flowfile_node_id=node_cloud_storage_writer.node_id,
-                    flowfile_flow_id=self.flow_id)
+                    flowfile_flow_id=self.flow_id,
+                )
                 external_database_writer = ExternalCloudWriter(settings, wait_on_completion=False)
                 node._fetch_cached_df = external_database_writer
                 external_database_writer.get_result()
@@ -1362,7 +1673,7 @@ class FlowGraph:
             node_type=node_type,
             setting_input=node_cloud_storage_writer,
             schema_callback=schema_callback,
-            input_node_ids=[node_cloud_storage_writer.depending_on_id]
+            input_node_ids=[node_cloud_storage_writer.depending_on_id],
         )
 
         node = self.get_node(node_cloud_storage_writer.node_id)
@@ -1380,49 +1691,53 @@ class FlowGraph:
         def _func():
             logger.info("Starting to run the schema callback for cloud storage reader")
             self.flow_logger.info("Starting to run the schema callback for cloud storage reader")
-            settings = CloudStorageReadSettingsInternal(read_settings=cloud_storage_read_settings,
-                                                        connection=get_cloud_connection_settings(
-                                                            connection_name=cloud_storage_read_settings.connection_name,
-                                                            user_id=node_cloud_storage_reader.user_id,
-                                                            auth_mode=cloud_storage_read_settings.auth_mode
-                                                        ))
+            settings = CloudStorageReadSettingsInternal(
+                read_settings=cloud_storage_read_settings,
+                connection=get_cloud_connection_settings(
+                    connection_name=cloud_storage_read_settings.connection_name,
+                    user_id=node_cloud_storage_reader.user_id,
+                    auth_mode=cloud_storage_read_settings.auth_mode,
+                ),
+            )
             fl = FlowDataEngine.from_cloud_storage_obj(settings)
             return fl
 
-        node = self.add_node_step(node_id=node_cloud_storage_reader.node_id,
-                                  function=_func,
-                                  cache_results=node_cloud_storage_reader.cache_results,
-                                  setting_input=node_cloud_storage_reader,
-                                  node_type=node_type,
-                                  )
-        if node_cloud_storage_reader.node_id not in set(start_node.node_id for start_node in self._flow_starts):
-            self._flow_starts.append(node)
+        node = self.add_node_step(
+            node_id=node_cloud_storage_reader.node_id,
+            function=_func,
+            cache_results=node_cloud_storage_reader.cache_results,
+            setting_input=node_cloud_storage_reader,
+            node_type=node_type,
+        )
+        self.add_node_to_starting_list(node)
 
-    def add_external_source(self,
-                            external_source_input: input_schema.NodeExternalSource):
+    def add_external_source(self, external_source_input: input_schema.NodeExternalSource):
         """Adds a node for a custom external data source.
 
         Args:
             external_source_input: The settings for the external source node.
         """
 
-        node_type = 'external_source'
+        node_type = "external_source"
         external_source_script = getattr(external_sources.custom_external_sources, external_source_input.identifier)
-        source_settings = (getattr(input_schema, snake_case_to_camel_case(external_source_input.identifier)).
-                           model_validate(external_source_input.source_settings))
-        if hasattr(external_source_script, 'initial_getter'):
-            initial_getter = getattr(external_source_script, 'initial_getter')(source_settings)
+        source_settings = getattr(
+            input_schema, snake_case_to_camel_case(external_source_input.identifier)
+        ).model_validate(external_source_input.source_settings)
+        if hasattr(external_source_script, "initial_getter"):
+            initial_getter = external_source_script.initial_getter(source_settings)
         else:
             initial_getter = None
         data_getter = external_source_script.getter(source_settings)
-        external_source = data_source_factory(source_type='custom',
-                                              data_getter=data_getter,
-                                              initial_data_getter=initial_getter,
-                                              orientation=external_source_input.source_settings.orientation,
-                                              schema=None)
+        external_source = data_source_factory(
+            source_type="custom",
+            data_getter=data_getter,
+            initial_data_getter=initial_getter,
+            orientation=external_source_input.source_settings.orientation,
+            schema=None,
+        )
 
         def _func():
-            logger.info('Calling external source')
+            logger.info("Calling external source")
             fl = FlowDataEngine.create_from_external_source(external_source=external_source)
             external_source_input.source_settings.fields = [c.get_minimal_field_info() for c in fl.schema]
             return fl
@@ -1434,31 +1749,39 @@ class FlowGraph:
             node.function = _func
             node.setting_input = external_source_input
             node.node_settings.cache_results = external_source_input.cache_results
-            if external_source_input.node_id not in set(start_node.node_id for start_node in self._flow_starts):
-                self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
+
         else:
-            node = FlowNode(external_source_input.node_id, function=_func,
-                            setting_input=external_source_input,
-                            name=node_type, node_type=node_type, parent_uuid=self.uuid)
+            node = FlowNode(
+                external_source_input.node_id,
+                function=_func,
+                setting_input=external_source_input,
+                name=node_type,
+                node_type=node_type,
+                parent_uuid=self.uuid,
+            )
             self._node_db[external_source_input.node_id] = node
-            self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
             self._node_ids.append(external_source_input.node_id)
         if external_source_input.source_settings.fields and len(external_source_input.source_settings.fields) > 0:
-            logger.info('Using provided schema in the node')
+            logger.info("Using provided schema in the node")
 
             def schema_callback():
-                return [FlowfileColumn.from_input(f.name, f.data_type) for f in
-                        external_source_input.source_settings.fields]
+                return [
+                    FlowfileColumn.from_input(f.name, f.data_type) for f in external_source_input.source_settings.fields
+                ]
 
             node.schema_callback = schema_callback
         else:
-            logger.warning('Removing schema')
+            logger.warning("Removing schema")
             node._schema_callback = None
-        self.add_node_step(node_id=external_source_input.node_id,
-                           function=_func,
-                           input_columns=[],
-                           node_type=node_type,
-                           setting_input=external_source_input)
+        self.add_node_step(
+            node_id=external_source_input.node_id,
+            function=_func,
+            input_columns=[],
+            node_type=node_type,
+            setting_input=external_source_input,
+        )
 
     def add_read(self, input_file: input_schema.NodeRead):
         """Adds a node to read data from a local file (e.g., CSV, Parquet, Excel).
@@ -1466,8 +1789,10 @@ class FlowGraph:
         Args:
             input_file: The settings for the read operation.
         """
-        if (input_file.received_file.file_type in ('xlsx', 'excel') and
-                input_file.received_file.table_settings.sheet_name == ''):
+        if (
+            input_file.received_file.file_type in ("xlsx", "excel")
+            and input_file.received_file.table_settings.sheet_name == ""
+        ):
             sheet_name = fastexcel.read_excel(input_file.received_file.path).sheet_names[0]
             input_file.received_file.table_settings.sheet_name = sheet_name
 
@@ -1476,14 +1801,17 @@ class FlowGraph:
 
         def _func():
             input_file.received_file.set_absolute_filepath()
-            if input_file.received_file.file_type == 'parquet':
+            if input_file.received_file.file_type == "parquet":
                 input_data = FlowDataEngine.create_from_path(input_file.received_file)
-            elif input_file.received_file.file_type == 'csv' and 'utf' in input_file.received_file.table_settings.encoding:
+            elif (
+                input_file.received_file.file_type == "csv"
+                and "utf" in input_file.received_file.table_settings.encoding
+            ):
                 input_data = FlowDataEngine.create_from_path(input_file.received_file)
             else:
-                input_data = FlowDataEngine.create_from_path_worker(input_file.received_file,
-                                                                    node_id=input_file.node_id,
-                                                                    flow_id=self.flow_id)
+                input_data = FlowDataEngine.create_from_path_worker(
+                    input_file.received_file, node_id=input_file.node_id, flow_id=self.flow_id
+                )
             input_data.name = input_file.received_file.name
             return input_data
 
@@ -1491,51 +1819,58 @@ class FlowGraph:
         schema_callback = None
         if node:
             start_hash = node.hash
-            node.node_type = 'read'
-            node.name = 'read'
+            node.node_type = "read"
+            node.name = "read"
             node.function = _func
             node.setting_input = input_file
-            if input_file.node_id not in set(start_node.node_id for start_node in self._flow_starts):
-                self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
 
             if start_hash != node.hash:
-                logger.info('Hash changed, updating schema')
+                logger.info("Hash changed, updating schema")
                 if len(received_file.fields) > 0:
                     # If the file has fields defined, we can use them to create the schema
                     def schema_callback():
                         return [FlowfileColumn.from_input(f.name, f.data_type) for f in received_file.fields]
 
-                elif input_file.received_file.file_type in ('csv', 'json', 'parquet'):
+                elif input_file.received_file.file_type in ("csv", "json", "parquet"):
                     # everything that can be scanned by polars
                     def schema_callback():
                         input_data = FlowDataEngine.create_from_path(input_file.received_file)
                         return input_data.schema
 
-                elif input_file.received_file.file_type in ('xlsx', 'excel'):
+                elif input_file.received_file.file_type in ("xlsx", "excel"):
                     # If the file is an Excel file, we need to use the openpyxl engine to read the schema
-                    schema_callback = get_xlsx_schema_callback(engine='openpyxl',
-                                                               file_path=received_file.file_path,
-                                                               sheet_name=received_file.table_settings.sheet_name,
-                                                               start_row=received_file.table_settings.start_row,
-                                                               end_row=received_file.table_settings.end_row,
-                                                               start_column=received_file.table_settings.start_column,
-                                                               end_column=received_file.table_settings.end_column,
-                                                               has_headers=received_file.table_settings.has_headers)
+                    schema_callback = get_xlsx_schema_callback(
+                        engine="openpyxl",
+                        file_path=received_file.file_path,
+                        sheet_name=received_file.table_settings.sheet_name,
+                        start_row=received_file.table_settings.start_row,
+                        end_row=received_file.table_settings.end_row,
+                        start_column=received_file.table_settings.start_column,
+                        end_column=received_file.table_settings.end_column,
+                        has_headers=received_file.table_settings.has_headers,
+                    )
                 else:
                     schema_callback = None
         else:
-            node = FlowNode(input_file.node_id, function=_func,
-                            setting_input=input_file,
-                            name='read', node_type='read', parent_uuid=self.uuid)
+            node = FlowNode(
+                input_file.node_id,
+                function=_func,
+                setting_input=input_file,
+                name="read",
+                node_type="read",
+                parent_uuid=self.uuid,
+            )
             self._node_db[input_file.node_id] = node
-            self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
             self._node_ids.append(input_file.node_id)
 
         if schema_callback is not None:
             node.schema_callback = schema_callback
+            node.user_provided_schema_callback = schema_callback
         return self
 
-    def add_datasource(self, input_file: Union[input_schema.NodeDatasource, input_schema.NodeManualInput]) -> "FlowGraph":
+    def add_datasource(self, input_file: input_schema.NodeDatasource | input_schema.NodeManualInput) -> "FlowGraph":
         """Adds a data source node to the graph.
 
         This method serves as a factory for creating starting nodes, handling both
@@ -1549,25 +1884,30 @@ class FlowGraph:
         """
         if isinstance(input_file, input_schema.NodeManualInput):
             input_data = FlowDataEngine(input_file.raw_data_format)
-            ref = 'manual_input'
+            ref = "manual_input"
         else:
             input_data = FlowDataEngine(path_ref=input_file.file_ref)
-            ref = 'datasource'
+            ref = "datasource"
         node = self.get_node(input_file.node_id)
         if node:
             node.node_type = ref
             node.name = ref
             node.function = input_data
             node.setting_input = input_file
-            if not input_file.node_id in set(start_node.node_id for start_node in self._flow_starts):
-                self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
+
         else:
             input_data.collect()
-            node = FlowNode(input_file.node_id, function=input_data,
-                            setting_input=input_file,
-                            name=ref, node_type=ref, parent_uuid=self.uuid)
+            node = FlowNode(
+                input_file.node_id,
+                function=input_data,
+                setting_input=input_file,
+                name=ref,
+                node_type=ref,
+                parent_uuid=self.uuid,
+            )
             self._node_db[input_file.node_id] = node
-            self._flow_starts.append(node)
+            self.add_node_to_starting_list(node)
             self._node_ids.append(input_file.node_id)
         return self
 
@@ -1582,7 +1922,7 @@ class FlowGraph:
         self.add_datasource(input_file)
 
     @property
-    def nodes(self) -> List[FlowNode]:
+    def nodes(self) -> list[FlowNode]:
         """Gets a list of all FlowNode objects in the graph."""
 
         return list(self._node_db.values())
@@ -1592,7 +1932,7 @@ class FlowGraph:
         """Gets the current execution mode ('Development' or 'Performance')."""
         return self.flow_settings.execution_mode
 
-    def get_implicit_starter_nodes(self) -> List[FlowNode]:
+    def get_implicit_starter_nodes(self) -> list[FlowNode]:
         """Finds nodes that can act as starting points but are not explicitly defined as such.
 
         Some nodes, like the Polars Code node, can function without an input. This
@@ -1638,24 +1978,31 @@ class FlowGraph:
         if not flow_node:
             raise Exception("Node not found found")
         skip_nodes, execution_order = compute_execution_plan(
-            nodes=self.nodes, flow_starts=self._flow_starts+self.get_implicit_starter_nodes()
+            nodes=self.nodes, flow_starts=self._flow_starts + self.get_implicit_starter_nodes()
         )
         if flow_node.node_id in [skip_node.node_id for skip_node in skip_nodes]:
             raise Exception("Node can not be executed because it does not have it's inputs")
 
-    def create_initial_run_information(self, number_of_nodes: int,
-                                       run_type: Literal["fetch_one", "full_run"]):
+    def create_initial_run_information(self, number_of_nodes: int, run_type: Literal["fetch_one", "full_run"]):
         return RunInformation(
-            flow_id=self.flow_id, start_time=datetime.datetime.now(), end_time=None,
-            success=None, number_of_nodes=number_of_nodes, node_step_result=[],
-            run_type=run_type
+            flow_id=self.flow_id,
+            start_time=datetime.datetime.now(),
+            end_time=None,
+            success=None,
+            number_of_nodes=number_of_nodes,
+            node_step_result=[],
+            run_type=run_type,
         )
 
     def create_empty_run_information(self) -> RunInformation:
         return RunInformation(
-            flow_id=self.flow_id, start_time=None, end_time=None,
-            success=None, number_of_nodes=0, node_step_result=[],
-            run_type="init"
+            flow_id=self.flow_id,
+            start_time=None,
+            end_time=None,
+            success=None,
+            number_of_nodes=0,
+            node_step_result=[],
+            run_type="init",
         )
 
     def trigger_fetch_node(self, node_id: int) -> RunInformation | None:
@@ -1669,14 +2016,16 @@ class FlowGraph:
         self.latest_run_info = self.create_initial_run_information(1, "fetch_one")
         node_logger = self.flow_logger.get_node_logger(flow_node.node_id)
         node_result = NodeResult(node_id=flow_node.node_id, node_name=flow_node.name)
-        logger.info(f'Starting to run: node {flow_node.node_id}, start time: {node_result.start_timestamp}')
+        logger.info(f"Starting to run: node {flow_node.node_id}, start time: {node_result.start_timestamp}")
         try:
             self.latest_run_info.node_step_result.append(node_result)
-            flow_node.execute_node(run_location=self.flow_settings.execution_location,
-                                   performance_mode=False,
-                                   node_logger=node_logger,
-                                   optimize_for_downstream=False,
-                                   reset_cache=True)
+            flow_node.execute_node(
+                run_location=self.flow_settings.execution_location,
+                performance_mode=False,
+                node_logger=node_logger,
+                optimize_for_downstream=False,
+                reset_cache=True,
+            )
             node_result.error = str(flow_node.results.errors)
             if self.flow_settings.is_canceled:
                 node_result.success = None
@@ -1691,12 +2040,12 @@ class FlowGraph:
             self.flow_settings.is_running = False
             return self.get_run_info()
         except Exception as e:
-            node_result.error = 'Node did not run'
+            node_result.error = "Node did not run"
             node_result.success = False
             node_result.end_timestamp = time()
             node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
             node_result.is_running = False
-            node_logger.error(f'Error in node {flow_node.node_id}: {e}')
+            node_logger.error(f"Error in node {flow_node.node_id}: {e}")
         finally:
             self.flow_settings.is_running = False
 
@@ -1713,39 +2062,38 @@ class FlowGraph:
             Exception: If the flow is already running.
         """
         if self.flow_settings.is_running:
-            raise Exception('Flow is already running')
+            raise Exception("Flow is already running")
         try:
-
             self.flow_settings.is_running = True
             self.flow_settings.is_canceled = False
             self.flow_logger.clear_log_file()
-            self.flow_logger.info('Starting to run flowfile flow...')
-
+            self.flow_logger.info("Starting to run flowfile flow...")
             skip_nodes, execution_order = compute_execution_plan(
-                nodes=self.nodes,
-                flow_starts=self._flow_starts+self.get_implicit_starter_nodes()
+                nodes=self.nodes, flow_starts=self._flow_starts + self.get_implicit_starter_nodes()
             )
 
             self.latest_run_info = self.create_initial_run_information(len(execution_order), "full_run")
 
             skip_node_message(self.flow_logger, skip_nodes)
             execution_order_message(self.flow_logger, execution_order)
-            performance_mode = self.flow_settings.execution_mode == 'Performance'
+            performance_mode = self.flow_settings.execution_mode == "Performance"
 
             for node in execution_order:
                 node_logger = self.flow_logger.get_node_logger(node.node_id)
                 if self.flow_settings.is_canceled:
-                    self.flow_logger.info('Flow canceled')
+                    self.flow_logger.info("Flow canceled")
                     break
                 if node in skip_nodes:
-                    node_logger.info(f'Skipping node {node.node_id}')
+                    node_logger.info(f"Skipping node {node.node_id}")
                     continue
                 node_result = NodeResult(node_id=node.node_id, node_name=node.name)
                 self.latest_run_info.node_step_result.append(node_result)
-                logger.info(f'Starting to run: node {node.node_id}, start time: {node_result.start_timestamp}')
-                node.execute_node(run_location=self.flow_settings.execution_location,
-                                  performance_mode=performance_mode,
-                                  node_logger=node_logger)
+                logger.info(f"Starting to run: node {node.node_id}, start time: {node_result.start_timestamp}")
+                node.execute_node(
+                    run_location=self.flow_settings.execution_location,
+                    performance_mode=performance_mode,
+                    node_logger=node_logger,
+                )
                 try:
                     node_result.error = str(node.results.errors)
                     if self.flow_settings.is_canceled:
@@ -1758,22 +2106,22 @@ class FlowGraph:
                     node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
                     node_result.is_running = False
                 except Exception as e:
-                    node_result.error = 'Node did not run'
+                    node_result.error = "Node did not run"
                     node_result.success = False
                     node_result.end_timestamp = time()
                     node_result.run_time = int(node_result.end_timestamp - node_result.start_timestamp)
                     node_result.is_running = False
-                    node_logger.error(f'Error in node {node.node_id}: {e}')
+                    node_logger.error(f"Error in node {node.node_id}: {e}")
                 if not node_result.success:
                     skip_nodes.extend(list(node.get_all_dependent_nodes()))
-                node_logger.info(f'Completed node with success: {node_result.success}')
+                node_logger.info(f"Completed node with success: {node_result.success}")
                 self.latest_run_info.nodes_completed += 1
             self.latest_run_info.end_time = datetime.datetime.now()
-            self.flow_logger.info('Flow completed!')
+            self.flow_logger.info("Flow completed!")
             self.end_datetime = datetime.datetime.now()
             self.flow_settings.is_running = False
             if self.flow_settings.is_canceled:
-                self.flow_logger.info('Flow canceled')
+                self.flow_logger.info("Flow canceled")
             return self.get_run_info()
         except Exception as e:
             raise e
@@ -1799,7 +2147,7 @@ class FlowGraph:
         return run_info
 
     @property
-    def node_connections(self) -> List[Tuple[int, int]]:
+    def node_connections(self) -> list[tuple[int, int]]:
         """Computes and returns a list of all connections in the graph.
 
         Returns:
@@ -1809,8 +2157,9 @@ class FlowGraph:
         for node in self.nodes:
             outgoing_connections = [(node.node_id, ltn.node_id) for ltn in node.leads_to_nodes]
             incoming_connections = [(don.node_id, node.node_id) for don in node.all_inputs]
-            node_connections = [c for c in outgoing_connections + incoming_connections if (c[0] is not None
-                                                                                           and c[1] is not None)]
+            node_connections = [
+                c for c in outgoing_connections + incoming_connections if (c[0] is not None and c[1] is not None)
+            ]
             for node_connection in node_connections:
                 if node_connection not in connections:
                     connections.add(node_connection)
@@ -1871,16 +2220,18 @@ class FlowGraph:
         Returns:
             A FlowInformation object representing the complete graph.
         """
-        node_information = {node.node_id: node.get_node_information() for
-                            node in self.nodes if node.is_setup and node.is_correct}
+        node_information = {
+            node.node_id: node.get_node_information() for node in self.nodes if node.is_setup and node.is_correct
+        }
 
-        return schemas.FlowInformation(flow_id=self.flow_id,
-                                       flow_name=self.__name__,
-                                       flow_settings=self.flow_settings,
-                                       data=node_information,
-                                       node_starts=[v.node_id for v in self._flow_starts],
-                                       node_connections=self.node_connections
-                                       )
+        return schemas.FlowInformation(
+            flow_id=self.flow_id,
+            flow_name=self.__name__,
+            flow_settings=self.flow_settings,
+            data=node_information,
+            node_starts=[v.node_id for v in self._flow_starts],
+            node_connections=self.node_connections,
+        )
 
     def cancel(self):
         """Cancels an ongoing graph execution."""
@@ -1901,7 +2252,11 @@ class FlowGraph:
         """
         Handle the rename of a flow when it is being saved.
         """
-        if self.flow_settings and self.flow_settings.path and Path(self.flow_settings.path).absolute() != new_path.absolute():
+        if (
+            self.flow_settings
+            and self.flow_settings.path
+            and Path(self.flow_settings.path).absolute() != new_path.absolute()
+        ):
             self.__name__ = new_name
             self.flow_settings.save_location = str(new_path.absolute())
             self.flow_settings.name = new_name
@@ -1928,27 +2283,27 @@ class FlowGraph:
         self._handle_flow_renaming(new_flow_name, path)
         self.flow_settings.modified_on = datetime.datetime.now().timestamp()
         try:
-            if suffix == '.flowfile':
+            if suffix == ".flowfile":
                 raise DeprecationWarning(
-                    f"The .flowfile format is deprecated. Please use .yaml or .json formats.\n\n"
+                    "The .flowfile format is deprecated. Please use .yaml or .json formats.\n\n"
                     "Or stay on v0.4.1 if you still need .flowfile support.\n\n"
                 )
-            elif suffix in ('.yaml', '.yml'):
+            elif suffix in (".yaml", ".yml"):
                 flowfile_data = self.get_flowfile_data()
-                data = flowfile_data.model_dump(mode='json')
-                with open(flow_path, 'w', encoding='utf-8') as f:
+                data = flowfile_data.model_dump(mode="json")
+                with open(flow_path, "w", encoding="utf-8") as f:
                     yaml.dump(data, f, default_flow_style=False, sort_keys=False, allow_unicode=True)
-            elif suffix == '.json':
+            elif suffix == ".json":
                 flowfile_data = self.get_flowfile_data()
-                data = flowfile_data.model_dump(mode='json')
-                with open(flow_path, 'w', encoding='utf-8') as f:
+                data = flowfile_data.model_dump(mode="json")
+                with open(flow_path, "w", encoding="utf-8") as f:
                     json.dump(data, f, indent=2, ensure_ascii=False)
 
             else:
                 flowfile_data = self.get_flowfile_data()
                 logger.warning(f"Unknown file extension {suffix}. Defaulting to YAML format.")
-                data = flowfile_data.model_dump(mode='json')
-                with open(flow_path, 'w', encoding='utf-8') as f:
+                data = flowfile_data.model_dump(mode="json")
+                with open(flow_path, "w", encoding="utf-8") as f:
                     yaml.dump(data, f, default_flow_style=False, sort_keys=False, allow_unicode=True)
 
         except Exception as e:
@@ -1966,11 +2321,7 @@ class FlowGraph:
         Returns:
             A dictionary representing the graph in Drawflow format.
         """
-        result = {
-            'Home': {
-                "data": {}
-            }
-        }
+        result = {"Home": {"data": {}}}
         flow_info: schemas.FlowInformation = self.get_node_storage()
 
         for node_id, node_info in flow_info.data.items():
@@ -1989,7 +2340,7 @@ class FlowGraph:
                         "inputs": {},
                         "outputs": {},
                         "pos_x": pos_x,
-                        "pos_y": pos_y
+                        "pos_y": pos_y,
                     }
                 except Exception as e:
                     logger.error(e)
@@ -2003,24 +2354,27 @@ class FlowGraph:
                     leading_to_node = self.get_node(output_node_id)
                     input_types = leading_to_node.get_input_type(node_info.id)
                     for input_type in input_types:
-                        if input_type == 'main':
-                            input_frontend_id = 'input_1'
-                        elif input_type == 'right':
-                            input_frontend_id = 'input_2'
-                        elif input_type == 'left':
-                            input_frontend_id = 'input_3'
+                        if input_type == "main":
+                            input_frontend_id = "input_1"
+                        elif input_type == "right":
+                            input_frontend_id = "input_2"
+                        elif input_type == "left":
+                            input_frontend_id = "input_3"
                         else:
-                            input_frontend_id = 'input_1'
+                            input_frontend_id = "input_1"
                         connection = {"node": str(output_node_id), "input": input_frontend_id}
                         connections.append(connection)
 
-                result["Home"]["data"][str(node_id)]["outputs"]["output_1"] = {
-                    "connections": connections}
+                result["Home"]["data"][str(node_id)]["outputs"]["output_1"] = {"connections": connections}
             else:
                 result["Home"]["data"][str(node_id)]["outputs"] = {"output_1": {"connections": []}}
 
             # Add input to the node based on `depending_on_id` in your backend data
-            if node_info.left_input_id is not None or node_info.right_input_id is not None or node_info.input_ids is not None:
+            if (
+                node_info.left_input_id is not None
+                or node_info.right_input_id is not None
+                or node_info.input_ids is not None
+            ):
                 main_inputs = node_info.main_input_ids
                 result["Home"]["data"][str(node_id)]["inputs"]["input_1"] = {
                     "connections": [{"node": str(main_node_id), "input": "output_1"} for main_node_id in main_inputs]
@@ -2041,8 +2395,8 @@ class FlowGraph:
         Returns:
             A VueFlowInput object.
         """
-        edges: List[schemas.NodeEdge] = []
-        nodes: List[schemas.NodeInput] = []
+        edges: list[schemas.NodeEdge] = []
+        nodes: list[schemas.NodeInput] = []
         for node in self.nodes:
             nodes.append(node.get_node_input())
             edges.extend(node.get_edge_input())
@@ -2054,7 +2408,9 @@ class FlowGraph:
         for node in self.nodes:
             node.reset(True)
 
-    def copy_node(self, new_node_settings: input_schema.NodePromise, existing_setting_input: Any, node_type: str) -> None:
+    def copy_node(
+        self, new_node_settings: input_schema.NodePromise, existing_setting_input: Any, node_type: str
+    ) -> None:
         """Creates a copy of an existing node.
 
         Args:
@@ -2067,9 +2423,7 @@ class FlowGraph:
         if isinstance(existing_setting_input, input_schema.NodePromise):
             return
 
-        combined_settings = combine_existing_settings_and_new_settings(
-            existing_setting_input, new_node_settings
-        )
+        combined_settings = combine_existing_settings_and_new_settings(existing_setting_input, new_node_settings)
         getattr(self, f"add_{node_type}")(combined_settings)
 
     def generate_code(self):
@@ -2077,6 +2431,7 @@ class FlowGraph:
         This method exports the flow graph to a Polars-compatible format.
         """
         from flowfile_core.flowfile.code_generator.code_generator import export_flow_to_polars
+
         print(export_flow_to_polars(self))
 
 
@@ -2095,13 +2450,7 @@ def combine_existing_settings_and_new_settings(setting_input: Any, new_settings:
     copied_setting_input = deepcopy(setting_input)
 
     # Update only attributes that exist on new_settings
-    fields_to_update = (
-        "node_id",
-        "pos_x",
-        "pos_y",
-        "description",
-        "flow_id"
-    )
+    fields_to_update = ("node_id", "pos_x", "pos_y", "description", "flow_id")
 
     for field in fields_to_update:
         if hasattr(new_settings, field) and getattr(new_settings, field) is not None:
@@ -2117,12 +2466,12 @@ def add_connection(flow: FlowGraph, node_connection: input_schema.NodeConnection
         flow: The FlowGraph instance to modify.
         node_connection: An object defining the source and target of the connection.
     """
-    logger.info('adding a connection')
+    logger.info("adding a connection")
     from_node = flow.get_node(node_connection.output_connection.node_id)
     to_node = flow.get_node(node_connection.input_connection.node_id)
-    logger.info(f'from_node={from_node}, to_node={to_node}')
+    logger.info(f"from_node={from_node}, to_node={to_node}")
     if not (from_node and to_node):
-        raise HTTPException(404, 'Not not available')
+        raise HTTPException(404, "Not not available")
     else:
         to_node.add_node_connection(from_node, node_connection.input_connection.get_node_input_connection_type())